diff options
Diffstat (limited to 'meta-facebook/meta-wedge/recipes-wedge/fan-ctrl/fan-ctrl')
9 files changed, 1365 insertions, 0 deletions
diff --git a/meta-facebook/meta-wedge/recipes-wedge/fan-ctrl/fan-ctrl/Makefile b/meta-facebook/meta-wedge/recipes-wedge/fan-ctrl/fan-ctrl/Makefile new file mode 100644 index 0000000..fa9a8f4 --- /dev/null +++ b/meta-facebook/meta-wedge/recipes-wedge/fan-ctrl/fan-ctrl/Makefile @@ -0,0 +1,10 @@ +# Copyright 2014-present Facebook. All Rights Reserved. +all: fand + +fand: fand.cpp watchdog.cpp + $(CXX) -pthread -o $@ $^ $(LDFLAGS) -lwedge_eeprom + +.PHONY: clean + +clean: + rm -rf *.o fand diff --git a/meta-facebook/meta-wedge/recipes-wedge/fan-ctrl/fan-ctrl/README b/meta-facebook/meta-wedge/recipes-wedge/fan-ctrl/fan-ctrl/README new file mode 100644 index 0000000..2a92b9d --- /dev/null +++ b/meta-facebook/meta-wedge/recipes-wedge/fan-ctrl/fan-ctrl/README @@ -0,0 +1,5 @@ +The AST PWM/Tach driver is in the kernel sources at drivers/hwmon/ast_pwm_fan.c + +There are 7 PWM output pins. Each PWM can be configured in one of 3 types (M, +N, or O). The clock settings for each type are configurable. See init_pwm.sh +for more comments about how we configure the settings. diff --git a/meta-facebook/meta-wedge/recipes-wedge/fan-ctrl/fan-ctrl/fand.cpp b/meta-facebook/meta-wedge/recipes-wedge/fan-ctrl/fan-ctrl/fand.cpp new file mode 100644 index 0000000..24e107c --- /dev/null +++ b/meta-facebook/meta-wedge/recipes-wedge/fan-ctrl/fan-ctrl/fand.cpp @@ -0,0 +1,851 @@ +/* + * fand + * + * Copyright 2014-present Facebook. All Rights Reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. + * + * Daemon to manage the fan speed to ensure that we stay within a reasonable + * temperature range. We're using a simplistic algorithm to get started: + * + * If the fan is already on high, we'll move it to medium if we fall below + * a top temperature. If we're on medium, we'll move it to high + * if the temperature goes over the top value, and to low if the + * temperature falls to a bottom level. If the fan is on low, + * we'll increase the speed if the temperature rises to the top level. + * + * To ensure that we're not just turning the fans up, then back down again, + * we'll require an extra few degrees of temperature drop before we lower + * the fan speed. + * + * We check the RPM of the fans against the requested RPMs to determine + * whether the fans are failing, in which case we'll turn up all of + * the other fans and report the problem.. + * + * TODO: Implement a PID algorithm to closely track the ideal temperature. + * TODO: Determine if the daemon is already started. + */ + +/* Yeah, the file ends in .cpp, but it's a C program. Deal. */ + +#include <stdio.h> +#include <stdlib.h> +#include <unistd.h> +#include <string.h> +#include <errno.h> +#include <signal.h> +#include <syslog.h> +#include "watchdog.h" + +#include "facebook/wedge_eeprom.h" + +/* Sensor definitions */ + +#define INTERNAL_TEMPS(x) ((x) * 1000) // stored a C * 1000 +#define EXTERNAL_TEMPS(x) ((x) / 1000) + +#define I2C_BUS_3_DIR "/sys/class/i2c-adapter/i2c-3/" +#define I2C_BUS_4_DIR "/sys/class/i2c-adapter/i2c-4/" + +#define INTAKE_TEMP_DEVICE I2C_BUS_3_DIR "3-0048" +#define T2_TEMP_DEVICE I2C_BUS_3_DIR "3-0049" +#define EXHAUST_TEMP_DEVICE I2C_BUS_3_DIR "3-004a" +#define USERVER_TEMP_DEVICE I2C_BUS_4_DIR "4-0040" + +/* + * The sensor for the uServer CPU is not on the CPU itself, so it reads + * a little low. We are special casing this, but we should obviously + * be thinking about a way to generalize these tweaks, and perhaps + * the entire configuration. JSON file? + */ + +#define USERVER_TEMP_FUDGE INTERNAL_TEMPS(10) + +#define BAD_TEMP INTERNAL_TEMPS(-60) + +#define BAD_READ_THRESHOLD 4 /* How many times can reads fail */ +#define FAN_FAILURE_THRESHOLD 4 /* How many times can a fan fail */ + +#define PWM_DIR "/sys/devices/platform/ast_pwm_tacho.0" + +#define PWM_UNIT_MAX 96 + +#define LARGEST_DEVICE_NAME 120 + +#define GPIO_USERVER_POWER_DIRECTION "/sys/class/gpio/gpio25/direction" +#define GPIO_USERVER_POWER "/sys/class/gpio/gpio25/value" +#define GPIO_T2_POWER_DIRECTION "/sys/class/gpio/gpio41/direction" +#define GPIO_T2_POWER "/sys/class/gpio/gpio41/value" + +#define GPIO_FAN0_LED "/sys/class/gpio/gpio53/value" +#define GPIO_FAN1_LED "/sys/class/gpio/gpio54/value" +#define GPIO_FAN2_LED "/sys/class/gpio/gpio55/value" +#define GPIO_FAN3_LED "/sys/class/gpio/gpio72/value" + +const char *fan_led[] = {GPIO_FAN0_LED, GPIO_FAN1_LED, + GPIO_FAN2_LED, GPIO_FAN3_LED}; + +#define FAN_LED_RED "0" +#define FAN_LED_BLUE "1" + +#define GPIO_PIN_ID "/sys/class/gpio/gpio%d/value" +#define REV_IDS 3 +#define GPIO_REV_ID_START 192 + +#define BOARD_IDS 4 +#define GPIO_BOARD_ID_START 160 + +/* + * With hardware revisions after 3, we use a different set of pins for + * the BOARD_ID. + */ + +#define REV_ID_NEW_BOARD_ID 3 +#define GPIO_BOARD_ID_START_NEW 166 + +#define REPORT_TEMP 720 /* Report temp every so many cycles */ + +/* Sensor limits and tuning parameters */ + +#define INTAKE_LIMIT INTERNAL_TEMPS(60) +#define T2_LIMIT INTERNAL_TEMPS(95) +#define USERVER_LIMIT INTERNAL_TEMPS(75) + +#define TEMP_TOP INTERNAL_TEMPS(70) +#define TEMP_BOTTOM INTERNAL_TEMPS(40) + +/* + * Toggling the fan constantly will wear it out (and annoy anyone who + * can hear it), so we'll only turn down the fan after the temperature + * has dipped a bit below the point at which we'd otherwise switch + * things up. + */ + +#define COOLDOWN_SLOP INTERNAL_TEMPS(6) + +#define FAN_LOW 35 +#define FAN_MEDIUM 50 +#define FAN_HIGH 70 +#define FAN_MAX 99 + +/* + * Mapping physical to hardware addresses for fans; it's different for + * RPM measuring and PWM setting, naturally. Doh. + */ + +int fan_to_rpm_map[] = {3, 2, 0, 1}; +int fan_to_pwm_map[] = {7, 6, 0, 1}; + +#define FANS 4 + +/* + * The measured RPM of the fans doesn't match linearly to the requested + * rate. In addition, there are coaxially mounted fans, so the rear fans + * feed into the front fans. The rear fans will run slower since they're + * grabbing still air, and the front fants are getting an extra boost. + * + * We'd like to measure the fan RPM and compare it to the expected RPM + * so that we can detect failed fans, so we have a table (derived from + * hardware testing): + */ + +struct rpm_to_pct_map { + ushort pct; + ushort rpm; +}; + +struct rpm_to_pct_map rpm_front_map[] = {{30, 6150}, + {35, 7208}, + {40, 8195}, + {45, 9133}, + {50, 10017}, + {55, 10847}, + {60, 11612}, + {65, 12342}, + {70, 13057}, + {75, 13717}, + {80, 14305}, + {85, 14869}, + {90, 15384}, + {95, 15871}, + {100, 16095}}; + +struct rpm_to_pct_map rpm_rear_map[] = {{30, 3911}, + {35, 4760}, + {40, 5587}, + {45, 6434}, + {50, 7295}, + {55, 8187}, + {60, 9093}, + {65, 10008}, + {70, 10949}, + {75, 11883}, + {80, 12822}, + {85, 13726}, + {90, 14690}, + {95, 15516}, + {100, 15897}}; + +#define FAN_FAILURE_OFFSET 30 + +int fan_low = FAN_LOW; +int fan_medium = FAN_MEDIUM; +int fan_high = FAN_HIGH; +int fan_max = FAN_MAX; +int total_fans = FANS; +int fan_offset = 0; + +int temp_bottom = TEMP_BOTTOM; +int temp_top = TEMP_TOP; + +int report_temp = REPORT_TEMP; +bool verbose = false; + +void usage() { + fprintf(stderr, + "fand [-v] [-l <low-pct>] [-m <medium-pct>] " + "[-h <high-pct>]\n" + "\t[-b <temp-bottom>] [-t <temp-top>] [-r <report-temp>]\n\n" + "\tlow-pct defaults to %d%% fan\n" + "\tmedium-pct defaults to %d%% fan\n" + "\thigh-pct defaults to %d%% fan\n" + "\ttemp-bottom defaults to %dC\n" + "\ttemp-top defaults to %dC\n" + "\treport-temp defaults to every %d measurements\n\n" + "fand compensates for uServer temperature reading %d degrees low\n" + "kill with SIGUSR1 to stop watchdog\n", + fan_low, + fan_medium, + fan_high, + EXTERNAL_TEMPS(temp_bottom), + EXTERNAL_TEMPS(temp_top), + report_temp, + EXTERNAL_TEMPS(USERVER_TEMP_FUDGE)); + exit(1); +} + +/* We need to open the device each time to read a value */ + +int read_device(const char *device, int *value) { + FILE *fp; + int rc; + + fp = fopen(device, "r"); + if (!fp) { + int err = errno; + + syslog(LOG_INFO, "failed to open device %s", device); + return err; + } + + rc = fscanf(fp, "%d", value); + fclose(fp); + + if (rc != 1) { + syslog(LOG_INFO, "failed to read device %s", device); + return ENOENT; + } else { + return 0; + } +} + +/* We need to open the device again each time to write a value */ + +int write_device(const char *device, const char *value) { + FILE *fp; + int rc; + + fp = fopen(device, "w"); + if (!fp) { + int err = errno; + + syslog(LOG_INFO, "failed to open device for write %s", device); + return err; + } + + rc = fputs(value, fp); + fclose(fp); + + if (rc < 0) { + syslog(LOG_INFO, "failed to write device %s", device); + return ENOENT; + } else { + return 0; + } +} + +int read_temp(const char *device, int *value) { + char full_name[LARGEST_DEVICE_NAME + 1]; + + /* We set an impossible value to check for errors */ + *value = BAD_TEMP; + snprintf( + full_name, LARGEST_DEVICE_NAME, "%s/temp1_input", device); + return read_device(full_name, value); +} + +int read_gpio_value(const int id, const char *device, int *value) { + char full_name[LARGEST_DEVICE_NAME]; + + snprintf(full_name, LARGEST_DEVICE_NAME, device, id); + return read_device(full_name, value); +} + +int read_gpio_values(const int start, const int count, + const char *device, int *result) { + int status = 0; + int value; + + *result = 0; + for (int i = 0; i < count; i++) { + status |= read_gpio_value(start + i, GPIO_PIN_ID, &value); + *result |= value << i; + } + return status; +} + +int read_ids(int *rev_id, int *board_id) { + int status = 0; + int value; + + status = read_gpio_values(GPIO_REV_ID_START, REV_IDS, GPIO_PIN_ID, rev_id); + if (status != 0) { + syslog(LOG_INFO, "failed to read rev_id"); + return status; + } + + int board_id_start; + if (*rev_id >= REV_ID_NEW_BOARD_ID) { + board_id_start = GPIO_BOARD_ID_START_NEW; + } else { + board_id_start = GPIO_BOARD_ID_START; + } + + status = read_gpio_values(board_id_start, BOARD_IDS, GPIO_PIN_ID, board_id); + if (status != 0) { + syslog(LOG_INFO, "failed to read board_id"); + } + return status; +} + +bool is_two_fan_board(bool verbose) { + struct wedge_eeprom_st eeprom; + /* Retrieve the board type from EEPROM */ + if (wedge_eeprom_parse(NULL, &eeprom) == 0) { + /* able to parse EEPROM */ + if (verbose) { + syslog(LOG_INFO, "board type is %s", eeprom.fbw_location); + } + /* only WEDGE is NOT two-fan board */ + return strncasecmp(eeprom.fbw_location, "wedge", + sizeof(eeprom.fbw_location)); + } else { + int status; + int board_id = 0; + int rev_id = 0; + /* + * Could not parse EEPROM. Most likely, it is an old HW without EEPROM. + * In this case, use board ID to distinguish if it is wedge or 6-pack. + */ + status = read_ids(&rev_id, &board_id); + if (verbose) { + syslog(LOG_INFO, "rev ID %d, board id %d", rev_id, board_id); + } + if (status == 0 && board_id != 0xf) { + return true; + } else { + return false; + } + } +} + +int read_fan_value(const int fan, const char *device, int *value) { + char device_name[LARGEST_DEVICE_NAME]; + char output_value[LARGEST_DEVICE_NAME]; + char full_name[LARGEST_DEVICE_NAME]; + + snprintf(device_name, LARGEST_DEVICE_NAME, device, fan); + snprintf(full_name, LARGEST_DEVICE_NAME, "%s/%s", PWM_DIR, device_name); + return read_device(full_name, value); +} + +int write_fan_value(const int fan, const char *device, const int value) { + char full_name[LARGEST_DEVICE_NAME]; + char device_name[LARGEST_DEVICE_NAME]; + char output_value[LARGEST_DEVICE_NAME]; + + snprintf(device_name, LARGEST_DEVICE_NAME, device, fan); + snprintf(full_name, LARGEST_DEVICE_NAME, "%s/%s", PWM_DIR, device_name); + snprintf(output_value, LARGEST_DEVICE_NAME, "%d", value); + return write_device(full_name, output_value); +} + +/* Return fan speed as a percentage of maximum -- not necessarily linear. */ + +int fan_rpm_to_pct(const struct rpm_to_pct_map *table, + const int table_len, + int rpm) { + int i; + + for (i = 0; i < table_len; i++) { + if (table[i].rpm > rpm) { + break; + } + } + + /* + * If the fan RPM is lower than the lowest value in the table, + * we may have a problem -- fans can only go so slow, and it might + * have stopped. In this case, we'll return an interpolated + * percentage, as just returning zero is even more problematic. + */ + + if (i == 0) { + return (rpm * table[i].pct) / table[i].rpm; + } else if (i == table_len) { // Fell off the top? + return table[i - 1].pct; + } + + // Interpolate the right percentage value: + + int percent_diff = table[i].pct - table[i - 1].pct; + int rpm_diff = table[i].rpm - table[i - 1].rpm; + int fan_diff = table[i].rpm - rpm; + + return table[i].pct - (fan_diff * percent_diff / rpm_diff); +} + +int fan_speed_okay(const int fan, const int speed, const int slop) { + int front_fan, rear_fan; + int front_pct, rear_pct; + int real_fan; + int okay; + + /* + * The hardware fan numbers are different from the physical order + * in the box, so we have to map them: + */ + + real_fan = fan_to_rpm_map[fan]; + + front_fan = 0; + rear_fan = 0; + read_fan_value(real_fan, "tacho%d_rpm", &front_fan); + read_fan_value(real_fan + 4, "tacho%d_rpm", &rear_fan); + + front_pct = + fan_rpm_to_pct(rpm_front_map, + sizeof(rpm_front_map) / sizeof(struct rpm_to_pct_map), + front_fan); + rear_pct = + fan_rpm_to_pct(rpm_rear_map, + sizeof(rpm_rear_map) / sizeof(struct rpm_to_pct_map), + rear_fan); + + /* + * If the fans are broken, the measured rate will be rather + * different from the requested rate, and we can turn up the + * rest of the fans to compensate. The slop is the percentage + * of error that we'll tolerate. + * + * XXX: I suppose that we should only measure negative values; + * running too fast isn't really a problem. + */ + + okay = (abs(front_pct - speed) * 100 / speed < slop && + abs(rear_pct - speed) * 100 / speed < slop); + + if (!okay || verbose) { + syslog(!okay ? LOG_ALERT : LOG_INFO, + "fan %d rear %d (%d%%), front %d (%d%%), expected %d", + fan, + rear_fan, + rear_pct, + front_fan, + front_pct, + speed); + } + + return okay; +} + +/* Set fan speed as a percentage */ + +int write_fan_speed(const int fan, const int value) { + /* + * The hardware fan numbers for pwm control are different from + * both the physical order in the box, and the mapping for reading + * the RPMs per fan, above. + */ + + int real_fan = fan_to_pwm_map[fan]; + + if (value == 0) { + return write_fan_value(real_fan, "pwm%d_en", 0); + } else { + int unit = value * PWM_UNIT_MAX / 100; + int status; + + if (unit == PWM_UNIT_MAX) + unit = 0; + + if ((status = write_fan_value(real_fan, "pwm%d_type", 0)) != 0 || + (status = write_fan_value(real_fan, "pwm%d_rising", 0)) != 0 || + (status = write_fan_value(real_fan, "pwm%d_falling", unit)) != 0 || + (status = write_fan_value(real_fan, "pwm%d_en", 1)) != 0) { + return status; + } + } +} + +/* Set up fan LEDs */ + +int write_fan_led(const int fan, const char *color) +{ + return write_device(fan_led[fan], color); +} + +int server_shutdown(const char *why) { + int fan; + for (fan = 0; fan < total_fans; fan++) { + write_fan_speed(fan + fan_offset, fan_max); + } + + syslog(LOG_EMERG, "Shutting down: %s", why); + write_device(GPIO_USERVER_POWER_DIRECTION, "out"); + write_device(GPIO_USERVER_POWER, "0"); + /* + * Putting T2 in reset generating a non-maskable interrupt to uS, + * the kernel running on uS might panic depending on its version. + * sleep 5s here to make sure uS is completely down. + */ + sleep(5); + write_device(GPIO_T2_POWER_DIRECTION, "out"); + write_device(GPIO_T2_POWER, "0"); + + /* + * We have to stop the watchdog, or the system will be automatically + * rebooted some seconds after fand exits (and stops kicking the + * watchdog). + */ + + stop_watchdog(); + + sleep(2); + exit(2); +} + +/* Gracefully shut down on receipt of a signal */ + +void fand_interrupt(int sig) +{ + int fan; + for (fan = 0; fan < total_fans; fan++) { + write_fan_speed(fan + fan_offset, fan_max); + } + + syslog(LOG_ALERT, "Shutting down fand on signal %s", strsignal(sig)); + if (sig == SIGUSR1) { + stop_watchdog(); + } + exit(3); +} + +int main(int argc, char **argv) { + /* Sensor values */ + + int intake_temp; + int exhaust_temp; + int t2_temp; + int userver_temp; + + int fan_speed = FAN_HIGH; + int bad_reads = 0; + int fan_failure = 0; + int fan_speed_changes = 0; + int old_speed; + + int fan_bad[FANS]; + int fan; + + unsigned log_count = 0; // How many times have we logged our temps? + int opt; + int prev_fans_bad = 0; + + struct sigaction sa; + + sa.sa_handler = fand_interrupt; + sa.sa_flags = 0; + sigemptyset(&sa.sa_mask); + + sigaction(SIGTERM, &sa, NULL); + sigaction(SIGINT, &sa, NULL); + sigaction(SIGUSR1, &sa, NULL); + + while ((opt = getopt(argc, argv, "l:m:h:b:t:r:v")) != -1) { + switch (opt) { + case 'l': + fan_low = atoi(optarg); + break; + case 'm': + fan_medium = atoi(optarg); + break; + case 'h': + fan_high = atoi(optarg); + break; + case 'b': + temp_bottom = INTERNAL_TEMPS(atoi(optarg)); + break; + case 't': + temp_top = INTERNAL_TEMPS(atoi(optarg)); + break; + case 'r': + report_temp = atoi(optarg); + break; + case 'v': + verbose = true; + break; + default: + usage(); + break; + } + } + + if (optind > argc) { + usage(); + } + + if (temp_bottom > temp_top) { + fprintf(stderr, + "Should temp-bottom (%d) be higher than " + "temp-top (%d)? Starting anyway.\n", + EXTERNAL_TEMPS(temp_bottom), + EXTERNAL_TEMPS(temp_top)); + } + + if (fan_low > fan_medium || fan_low > fan_high || fan_medium > fan_high) { + fprintf(stderr, + "fan RPMs not strictly increasing " + "-- %d, %d, %d, starting anyway\n", + fan_low, + fan_medium, + fan_high); + } + + daemon(1, 0); + openlog("fand", LOG_CONS, LOG_DAEMON); + + /* Start watchdog in manual mode */ + start_watchdog(0); + + /* Set watchdog to persistent mode so timer expiry will happen independent + * of this process's liveliness. */ + set_persistent_watchdog(WATCHDOG_SET_PERSISTENT); + + if (is_two_fan_board(verbose)) { + /* Alternate, two fan configuration */ + total_fans = 2; + fan_offset = 2; /* fan 3 is the first */ + } + + for (fan = 0; fan < total_fans; fan++) { + fan_bad[fan] = 0; + write_fan_speed(fan + fan_offset, fan_speed); + write_fan_led(fan + fan_offset, FAN_LED_BLUE); + } + + sleep(5); /* Give the fans time to come up to speed */ + + while (1) { + int max_temp; + old_speed = fan_speed; + + /* Read sensors */ + + read_temp(INTAKE_TEMP_DEVICE, &intake_temp); + read_temp(EXHAUST_TEMP_DEVICE, &exhaust_temp); + read_temp(T2_TEMP_DEVICE, &t2_temp); + read_temp(USERVER_TEMP_DEVICE, &userver_temp); + + /* + * uServer can be powered down, but all of the rest of the sensors + * should be readable at any time. + */ + + if ((intake_temp == BAD_TEMP || exhaust_temp == BAD_TEMP || + t2_temp == BAD_TEMP)) { + bad_reads++; + } + + if (bad_reads > BAD_READ_THRESHOLD) { + server_shutdown("Some sensors couldn't be read"); + } + + if (log_count++ % report_temp == 0) { + syslog(LOG_DEBUG, + "Temp intake %d, t2 %d, userver %d, exhaust %d, " + "fan speed %d, speed changes %d", + intake_temp, + t2_temp, + userver_temp, + exhaust_temp, + fan_speed, + fan_speed_changes); + } + + /* Protection heuristics */ + + if (intake_temp > INTAKE_LIMIT) { + server_shutdown("Intake temp limit reached"); + } + + if (t2_temp > T2_LIMIT) { + server_shutdown("T2 temp limit reached"); + } + + if (userver_temp + USERVER_TEMP_FUDGE > USERVER_LIMIT) { + server_shutdown("uServer temp limit reached"); + } + + /* + * Calculate change needed -- we should eventually + * do something more sophisticated, like PID. + * + * We should use the intake temperature to adjust this + * as well. + */ + + if (t2_temp > userver_temp + USERVER_TEMP_FUDGE) { + max_temp = t2_temp; + } else { + max_temp = userver_temp + USERVER_TEMP_FUDGE; + } + + /* + * If recovering from a fan problem, spin down fans gradually in case + * temperatures are still high. Gradual spin down also reduces wear on + * the fans. + */ + if (fan_speed == fan_max) { + if (fan_failure == 0) { + fan_speed = fan_high; + } + } else if (fan_speed == fan_high) { + if (max_temp + COOLDOWN_SLOP < temp_top) { + fan_speed = fan_medium; + } + } else if (fan_speed == fan_medium) { + if (max_temp > temp_top) { + fan_speed = fan_high; + } else if (max_temp + COOLDOWN_SLOP < temp_bottom) { + fan_speed = fan_low; + } + } else {/* low */ + if (max_temp > temp_bottom) { + fan_speed = fan_medium; + } + } + + /* + * Update fans only if there are no failed ones. If any fans failed + * earlier, all remaining fans should continue to run at max speed. + */ + + if (fan_failure == 0 && fan_speed != old_speed) { + syslog(LOG_NOTICE, + "Fan speed changing from %d to %d", + old_speed, + fan_speed); + fan_speed_changes++; + for (fan = 0; fan < total_fans; fan++) { + write_fan_speed(fan + fan_offset, fan_speed); + } + } + + /* + * Wait for some change. Typical I2C temperature sensors + * only provide a new value every second and a half, so + * checking again more quickly than that is a waste. + * + * We also have to wait for the fan changes to take effect + * before measuring them. + */ + + sleep(5); + + /* Check fan RPMs */ + + for (fan = 0; fan < total_fans; fan++) { + /* + * Make sure that we're within some percentage + * of the requested speed. + */ + if (fan_speed_okay(fan + fan_offset, fan_speed, FAN_FAILURE_OFFSET)) { + if (fan_bad[fan] > FAN_FAILURE_THRESHOLD) { + write_fan_led(fan + fan_offset, FAN_LED_BLUE); + syslog(LOG_NOTICE, + "Fan %d has recovered", + fan); + } + fan_bad[fan] = 0; + } else { + fan_bad[fan]++; + } + } + + fan_failure = 0; + for (fan = 0; fan < total_fans; fan++) { + if (fan_bad[fan] > FAN_FAILURE_THRESHOLD) { + fan_failure++; + write_fan_led(fan + fan_offset, FAN_LED_RED); + } + } + + if (fan_failure > 0) { + if (prev_fans_bad != fan_failure) { + syslog(LOG_ALERT, "%d fans failed", fan_failure); + } + + /* + * If fans are bad, we need to blast all of the + * fans at 100%; we don't bother to turn off + * the bad fans, in case they are all that is left. + * + * Note that we have a temporary bug with setting fans to + * 100% so we only do fan_max = 99%. + */ + + fan_speed = fan_max; + for (fan = 0; fan < total_fans; fan++) { + write_fan_speed(fan + fan_offset, fan_speed); + } + + /* + * Fans can be hot swapped and replaced; in which case the fan daemon + * will automatically detect the new fan and (assuming the new fan isn't + * itself faulty), automatically readjust the speeds for all fans down + * to a more suitable rpm. The fan daemon does not need to be restarted. + */ + } + + /* Suppress multiple warnings for similar number of fan failures. */ + prev_fans_bad = fan_failure; + + /* if everything is fine, restart the watchdog countdown. If this process + * is terminated, the persistent watchdog setting will cause the system + * to reboot after the watchdog timeout. */ + kick_watchdog(); + } +} diff --git a/meta-facebook/meta-wedge/recipes-wedge/fan-ctrl/fan-ctrl/get_fan_speed.sh b/meta-facebook/meta-wedge/recipes-wedge/fan-ctrl/fan-ctrl/get_fan_speed.sh new file mode 100755 index 0000000..4e36a7c --- /dev/null +++ b/meta-facebook/meta-wedge/recipes-wedge/fan-ctrl/fan-ctrl/get_fan_speed.sh @@ -0,0 +1,59 @@ +#!/bin/sh +# +# Copyright 2004-present Facebook. All Rights Reserved. +# +# This program file is free software; you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the +# Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License +# for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program in a file named COPYING; if not, write to the +# Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, +# Boston, MA 02110-1301 USA +# +usage() { + echo "Usage: $0 [Fan Unit (0..3)]" >&2 +} + +PWM_DIR=/sys/devices/platform/ast_pwm_tacho.0 +set -e + +# refer to the comments in init_pwn.sh regarding +# the fan unit and tacho mapping +if [ "$#" -eq 0 ]; then + TACHOS="0:3 1:2 2:0 3:1" +elif [ "$#" -eq 1 ]; then + case "$1" in + "0") + TACHOS="0:3" + ;; + "1") + TACHOS="1:2" + ;; + "2") + TACHOS="2:0" + ;; + "3") + TACHOS="3:1" + ;; + *) + usage + exit 1 + ;; + esac +else + usage + exit 1 +fi + +for fan_tacho in $TACHOS; do + fan=${fan_tacho%%:*} + tacho=${fan_tacho##*:} + echo "Fan $fan RPMs: $(cat $PWM_DIR/tacho${tacho}_rpm), $(cat $PWM_DIR/tacho$((tacho+4))_rpm)" +done diff --git a/meta-facebook/meta-wedge/recipes-wedge/fan-ctrl/fan-ctrl/init_pwm.sh b/meta-facebook/meta-wedge/recipes-wedge/fan-ctrl/fan-ctrl/init_pwm.sh new file mode 100755 index 0000000..cf545be --- /dev/null +++ b/meta-facebook/meta-wedge/recipes-wedge/fan-ctrl/fan-ctrl/init_pwm.sh @@ -0,0 +1,71 @@ +#!/bin/sh +# +# Copyright 2004-present Facebook. All Rights Reserved. +# +# This program file is free software; you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the +# Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License +# for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program in a file named COPYING; if not, write to the +# Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, +# Boston, MA 02110-1301 USA +# +PWM_DIR=/sys/devices/platform/ast_pwm_tacho.0 + +set -e + +# The PWM frequency is + +# clk_source / ((2 ^ division_) * (2 * division_l) * (unit + 1)) +# +# Our clk_source is 24Mhz. 4-pin fans are generally supposed to be driven with +# a 25Khz PWM control signal. Therefore we want the divisor to equal 960. +# +# We also want the unit to be as large as possible, since this controls the +# granularity with which we can modulate the PWM signal. The following +# settings allow us to set the fan from 0 to 100% in increments of 1/96th. +# +# The AST chip supports 3 different PWM clock configurations, but we only use +# type M for now. +echo 0 > $PWM_DIR/pwm_type_m_division_h +echo 5 > $PWM_DIR/pwm_type_m_division_l +echo 95 > $PWM_DIR/pwm_type_m_unit + +# On wedge, there are 4 fans connected. +# Each fan has one PWM input and 2 tacho outputs. +# Here is the mapping between the fan and PWN/Tacho, +# staring from the one from the edge +# Fan 0: PWM 7, Tacho3, Tacho7 +# Fan 1: PWM 6, Tacho2, Tacho6 +# Fan 2: PWM 0, Tacho0, Tacho4 +# Fan 3: PWM 1, Tacho1, Tacho5 + +# For each fan, setting the type, and 100% initially +for pwm in 0 1 6 7; do + echo 0 > $PWM_DIR/pwm${pwm}_type + echo 0 > $PWM_DIR/pwm${pwm}_rising + echo 0 > $PWM_DIR/pwm${pwm}_falling + echo 1 > $PWM_DIR/pwm${pwm}_en +done + +# Enable Tach 0..7 +echo 0 > $PWM_DIR/tacho0_source +echo 0 > $PWM_DIR/tacho4_source +echo 1 > $PWM_DIR/tacho1_source +echo 1 > $PWM_DIR/tacho5_source +echo 6 > $PWM_DIR/tacho2_source +echo 6 > $PWM_DIR/tacho6_source +echo 7 > $PWM_DIR/tacho3_source +echo 7 > $PWM_DIR/tacho7_source +t=0 +while [ $t -le 7 ]; do + echo 1 > $PWM_DIR/tacho${t}_en + t=$((t+1)) +done diff --git a/meta-facebook/meta-wedge/recipes-wedge/fan-ctrl/fan-ctrl/set_fan_speed.sh b/meta-facebook/meta-wedge/recipes-wedge/fan-ctrl/fan-ctrl/set_fan_speed.sh new file mode 100755 index 0000000..a71850f --- /dev/null +++ b/meta-facebook/meta-wedge/recipes-wedge/fan-ctrl/fan-ctrl/set_fan_speed.sh @@ -0,0 +1,85 @@ +#!/bin/sh +# +# Copyright 2004-present Facebook. All Rights Reserved. +# +# This program file is free software; you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the +# Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License +# for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program in a file named COPYING; if not, write to the +# Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, +# Boston, MA 02110-1301 USA +# +usage() { + echo "Usage: $0 <PERCENT (0..100)> <Fan Unit (0..3)> " >&2 +} + +PWM_DIR=/sys/devices/platform/ast_pwm_tacho.0 + +# The maximum unit setting. +# This should be the value in pwm_type_m_unit plus 1 +PWM_UNIT_MAX=96 + +set -e + +if [ "$#" -ne 2 ] && [ "$#" -ne 1 ]; then + usage + exit 1 +fi + +# refer to the comments in init_pwn.sh regarding +# the fan unit and PWM mapping +if [ "$#" -eq 1 ]; then + PWMS="0:7 1:6 2:0 3:1" +else + case "$2" in + "0") + PWMS="0:7" + ;; + "1") + PWMS="1:6" + ;; + "2") + PWMS="2:0" + ;; + "3") + PWMS="3:1" + ;; + *) + usage + exit 1 + ;; + esac +fi + +# Convert the percentage to our 1/96th unit. +unit=$(( ( $1 * $PWM_UNIT_MAX ) / 100 )) + +for FAN_PWM in $PWMS; do + FAN_N=${FAN_PWM%%:*} + PWM_N=${FAN_PWM##*:} + if [ "$unit" -eq 0 ]; then + # For 0%, turn off the PWM entirely + echo 0 > $PWM_DIR/pwm${PWM_N}_en + else + if [ "$unit" -eq $PWM_UNIT_MAX ]; then + # For 100%, set falling and rising to the same value + unit=0 + fi + + # always use type M. refer to the comments in init_pwm.sh + echo 0 > $PWM_DIR/pwm${PWM_N}_type + echo 0 > $PWM_DIR/pwm${PWM_N}_rising + echo "$unit" > $PWM_DIR/pwm${PWM_N}_falling + echo 1 > $PWM_DIR/pwm${PWM_N}_en + fi + + echo "Successfully set fan ${FAN_N} (PWM: $PWM_N) speed to $1%" +done diff --git a/meta-facebook/meta-wedge/recipes-wedge/fan-ctrl/fan-ctrl/setup-fan.sh b/meta-facebook/meta-wedge/recipes-wedge/fan-ctrl/fan-ctrl/setup-fan.sh new file mode 100644 index 0000000..0cbdb24 --- /dev/null +++ b/meta-facebook/meta-wedge/recipes-wedge/fan-ctrl/fan-ctrl/setup-fan.sh @@ -0,0 +1,23 @@ +#!/bin/sh +# +# Copyright 2014-present Facebook. All Rights Reserved. +# +### BEGIN INIT INFO +# Provides: setup-fan +# Required-Start: board-id +# Required-Stop: +# Default-Start: S +# Default-Stop: +# Short-Description: Set fan speed +### END INIT INFO + +. /usr/local/fbpackages/utils/ast-functions + +# Enable the isolation buffer +wedge_iso_buf_enable + +echo -n "Setup fan speed... " +/usr/local/bin/init_pwm.sh +/usr/local/bin/set_fan_speed.sh 50 +/usr/local/bin/fand +echo "done." diff --git a/meta-facebook/meta-wedge/recipes-wedge/fan-ctrl/fan-ctrl/watchdog.cpp b/meta-facebook/meta-wedge/recipes-wedge/fan-ctrl/fan-ctrl/watchdog.cpp new file mode 100644 index 0000000..ebb390a --- /dev/null +++ b/meta-facebook/meta-wedge/recipes-wedge/fan-ctrl/fan-ctrl/watchdog.cpp @@ -0,0 +1,201 @@ +/* + * watchdog + * + * Copyright 2014-present Facebook. All Rights Reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. + */ + +#include "watchdog.h" + +#include <errno.h> +#include <fcntl.h> +#include <pthread.h> +#include <syslog.h> +#include <unistd.h> + +#define WATCHDOG_START_KEY "x" +#define WATCHDOG_STOP_KEY "X" +/* The "magic close" character (as defined in Linux watchdog specs). */ +#define WATCHDOG_PERSISTENT_KEY "V" +#define WATCHDOG_NON_PERSISTENT_KEY "a" + +static int watchdog_dev = -1; + +/* This is needed to prevent rapid consecutive stop/start watchdog calls from + * generating multiple threads. */ +static int watchdog_started = 0; + +static const char* watchdog_kick_key = WATCHDOG_PERSISTENT_KEY; +static pthread_t watchdog_tid; +static pthread_mutex_t watchdog_lock = PTHREAD_MUTEX_INITIALIZER; + +/* Forward declarations. */ +static void* watchdog_thread(void* args); +static int kick_watchdog_unsafe(); + +/* + * When started, this background thread will constantly reset the watchdog + * at every 5 second interval. + */ + +static void* watchdog_thread(void* args) { + + pthread_detach(pthread_self()); + + /* Make sure another instance of the thread hasn't already been started. */ + pthread_mutex_lock(&watchdog_lock); + if (watchdog_started) { + goto done; + } else { + watchdog_started = 1; + } + pthread_mutex_unlock(&watchdog_lock); + + /* Actual loop for refreshing the watchdog timer. */ + while (1) { + pthread_mutex_lock(&watchdog_lock); + if (watchdog_dev != -1) { + kick_watchdog_unsafe(); + } else { + break; + } + pthread_mutex_unlock(&watchdog_lock); + sleep(5); + } + + /* Broke out of loop because watchdog was stopped. */ + watchdog_started = 0; +done: + pthread_mutex_unlock(&watchdog_lock); + return NULL; +} + +/* + * Starts the watchdog timer. timer counts down and restarts the ARM chip + * upon timeout. use kick_watchdog() to restart the timer. + * + * Returns: 1 on success; 0 otherwise. + */ + +int start_watchdog(const int auto_mode) { + int status; + + pthread_mutex_lock(&watchdog_lock); + + /* Don't start the watchdog again if it has already been started. */ + if (watchdog_dev != -1) { + while ((status = write(watchdog_dev, WATCHDOG_START_KEY, 1)) == 0 + && errno == EINTR); + pthread_mutex_unlock(&watchdog_lock); + syslog(LOG_ALERT, "system watchdog already started.\n"); + return 0; + } + + while (((watchdog_dev = open("/dev/watchdog", O_WRONLY)) == -1) && + errno == EINTR); + + /* Fail if watchdog device is invalid or if the user asked for auto + * mode and the thread failed to spawn. */ + if ((watchdog_dev == -1) || + (auto_mode == 1 && watchdog_started == 0 && + pthread_create(&watchdog_tid, NULL, watchdog_thread, NULL) != 0)) { + goto fail; + } + + while ((status = write(watchdog_dev, WATCHDOG_START_KEY, 1)) == 0 + && errno == EINTR); + pthread_mutex_unlock(&watchdog_lock); + syslog(LOG_INFO, "system watchdog started.\n"); + return 1; + +fail: + if (watchdog_dev != -1) { + close(watchdog_dev); + watchdog_dev = -1; + } + + pthread_mutex_unlock(&watchdog_lock); + syslog(LOG_ALERT, "system watchdog failed to start!\n"); + return 0; +} + +/* + * Toggles between watchdog persistent modes. In persistent mode, the watchdog + * timer will continue to tick even after process shutdown. Under non- + * persistent mode, the watchdog timer will automatically be disabled when the + * process shuts down. + */ +void set_persistent_watchdog(enum watchdog_persistent_en persistent) { + switch (persistent) { + case WATCHDOG_SET_PERSISTENT: + watchdog_kick_key = WATCHDOG_PERSISTENT_KEY; + break; + default: + watchdog_kick_key = WATCHDOG_NON_PERSISTENT_KEY; + break; + } + kick_watchdog(); +} + +/* + * Restarts the countdown timer on the watchdog, delaying restart by another + * timeout period (default: 11 seconds as configured in the device driver). + * + * This function assumes the watchdog lock has already been acquired and is + * only used internally within the watchdog code. + * + * Returns 1 on success; 0 or -1 indicates failure (check errno). + */ + +static int kick_watchdog_unsafe() { + int status = 0; + if (watchdog_dev != -1) { + while ((status = write(watchdog_dev, watchdog_kick_key, 1)) == 0 + && errno == EINTR); + } + return status; +} + +/* + * Acquires the watchdog lock and resets the watchdog atomically. For use by + * library users. + */ + +int kick_watchdog() { + int result; + pthread_mutex_lock(&watchdog_lock); + result = kick_watchdog_unsafe(); + pthread_mutex_unlock(&watchdog_lock); + + return result; +} + +/* Shuts down the watchdog gracefully and disables the watchdog timer so that + * restarts no longer happen. + */ + +void stop_watchdog() { + int status; + pthread_mutex_lock(&watchdog_lock); + if (watchdog_dev != -1) { + while ((status = write(watchdog_dev, WATCHDOG_STOP_KEY, 1)) == 0 + && errno == EINTR); + close(watchdog_dev); + watchdog_dev = -1; + syslog(LOG_INFO, "system watchdog stopped.\n"); + } + pthread_mutex_unlock(&watchdog_lock); +} diff --git a/meta-facebook/meta-wedge/recipes-wedge/fan-ctrl/fan-ctrl/watchdog.h b/meta-facebook/meta-wedge/recipes-wedge/fan-ctrl/fan-ctrl/watchdog.h new file mode 100644 index 0000000..19b9944 --- /dev/null +++ b/meta-facebook/meta-wedge/recipes-wedge/fan-ctrl/fan-ctrl/watchdog.h @@ -0,0 +1,60 @@ +/* + * watchdog + * + * Copyright 2014-present Facebook. All Rights Reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. + * + * Utility library to handle the aspeed watchdog. Only one watchdog + * should be in use at any time throughout the entire system (multiple users + * will not cause adverse effects but the behavior of the watchdog becomes + * undefined). + * + * The watchdog can be started in either manual or automatic mode. In manual + * mode, the watchdog has to be constantly reset by the user via the + * kick_watchdog() function. Under automatic mode, the watchdog will + * run in a separate thread and reset the timer on its own so no intervention + * is required from the user. + * + * In both modes, the watchdog timer will not stop when the process is + * terminated, unless a call to stop_watchdog() has been made beforehand, or + * if the user runs in manual mode and uses a non persistent watchdog kick. + * + * The default timeout for the watchdog is 11 seconds. When this time period + * elapses, the ARM chip is restarted and the kernel is rebooted. Other + * hardware state is not reset, so this may introduce strange behavior on + * reboot (example: an I2C bus may be left in the open state, triggering + * constant interrupts). In rare cases, this could result in the kernel + * failing to fully restart itself and thus preclude the possibility of + * reinitializing the watchdog timer. Someone will then have to go over and + * physically restart the machine. + * + * The alternative to the soft reset is to request the watchdog device driver + * for a hard reset on timeout. However this will stop the fans. If the + * kernel fails to fully boot and restart the fan daemon, the system could + * overheat. For this reason, we've chosen to take the risk of a stuck soft + * reset instead. + * + */ + +/* Forward declarations. */ +int start_watchdog(const int auto_mode); +enum watchdog_persistent_en { + WATCHDOG_SET_PERSISTENT, + WATCHDOG_SET_NONPERSISTENT, +}; +void set_persistent_watchdog(enum watchdog_persistent_en persistent); +int kick_watchdog(); +void stop_watchdog(); |