diff options
612 files changed, 20250 insertions, 11239 deletions
diff --git a/Documentation/devicetree/bindings/arm/cpu-capacity.txt b/Documentation/devicetree/bindings/arm/cpu-capacity.txt new file mode 100644 index 0000000..7809fbe --- /dev/null +++ b/Documentation/devicetree/bindings/arm/cpu-capacity.txt @@ -0,0 +1,236 @@ +========================================== +ARM CPUs capacity bindings +========================================== + +========================================== +1 - Introduction +========================================== + +ARM systems may be configured to have cpus with different power/performance +characteristics within the same chip. In this case, additional information has +to be made available to the kernel for it to be aware of such differences and +take decisions accordingly. + +========================================== +2 - CPU capacity definition +========================================== + +CPU capacity is a number that provides the scheduler information about CPUs +heterogeneity. Such heterogeneity can come from micro-architectural differences +(e.g., ARM big.LITTLE systems) or maximum frequency at which CPUs can run +(e.g., SMP systems with multiple frequency domains). Heterogeneity in this +context is about differing performance characteristics; this binding tries to +capture a first-order approximation of the relative performance of CPUs. + +CPU capacities are obtained by running a suitable benchmark. This binding makes +no guarantees on the validity or suitability of any particular benchmark, the +final capacity should, however, be: + +* A "single-threaded" or CPU affine benchmark +* Divided by the running frequency of the CPU executing the benchmark +* Not subject to dynamic frequency scaling of the CPU + +For the time being we however advise usage of the Dhrystone benchmark. What +above thus becomes: + +CPU capacities are obtained by running the Dhrystone benchmark on each CPU at +max frequency (with caches enabled). The obtained DMIPS score is then divided +by the frequency (in MHz) at which the benchmark has been run, so that +DMIPS/MHz are obtained. Such values are then normalized w.r.t. the highest +score obtained in the system. + +========================================== +3 - capacity-dmips-mhz +========================================== + +capacity-dmips-mhz is an optional cpu node [1] property: u32 value +representing CPU capacity expressed in normalized DMIPS/MHz. At boot time, the +maximum frequency available to the cpu is then used to calculate the capacity +value internally used by the kernel. + +capacity-dmips-mhz property is all-or-nothing: if it is specified for a cpu +node, it has to be specified for every other cpu nodes, or the system will +fall back to the default capacity value for every CPU. If cpufreq is not +available, final capacities are calculated by directly using capacity-dmips- +mhz values (normalized w.r.t. the highest value found while parsing the DT). + +=========================================== +4 - Examples +=========================================== + +Example 1 (ARM 64-bit, 6-cpu system, two clusters): +capacities-dmips-mhz are scaled w.r.t. 1024 (cpu@0 and cpu@1) +supposing cluster0@max-freq=1100 and custer1@max-freq=850, +final capacities are 1024 for cluster0 and 446 for cluster1 + +cpus { + #address-cells = <2>; + #size-cells = <0>; + + cpu-map { + cluster0 { + core0 { + cpu = <&A57_0>; + }; + core1 { + cpu = <&A57_1>; + }; + }; + + cluster1 { + core0 { + cpu = <&A53_0>; + }; + core1 { + cpu = <&A53_1>; + }; + core2 { + cpu = <&A53_2>; + }; + core3 { + cpu = <&A53_3>; + }; + }; + }; + + idle-states { + entry-method = "arm,psci"; + + CPU_SLEEP_0: cpu-sleep-0 { + compatible = "arm,idle-state"; + arm,psci-suspend-param = <0x0010000>; + local-timer-stop; + entry-latency-us = <100>; + exit-latency-us = <250>; + min-residency-us = <150>; + }; + + CLUSTER_SLEEP_0: cluster-sleep-0 { + compatible = "arm,idle-state"; + arm,psci-suspend-param = <0x1010000>; + local-timer-stop; + entry-latency-us = <800>; + exit-latency-us = <700>; + min-residency-us = <2500>; + }; + }; + + A57_0: cpu@0 { + compatible = "arm,cortex-a57","arm,armv8"; + reg = <0x0 0x0>; + device_type = "cpu"; + enable-method = "psci"; + next-level-cache = <&A57_L2>; + clocks = <&scpi_dvfs 0>; + cpu-idle-states = <&CPU_SLEEP_0 &CLUSTER_SLEEP_0>; + capacity-dmips-mhz = <1024>; + }; + + A57_1: cpu@1 { + compatible = "arm,cortex-a57","arm,armv8"; + reg = <0x0 0x1>; + device_type = "cpu"; + enable-method = "psci"; + next-level-cache = <&A57_L2>; + clocks = <&scpi_dvfs 0>; + cpu-idle-states = <&CPU_SLEEP_0 &CLUSTER_SLEEP_0>; + capacity-dmips-mhz = <1024>; + }; + + A53_0: cpu@100 { + compatible = "arm,cortex-a53","arm,armv8"; + reg = <0x0 0x100>; + device_type = "cpu"; + enable-method = "psci"; + next-level-cache = <&A53_L2>; + clocks = <&scpi_dvfs 1>; + cpu-idle-states = <&CPU_SLEEP_0 &CLUSTER_SLEEP_0>; + capacity-dmips-mhz = <578>; + }; + + A53_1: cpu@101 { + compatible = "arm,cortex-a53","arm,armv8"; + reg = <0x0 0x101>; + device_type = "cpu"; + enable-method = "psci"; + next-level-cache = <&A53_L2>; + clocks = <&scpi_dvfs 1>; + cpu-idle-states = <&CPU_SLEEP_0 &CLUSTER_SLEEP_0>; + capacity-dmips-mhz = <578>; + }; + + A53_2: cpu@102 { + compatible = "arm,cortex-a53","arm,armv8"; + reg = <0x0 0x102>; + device_type = "cpu"; + enable-method = "psci"; + next-level-cache = <&A53_L2>; + clocks = <&scpi_dvfs 1>; + cpu-idle-states = <&CPU_SLEEP_0 &CLUSTER_SLEEP_0>; + capacity-dmips-mhz = <578>; + }; + + A53_3: cpu@103 { + compatible = "arm,cortex-a53","arm,armv8"; + reg = <0x0 0x103>; + device_type = "cpu"; + enable-method = "psci"; + next-level-cache = <&A53_L2>; + clocks = <&scpi_dvfs 1>; + cpu-idle-states = <&CPU_SLEEP_0 &CLUSTER_SLEEP_0>; + capacity-dmips-mhz = <578>; + }; + + A57_L2: l2-cache0 { + compatible = "cache"; + }; + + A53_L2: l2-cache1 { + compatible = "cache"; + }; +}; + +Example 2 (ARM 32-bit, 4-cpu system, two clusters, + cpus 0,1@1GHz, cpus 2,3@500MHz): +capacities-dmips-mhz are scaled w.r.t. 2 (cpu@0 and cpu@1), this means that first +cpu@0 and cpu@1 are twice fast than cpu@2 and cpu@3 (at the same frequency) + +cpus { + #address-cells = <1>; + #size-cells = <0>; + + cpu0: cpu@0 { + device_type = "cpu"; + compatible = "arm,cortex-a15"; + reg = <0>; + capacity-dmips-mhz = <2>; + }; + + cpu1: cpu@1 { + device_type = "cpu"; + compatible = "arm,cortex-a15"; + reg = <1>; + capacity-dmips-mhz = <2>; + }; + + cpu2: cpu@2 { + device_type = "cpu"; + compatible = "arm,cortex-a15"; + reg = <0x100>; + capacity-dmips-mhz = <1>; + }; + + cpu3: cpu@3 { + device_type = "cpu"; + compatible = "arm,cortex-a15"; + reg = <0x101>; + capacity-dmips-mhz = <1>; + }; +}; + +=========================================== +5 - References +=========================================== + +[1] ARM Linux Kernel documentation - CPUs bindings + Documentation/devicetree/bindings/arm/cpus.txt diff --git a/Documentation/devicetree/bindings/arm/cpus.txt b/Documentation/devicetree/bindings/arm/cpus.txt index e6782d5..c1dcf4c 100644 --- a/Documentation/devicetree/bindings/arm/cpus.txt +++ b/Documentation/devicetree/bindings/arm/cpus.txt @@ -241,6 +241,14 @@ nodes to be present and contain the properties described below. # List of phandles to idle state nodes supported by this cpu [3]. + - capacity-dmips-mhz + Usage: Optional + Value type: <u32> + Definition: + # u32 value representing CPU capacity [3] in + DMIPS/MHz, relative to highest capacity-dmips-mhz + in the system. + - rockchip,pmu Usage: optional for systems that have an "enable-method" property value of "rockchip,rk3066-smp" @@ -464,3 +472,5 @@ cpus { [2] arm/msm/qcom,kpss-acc.txt [3] ARM Linux kernel documentation - idle states bindings Documentation/devicetree/bindings/arm/idle-states.txt +[3] ARM Linux kernel documentation - cpu capacity bindings + Documentation/devicetree/bindings/arm/cpu-capacity.txt diff --git a/Documentation/devicetree/bindings/hwmon/mcp3021.txt b/Documentation/devicetree/bindings/hwmon/mcp3021.txt new file mode 100644 index 0000000..294318b --- /dev/null +++ b/Documentation/devicetree/bindings/hwmon/mcp3021.txt @@ -0,0 +1,21 @@ +mcp3021 properties + +Required properties: +- compatible: Must be one of the following: + - "microchip,mcp3021" for mcp3021 + - "microchip,mcp3221" for mcp3221 +- reg: I2C address + +Optional properties: + +- reference-voltage-microvolt + Reference voltage in microvolt (uV) + +Example: + +mcp3021@4d { + compatible = "microchip,mcp3021"; + reg = <0x4d>; + + reference-voltage-microvolt = <4500000>; /* 4.5 V */ +}; diff --git a/Documentation/devicetree/bindings/hwmon/tmp108.txt b/Documentation/devicetree/bindings/hwmon/tmp108.txt new file mode 100644 index 0000000..8c4b10d --- /dev/null +++ b/Documentation/devicetree/bindings/hwmon/tmp108.txt @@ -0,0 +1,14 @@ +TMP108 temperature sensor +------------------------- + +This device supports I2C only. + +Requires node properties: +- compatible : "ti,tmp108" +- reg : the I2C address of the device. This is 0x48, 0x49, 0x4a, or 0x4b. + +Example: + tmp108@48 { + compatible = "ti,tmp108"; + reg = <0x48>; + }; diff --git a/Documentation/devicetree/bindings/i2c/trivial-devices.txt b/Documentation/devicetree/bindings/i2c/trivial-devices.txt index 03349ad..df720ca 100644 --- a/Documentation/devicetree/bindings/i2c/trivial-devices.txt +++ b/Documentation/devicetree/bindings/i2c/trivial-devices.txt @@ -124,6 +124,8 @@ microchip,mcp4662-502 Microchip 8-bit Dual I2C Digital Potentiometer with NV Mem microchip,mcp4662-103 Microchip 8-bit Dual I2C Digital Potentiometer with NV Memory (10k) microchip,mcp4662-503 Microchip 8-bit Dual I2C Digital Potentiometer with NV Memory (50k) microchip,mcp4662-104 Microchip 8-bit Dual I2C Digital Potentiometer with NV Memory (100k) +microchip,tc654 PWM Fan Speed Controller With Fan Fault Detection +microchip,tc655 PWM Fan Speed Controller With Fan Fault Detection miramems,da226 MiraMEMS DA226 2-axis 14-bit digital accelerometer miramems,da280 MiraMEMS DA280 3-axis 14-bit digital accelerometer miramems,da311 MiraMEMS DA311 3-axis 12-bit digital accelerometer diff --git a/Documentation/devicetree/bindings/scsi/hisilicon-sas.txt b/Documentation/devicetree/bindings/scsi/hisilicon-sas.txt index bf2411f..2a42a32 100644 --- a/Documentation/devicetree/bindings/scsi/hisilicon-sas.txt +++ b/Documentation/devicetree/bindings/scsi/hisilicon-sas.txt @@ -6,6 +6,7 @@ Main node required properties: - compatible : value should be as follows: (a) "hisilicon,hip05-sas-v1" for v1 hw in hip05 chipset (b) "hisilicon,hip06-sas-v2" for v2 hw in hip06 chipset + (c) "hisilicon,hip07-sas-v2" for v2 hw in hip07 chipset - sas-addr : array of 8 bytes for host SAS address - reg : Address and length of the SAS register - hisilicon,sas-syscon: phandle of syscon used for sas control diff --git a/Documentation/devicetree/bindings/spi/sh-msiof.txt b/Documentation/devicetree/bindings/spi/sh-msiof.txt index aa005c1..da6614c 100644 --- a/Documentation/devicetree/bindings/spi/sh-msiof.txt +++ b/Documentation/devicetree/bindings/spi/sh-msiof.txt @@ -10,6 +10,7 @@ Required properties: "renesas,msiof-r8a7792" (R-Car V2H) "renesas,msiof-r8a7793" (R-Car M2-N) "renesas,msiof-r8a7794" (R-Car E2) + "renesas,msiof-r8a7796" (R-Car M3-W) "renesas,msiof-sh73a0" (SH-Mobile AG5) - reg : A list of offsets and lengths of the register sets for the device. diff --git a/Documentation/devicetree/bindings/spi/spi-armada-3700.txt b/Documentation/devicetree/bindings/spi/spi-armada-3700.txt new file mode 100644 index 0000000..1564aa8 --- /dev/null +++ b/Documentation/devicetree/bindings/spi/spi-armada-3700.txt @@ -0,0 +1,25 @@ +* Marvell Armada 3700 SPI Controller + +Required Properties: + +- compatible: should be "marvell,armada-3700-spi" +- reg: physical base address of the controller and length of memory mapped + region. +- interrupts: The interrupt number. The interrupt specifier format depends on + the interrupt controller and of its driver. +- clocks: Must contain the clock source, usually from the North Bridge clocks. +- num-cs: The number of chip selects that is supported by this SPI Controller +- #address-cells: should be 1. +- #size-cells: should be 0. + +Example: + + spi0: spi@10600 { + compatible = "marvell,armada-3700-spi"; + #address-cells = <1>; + #size-cells = <0>; + reg = <0x10600 0x5d>; + clocks = <&nb_perih_clk 7>; + interrupts = <GIC_SPI 0 IRQ_TYPE_LEVEL_HIGH>; + num-cs = <4>; + }; diff --git a/Documentation/devicetree/bindings/spi/spi-fsl-lpspi.txt b/Documentation/devicetree/bindings/spi/spi-fsl-lpspi.txt new file mode 100644 index 0000000..225ace1 --- /dev/null +++ b/Documentation/devicetree/bindings/spi/spi-fsl-lpspi.txt @@ -0,0 +1,19 @@ +* Freescale Low Power SPI (LPSPI) for i.MX + +Required properties: +- compatible : + - "fsl,imx7ulp-spi" for LPSPI compatible with the one integrated on i.MX7ULP soc +- reg : address and length of the lpspi master registers +- interrupt-parent : core interrupt controller +- interrupts : lpspi interrupt +- clocks : lpspi clock specifier + +Examples: + +lpspi2: lpspi@40290000 { + compatible = "fsl,imx7ulp-spi"; + reg = <0x40290000 0x10000>; + interrupt-parent = <&intc>; + interrupts = <GIC_SPI 28 IRQ_TYPE_LEVEL_HIGH>; + clocks = <&clks IMX7ULP_CLK_LPSPI2>; +}; diff --git a/Documentation/devicetree/bindings/spi/spi-sun6i.txt b/Documentation/devicetree/bindings/spi/spi-sun6i.txt index 21de73d..2ec99b8 100644 --- a/Documentation/devicetree/bindings/spi/spi-sun6i.txt +++ b/Documentation/devicetree/bindings/spi/spi-sun6i.txt @@ -1,7 +1,7 @@ -Allwinner A31 SPI controller +Allwinner A31/H3 SPI controller Required properties: -- compatible: Should be "allwinner,sun6i-a31-spi". +- compatible: Should be "allwinner,sun6i-a31-spi" or "allwinner,sun8i-h3-spi". - reg: Should contain register location and length. - interrupts: Should contain interrupt. - clocks: phandle to the clocks feeding the SPI controller. Two are @@ -12,6 +12,11 @@ Required properties: - resets: phandle to the reset controller asserting this device in reset +Optional properties: +- dmas: DMA specifiers for rx and tx dma. See the DMA client binding, + Documentation/devicetree/bindings/dma/dma.txt +- dma-names: DMA request names should include "rx" and "tx" if present. + Example: spi1: spi@01c69000 { @@ -22,3 +27,19 @@ spi1: spi@01c69000 { clock-names = "ahb", "mod"; resets = <&ahb1_rst 21>; }; + +spi0: spi@01c68000 { + compatible = "allwinner,sun8i-h3-spi"; + reg = <0x01c68000 0x1000>; + interrupts = <GIC_SPI 65 IRQ_TYPE_LEVEL_HIGH>; + clocks = <&ccu CLK_BUS_SPI0>, <&ccu CLK_SPI0>; + clock-names = "ahb", "mod"; + dmas = <&dma 23>, <&dma 23>; + dma-names = "rx", "tx"; + pinctrl-names = "default"; + pinctrl-0 = <&spi0_pins>; + resets = <&ccu RST_BUS_SPI0>; + status = "disabled"; + #address-cells = <1>; + #size-cells = <0>; +}; diff --git a/Documentation/devicetree/bindings/ufs/ufs-qcom.txt b/Documentation/devicetree/bindings/ufs/ufs-qcom.txt index 070baf4..b6b5130 100644 --- a/Documentation/devicetree/bindings/ufs/ufs-qcom.txt +++ b/Documentation/devicetree/bindings/ufs/ufs-qcom.txt @@ -7,8 +7,11 @@ To bind UFS PHY with UFS host controller, the controller node should contain a phandle reference to UFS PHY node. Required properties: -- compatible : compatible list, contains "qcom,ufs-phy-qmp-20nm" - or "qcom,ufs-phy-qmp-14nm" according to the relevant phy in use. +- compatible : compatible list, contains one of the following - + "qcom,ufs-phy-qmp-20nm" for 20nm ufs phy, + "qcom,ufs-phy-qmp-14nm" for legacy 14nm ufs phy, + "qcom,msm8996-ufs-phy-qmp-14nm" for 14nm ufs phy + present on MSM8996 chipset. - reg : should contain PHY register address space (mandatory), - reg-names : indicates various resources passed to driver (via reg proptery) by name. Required "reg-names" is "phy_mem". diff --git a/Documentation/filesystems/configfs/configfs.txt b/Documentation/filesystems/configfs/configfs.txt index 8ec9136..3828e85 100644 --- a/Documentation/filesystems/configfs/configfs.txt +++ b/Documentation/filesystems/configfs/configfs.txt @@ -174,7 +174,7 @@ among other things. For that, it needs a type. void (*release)(struct config_item *); int (*allow_link)(struct config_item *src, struct config_item *target); - int (*drop_link)(struct config_item *src, + void (*drop_link)(struct config_item *src, struct config_item *target); }; diff --git a/Documentation/filesystems/dax.txt b/Documentation/filesystems/dax.txt index 23d18b8..a7e6e14 100644 --- a/Documentation/filesystems/dax.txt +++ b/Documentation/filesystems/dax.txt @@ -58,22 +58,22 @@ Implementation Tips for Filesystem Writers Filesystem support consists of - adding support to mark inodes as being DAX by setting the S_DAX flag in i_flags -- implementing the direct_IO address space operation, and calling - dax_do_io() instead of blockdev_direct_IO() if S_DAX is set +- implementing ->read_iter and ->write_iter operations which use dax_iomap_rw() + when inode has S_DAX flag set - implementing an mmap file operation for DAX files which sets the VM_MIXEDMAP and VM_HUGEPAGE flags on the VMA, and setting the vm_ops to - include handlers for fault, pmd_fault and page_mkwrite (which should - probably call dax_fault(), dax_pmd_fault() and dax_mkwrite(), passing the - appropriate get_block() callback) -- calling dax_truncate_page() instead of block_truncate_page() for DAX files -- calling dax_zero_page_range() instead of zero_user() for DAX files + include handlers for fault, pmd_fault, page_mkwrite, pfn_mkwrite. These + handlers should probably call dax_iomap_fault() (for fault and page_mkwrite + handlers), dax_iomap_pmd_fault(), dax_pfn_mkwrite() passing the appropriate + iomap operations. +- calling iomap_zero_range() passing appropriate iomap operations instead of + block_truncate_page() for DAX files - ensuring that there is sufficient locking between reads, writes, truncates and page faults -The get_block() callback passed to the DAX functions may return -uninitialised extents. If it does, it must ensure that simultaneous -calls to get_block() (for example by a page-fault racing with a read() -or a write()) work correctly. +The iomap handlers for allocating blocks must make sure that allocated blocks +are zeroed out and converted to written extents before being returned to avoid +exposure of uninitialized data through mmap. These filesystems may be used for inspiration: - ext2: see Documentation/filesystems/ext2.txt diff --git a/Documentation/filesystems/ext4.txt b/Documentation/filesystems/ext4.txt index 6c0108e..3698ed3 100644 --- a/Documentation/filesystems/ext4.txt +++ b/Documentation/filesystems/ext4.txt @@ -351,14 +351,13 @@ nouid32 Disables 32-bit UIDs and GIDs. This is for interoperability with older kernels which only store and expect 16-bit values. -block_validity This options allows to enables/disables the in-kernel +block_validity(*) These options enable or disable the in-kernel noblock_validity facility for tracking filesystem metadata blocks - within internal data structures. This allows multi- - block allocator and other routines to quickly locate - extents which might overlap with filesystem metadata - blocks. This option is intended for debugging - purposes and since it negatively affects the - performance, it is off by default. + within internal data structures. This allows multi- + block allocator and other routines to notice + bugs or corrupted allocation bitmaps which cause + blocks to be allocated which overlap with + filesystem metadata blocks. dioread_lock Controls whether or not ext4 should use the DIO read dioread_nolock locking. If the dioread_nolock option is specified diff --git a/Documentation/hwmon/hwmon-kernel-api.txt b/Documentation/hwmon/hwmon-kernel-api.txt index ef9d749..2505ae6 100644 --- a/Documentation/hwmon/hwmon-kernel-api.txt +++ b/Documentation/hwmon/hwmon-kernel-api.txt @@ -23,7 +23,6 @@ Each hardware monitoring driver must #include <linux/hwmon.h> and, in most cases, <linux/hwmon-sysfs.h>. linux/hwmon.h declares the following register/unregister functions: -struct device *hwmon_device_register(struct device *dev); struct device * hwmon_device_register_with_groups(struct device *dev, const char *name, void *drvdata, @@ -38,36 +37,31 @@ struct device * hwmon_device_register_with_info(struct device *dev, const char *name, void *drvdata, const struct hwmon_chip_info *info, - const struct attribute_group **groups); + const struct attribute_group **extra_groups); struct device * devm_hwmon_device_register_with_info(struct device *dev, - const char *name, - void *drvdata, - const struct hwmon_chip_info *info, - const struct attribute_group **groups); + const char *name, + void *drvdata, + const struct hwmon_chip_info *info, + const struct attribute_group **extra_groups); void hwmon_device_unregister(struct device *dev); void devm_hwmon_device_unregister(struct device *dev); -hwmon_device_register registers a hardware monitoring device. The parameter -of this function is a pointer to the parent device. -This function returns a pointer to the newly created hardware monitoring device -or PTR_ERR for failure. If this registration function is used, hardware -monitoring sysfs attributes are expected to have been created and attached to -the parent device prior to calling hwmon_device_register. A name attribute must -have been created by the caller. - -hwmon_device_register_with_groups is similar to hwmon_device_register. However, -it has additional parameters. The name parameter is a pointer to the hwmon -device name. The registration function wil create a name sysfs attribute -pointing to this name. The drvdata parameter is the pointer to the local -driver data. hwmon_device_register_with_groups will attach this pointer -to the newly allocated hwmon device. The pointer can be retrieved by the driver -using dev_get_drvdata() on the hwmon device pointer. The groups parameter is +hwmon_device_register_with_groups registers a hardware monitoring device. +The first parameter of this function is a pointer to the parent device. +The name parameter is a pointer to the hwmon device name. The registration +function wil create a name sysfs attribute pointing to this name. +The drvdata parameter is the pointer to the local driver data. +hwmon_device_register_with_groups will attach this pointer to the newly +allocated hwmon device. The pointer can be retrieved by the driver using +dev_get_drvdata() on the hwmon device pointer. The groups parameter is a pointer to a list of sysfs attribute groups. The list must be NULL terminated. hwmon_device_register_with_groups creates the hwmon device with name attribute as well as all sysfs attributes attached to the hwmon device. +This function returns a pointer to the newly created hardware monitoring device +or PTR_ERR for failure. devm_hwmon_device_register_with_groups is similar to hwmon_device_register_with_groups. However, it is device managed, meaning the @@ -87,13 +81,13 @@ hwmon_device_unregister deregisters a registered hardware monitoring device. The parameter of this function is the pointer to the registered hardware monitoring device structure. This function must be called from the driver remove function if the hardware monitoring device was registered with -hwmon_device_register, hwmon_device_register_with_groups, or -hwmon_device_register_with_info. +hwmon_device_register_with_groups or hwmon_device_register_with_info. devm_hwmon_device_unregister does not normally have to be called. It is only needed for error handling, and only needed if the driver probe fails after -the call to devm_hwmon_device_register_with_groups and if the automatic -(device managed) removal would be too late. +the call to devm_hwmon_device_register_with_groups or +hwmon_device_register_with_info and if the automatic (device managed) +removal would be too late. Using devm_hwmon_device_register_with_info() -------------------------------------------- @@ -106,9 +100,9 @@ const char *name Device name void *drvdata Driver private data const struct hwmon_chip_info *info Pointer to chip description. -const struct attribute_group **groups - Null-terminated list of additional sysfs attribute - groups. +const struct attribute_group **extra_groups + Null-terminated list of additional non-standard + sysfs attribute groups. This function returns a pointer to the created hardware monitoring device on success and a negative error code for failure. @@ -160,7 +154,7 @@ It contains following fields: * type: The hardware monitoring sensor type. Supported sensor types are * hwmon_chip A virtual sensor type, used to describe attributes - which apply to the entire chip. + * which are not bound to a specific input or output * hwmon_temp Temperature sensor * hwmon_in Voltage sensor * hwmon_curr Current sensor @@ -293,9 +287,9 @@ Driver-provided sysfs attributes If the hardware monitoring device is registered with hwmon_device_register_with_info or devm_hwmon_device_register_with_info, -it is most likely not necessary to provide sysfs attributes. Only non-standard -sysfs attributes need to be provided when one of those registration functions -is used. +it is most likely not necessary to provide sysfs attributes. Only additional +non-standard sysfs attributes need to be provided when one of those registration +functions is used. The header file linux/hwmon-sysfs.h provides a number of useful macros to declare and use hardware monitoring sysfs attributes. diff --git a/Documentation/hwmon/tc654 b/Documentation/hwmon/tc654 new file mode 100644 index 0000000..91a2843 --- /dev/null +++ b/Documentation/hwmon/tc654 @@ -0,0 +1,31 @@ +Kernel driver tc654 +=================== + +Supported chips: + * Microship TC654 and TC655 + Prefix: 'tc654' + Datasheet: http://ww1.microchip.com/downloads/en/DeviceDoc/20001734C.pdf + +Authors: + Chris Packham <chris.packham@alliedtelesis.co.nz> + Masahiko Iwamoto <iwamoto@allied-telesis.co.jp> + +Description +----------- +This driver implements support for the Microchip TC654 and TC655. + +The TC654 uses the 2-wire interface compatible with the SMBUS 2.0 +specification. The TC654 has two (2) inputs for measuring fan RPM and +one (1) PWM output which can be used for fan control. + +Configuration Notes +------------------- +Ordinarily the pwm1_mode ABI is used for controlling the pwm output +mode. However, for this chip the output is always pwm, and the +pwm1_mode determines if the pwm output is controlled via the pwm1 value +or via the Vin analog input. + + +Setting pwm1_mode to 1 will cause the pwm output to be driven based on +the pwm1 value. Setting pwm1_mode to 0 will cause the pwm output to be +driven based on the Vin input. diff --git a/Documentation/hwmon/tmp108 b/Documentation/hwmon/tmp108 new file mode 100644 index 0000000..25802df --- /dev/null +++ b/Documentation/hwmon/tmp108 @@ -0,0 +1,36 @@ +Kernel driver tmp108 +==================== + +Supported chips: + * Texas Instruments TMP108 + Prefix: 'tmp108' + Addresses scanned: none + Datasheet: http://www.ti.com/product/tmp108 + +Author: + John Muir <john@jmuir.com> + +Description +----------- + +The Texas Instruments TMP108 implements one temperature sensor. An alert pin +can be set when temperatures exceed minimum or maximum values plus or minus a +hysteresis value. (This driver does not support interrupts for the alert pin, +and the device runs in comparator mode.) + +The sensor is accurate to 0.75C over the range of -25 to +85 C, and to 1.0 +degree from -40 to +125 C. Resolution of the sensor is 0.0625 degree. The +operating temperature has a minimum of -55 C and a maximum of +150 C. +Hysteresis values can be set to 0, 1, 2, or 4C. + +The TMP108 has a programmable update rate that can select between 8, 4, 1, and +0.5 Hz. + +By default the TMP108 reads the temperature continuously. To conserve power, +the TMP108 has a one-shot mode where the device is normally shut-down. When a +one shot is requested the temperature is read, the result can be retrieved, +and then the device is shut down automatically. (This driver only supports +continuous mode.) + +The driver provides the common sysfs-interface for temperatures (see +Documentation/hwmon/sysfs-interface under Temperatures). diff --git a/Documentation/virtual/kvm/00-INDEX b/Documentation/virtual/kvm/00-INDEX index fee9f2b..69fe1a8 100644 --- a/Documentation/virtual/kvm/00-INDEX +++ b/Documentation/virtual/kvm/00-INDEX @@ -6,6 +6,8 @@ cpuid.txt - KVM-specific cpuid leaves (x86). devices/ - KVM_CAP_DEVICE_CTRL userspace API. +halt-polling.txt + - notes on halt-polling hypercalls.txt - KVM hypercalls. locking.txt diff --git a/Documentation/virtual/kvm/api.txt b/Documentation/virtual/kvm/api.txt index 6bbceb9..03145b7 100644 --- a/Documentation/virtual/kvm/api.txt +++ b/Documentation/virtual/kvm/api.txt @@ -2034,6 +2034,8 @@ registers, find a list below: PPC | KVM_REG_PPC_WORT | 64 PPC | KVM_REG_PPC_SPRG9 | 64 PPC | KVM_REG_PPC_DBSR | 32 + PPC | KVM_REG_PPC_TIDR | 64 + PPC | KVM_REG_PPC_PSSCR | 64 PPC | KVM_REG_PPC_TM_GPR0 | 64 ... PPC | KVM_REG_PPC_TM_GPR31 | 64 @@ -2050,6 +2052,7 @@ registers, find a list below: PPC | KVM_REG_PPC_TM_VSCR | 32 PPC | KVM_REG_PPC_TM_DSCR | 64 PPC | KVM_REG_PPC_TM_TAR | 64 + PPC | KVM_REG_PPC_TM_XER | 64 | | MIPS | KVM_REG_MIPS_R0 | 64 ... @@ -2209,7 +2212,7 @@ after pausing the vcpu, but before it is resumed. 4.71 KVM_SIGNAL_MSI Capability: KVM_CAP_SIGNAL_MSI -Architectures: x86 arm64 +Architectures: x86 arm arm64 Type: vm ioctl Parameters: struct kvm_msi (in) Returns: >0 on delivery, 0 if guest blocked the MSI, and -1 on error diff --git a/Documentation/virtual/kvm/halt-polling.txt b/Documentation/virtual/kvm/halt-polling.txt new file mode 100644 index 0000000..4a84183 --- /dev/null +++ b/Documentation/virtual/kvm/halt-polling.txt @@ -0,0 +1,127 @@ +The KVM halt polling system +=========================== + +The KVM halt polling system provides a feature within KVM whereby the latency +of a guest can, under some circumstances, be reduced by polling in the host +for some time period after the guest has elected to no longer run by cedeing. +That is, when a guest vcpu has ceded, or in the case of powerpc when all of the +vcpus of a single vcore have ceded, the host kernel polls for wakeup conditions +before giving up the cpu to the scheduler in order to let something else run. + +Polling provides a latency advantage in cases where the guest can be run again +very quickly by at least saving us a trip through the scheduler, normally on +the order of a few micro-seconds, although performance benefits are workload +dependant. In the event that no wakeup source arrives during the polling +interval or some other task on the runqueue is runnable the scheduler is +invoked. Thus halt polling is especially useful on workloads with very short +wakeup periods where the time spent halt polling is minimised and the time +savings of not invoking the scheduler are distinguishable. + +The generic halt polling code is implemented in: + + virt/kvm/kvm_main.c: kvm_vcpu_block() + +The powerpc kvm-hv specific case is implemented in: + + arch/powerpc/kvm/book3s_hv.c: kvmppc_vcore_blocked() + +Halt Polling Interval +===================== + +The maximum time for which to poll before invoking the scheduler, referred to +as the halt polling interval, is increased and decreased based on the perceived +effectiveness of the polling in an attempt to limit pointless polling. +This value is stored in either the vcpu struct: + + kvm_vcpu->halt_poll_ns + +or in the case of powerpc kvm-hv, in the vcore struct: + + kvmppc_vcore->halt_poll_ns + +Thus this is a per vcpu (or vcore) value. + +During polling if a wakeup source is received within the halt polling interval, +the interval is left unchanged. In the event that a wakeup source isn't +received during the polling interval (and thus schedule is invoked) there are +two options, either the polling interval and total block time[0] were less than +the global max polling interval (see module params below), or the total block +time was greater than the global max polling interval. + +In the event that both the polling interval and total block time were less than +the global max polling interval then the polling interval can be increased in +the hope that next time during the longer polling interval the wake up source +will be received while the host is polling and the latency benefits will be +received. The polling interval is grown in the function grow_halt_poll_ns() and +is multiplied by the module parameter halt_poll_ns_grow. + +In the event that the total block time was greater than the global max polling +interval then the host will never poll for long enough (limited by the global +max) to wakeup during the polling interval so it may as well be shrunk in order +to avoid pointless polling. The polling interval is shrunk in the function +shrink_halt_poll_ns() and is divided by the module parameter +halt_poll_ns_shrink, or set to 0 iff halt_poll_ns_shrink == 0. + +It is worth noting that this adjustment process attempts to hone in on some +steady state polling interval but will only really do a good job for wakeups +which come at an approximately constant rate, otherwise there will be constant +adjustment of the polling interval. + +[0] total block time: the time between when the halt polling function is + invoked and a wakeup source received (irrespective of + whether the scheduler is invoked within that function). + +Module Parameters +================= + +The kvm module has 3 tuneable module parameters to adjust the global max +polling interval as well as the rate at which the polling interval is grown and +shrunk. These variables are defined in include/linux/kvm_host.h and as module +parameters in virt/kvm/kvm_main.c, or arch/powerpc/kvm/book3s_hv.c in the +powerpc kvm-hv case. + +Module Parameter | Description | Default Value +-------------------------------------------------------------------------------- +halt_poll_ns | The global max polling interval | KVM_HALT_POLL_NS_DEFAULT + | which defines the ceiling value | + | of the polling interval for | (per arch value) + | each vcpu. | +-------------------------------------------------------------------------------- +halt_poll_ns_grow | The value by which the halt | 2 + | polling interval is multiplied | + | in the grow_halt_poll_ns() | + | function. | +-------------------------------------------------------------------------------- +halt_poll_ns_shrink | The value by which the halt | 0 + | polling interval is divided in | + | the shrink_halt_poll_ns() | + | function. | +-------------------------------------------------------------------------------- + +These module parameters can be set from the debugfs files in: + + /sys/module/kvm/parameters/ + +Note: that these module parameters are system wide values and are not able to + be tuned on a per vm basis. + +Further Notes +============= + +- Care should be taken when setting the halt_poll_ns module parameter as a +large value has the potential to drive the cpu usage to 100% on a machine which +would be almost entirely idle otherwise. This is because even if a guest has +wakeups during which very little work is done and which are quite far apart, if +the period is shorter than the global max polling interval (halt_poll_ns) then +the host will always poll for the entire block time and thus cpu utilisation +will go to 100%. + +- Halt polling essentially presents a trade off between power usage and latency +and the module parameters should be used to tune the affinity for this. Idle +cpu time is essentially converted to host kernel time with the aim of decreasing +latency when entering the guest. + +- Halt polling will only be conducted by the host when no other tasks are +runnable on that cpu, otherwise the polling will cease immediately and +schedule will be invoked to allow that other task to run. Thus this doesn't +allow a guest to denial of service the cpu. diff --git a/MAINTAINERS b/MAINTAINERS index bbb4450..e376bbd 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -3192,15 +3192,15 @@ S: Supported F: drivers/clocksource CISCO FCOE HBA DRIVER -M: Hiral Patel <hiralpat@cisco.com> -M: Suma Ramars <sramars@cisco.com> -M: Brian Uchino <buchino@cisco.com> +M: Satish Kharat <satishkh@cisco.com> +M: Sesidhar Baddela <sebaddel@cisco.com> +M: Karan Tilak Kumar <kartilak@cisco.com> L: linux-scsi@vger.kernel.org S: Supported F: drivers/scsi/fnic/ CISCO SCSI HBA DRIVER -M: Narsimhulu Musini <nmusini@cisco.com> +M: Karan Tilak Kumar <kartilak@cisco.com> M: Sesidhar Baddela <sebaddel@cisco.com> L: linux-scsi@vger.kernel.org S: Supported @@ -4787,11 +4787,11 @@ M: David Woodhouse <dwmw2@infradead.org> L: linux-embedded@vger.kernel.org S: Maintained -EMULEX/AVAGO LPFC FC/FCOE SCSI DRIVER -M: James Smart <james.smart@avagotech.com> -M: Dick Kennedy <dick.kennedy@avagotech.com> +EMULEX/BROADCOM LPFC FC/FCOE SCSI DRIVER +M: James Smart <james.smart@broadcom.com> +M: Dick Kennedy <dick.kennedy@broadcom.com> L: linux-scsi@vger.kernel.org -W: http://www.avagotech.com +W: http://www.broadcom.com S: Supported F: drivers/scsi/lpfc/ @@ -5240,6 +5240,7 @@ F: include/linux/fscache*.h FS-CRYPTO: FILE SYSTEM LEVEL ENCRYPTION SUPPORT M: Theodore Y. Ts'o <tytso@mit.edu> M: Jaegeuk Kim <jaegeuk@kernel.org> +L: linux-fsdevel@vger.kernel.org S: Supported F: fs/crypto/ F: include/linux/fscrypto.h @@ -5716,7 +5717,6 @@ F: drivers/watchdog/hpwdt.c HEWLETT-PACKARD SMART ARRAY RAID DRIVER (hpsa) M: Don Brace <don.brace@microsemi.com> -L: iss_storagedev@hp.com L: esc.storagedev@microsemi.com L: linux-scsi@vger.kernel.org S: Supported @@ -5727,7 +5727,6 @@ F: include/uapi/linux/cciss*.h HEWLETT-PACKARD SMART CISS RAID DRIVER (cciss) M: Don Brace <don.brace@microsemi.com> -L: iss_storagedev@hp.com L: esc.storagedev@microsemi.com L: linux-scsi@vger.kernel.org S: Supported @@ -7967,12 +7966,12 @@ S: Maintained F: drivers/net/wireless/mediatek/mt7601u/ MEGARAID SCSI/SAS DRIVERS -M: Kashyap Desai <kashyap.desai@avagotech.com> -M: Sumit Saxena <sumit.saxena@avagotech.com> -M: Uday Lingala <uday.lingala@avagotech.com> -L: megaraidlinux.pdl@avagotech.com +M: Kashyap Desai <kashyap.desai@broadcom.com> +M: Sumit Saxena <sumit.saxena@broadcom.com> +M: Shivasharan S <shivasharan.srikanteshwara@broadcom.com> +L: megaraidlinux.pdl@broadcom.com L: linux-scsi@vger.kernel.org -W: http://www.lsi.com +W: http://www.avagotech.com/support/ S: Maintained F: Documentation/scsi/megaraid.txt F: drivers/scsi/megaraid.* @@ -8452,7 +8451,6 @@ F: drivers/scsi/arm/oak.c F: drivers/scsi/atari_scsi.* F: drivers/scsi/dmx3191d.c F: drivers/scsi/g_NCR5380.* -F: drivers/scsi/g_NCR5380_mmio.c F: drivers/scsi/mac_scsi.* F: drivers/scsi/sun3_scsi.* F: drivers/scsi/sun3_scsi_vme.c @@ -10557,7 +10555,7 @@ F: arch/s390/pci/ F: drivers/pci/hotplug/s390_pci_hpc.c S390 ZCRYPT DRIVER -M: Ingo Tuchscherer <ingo.tuchscherer@de.ibm.com> +M: Harald Freudenberger <freude@de.ibm.com> L: linux-s390@vger.kernel.org W: http://www.ibm.com/developerworks/linux/linux390/ S: Supported @@ -12546,7 +12544,8 @@ F: Documentation/scsi/ufs.txt F: drivers/scsi/ufs/ UNIVERSAL FLASH STORAGE HOST CONTROLLER DRIVER DWC HOOKS -M: Joao Pinto <Joao.Pinto@synopsys.com> +M: Manjunath M Bettegowda <manjumb@synopsys.com> +M: Prabu Thangamuthu <prabut@synopsys.com> L: linux-scsi@vger.kernel.org S: Supported F: drivers/scsi/ufs/*dwc* @@ -13354,7 +13353,6 @@ F: drivers/media/tuners/tuner-xc2028.* XEN HYPERVISOR INTERFACE M: Boris Ostrovsky <boris.ostrovsky@oracle.com> -M: David Vrabel <david.vrabel@citrix.com> M: Juergen Gross <jgross@suse.com> L: xen-devel@lists.xenproject.org (moderated for non-subscribers) T: git git://git.kernel.org/pub/scm/linux/kernel/git/xen/tip.git diff --git a/arch/arm/include/asm/xen/hypercall.h b/arch/arm/include/asm/xen/hypercall.h index 9d874db..3522cba 100644 --- a/arch/arm/include/asm/xen/hypercall.h +++ b/arch/arm/include/asm/xen/hypercall.h @@ -1,87 +1 @@ -/****************************************************************************** - * hypercall.h - * - * Linux-specific hypervisor handling. - * - * Stefano Stabellini <stefano.stabellini@eu.citrix.com>, Citrix, 2012 - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License version 2 - * as published by the Free Software Foundation; or, when distributed - * separately from the Linux kernel or incorporated into other - * software packages, subject to the following license: - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this source file (the "Software"), to deal in the Software without - * restriction, including without limitation the rights to use, copy, modify, - * merge, publish, distribute, sublicense, and/or sell copies of the Software, - * and to permit persons to whom the Software is furnished to do so, subject to - * the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -#ifndef _ASM_ARM_XEN_HYPERCALL_H -#define _ASM_ARM_XEN_HYPERCALL_H - -#include <linux/bug.h> - -#include <xen/interface/xen.h> -#include <xen/interface/sched.h> -#include <xen/interface/platform.h> - -long privcmd_call(unsigned call, unsigned long a1, - unsigned long a2, unsigned long a3, - unsigned long a4, unsigned long a5); -int HYPERVISOR_xen_version(int cmd, void *arg); -int HYPERVISOR_console_io(int cmd, int count, char *str); -int HYPERVISOR_grant_table_op(unsigned int cmd, void *uop, unsigned int count); -int HYPERVISOR_sched_op(int cmd, void *arg); -int HYPERVISOR_event_channel_op(int cmd, void *arg); -unsigned long HYPERVISOR_hvm_op(int op, void *arg); -int HYPERVISOR_memory_op(unsigned int cmd, void *arg); -int HYPERVISOR_physdev_op(int cmd, void *arg); -int HYPERVISOR_vcpu_op(int cmd, int vcpuid, void *extra_args); -int HYPERVISOR_tmem_op(void *arg); -int HYPERVISOR_vm_assist(unsigned int cmd, unsigned int type); -int HYPERVISOR_platform_op_raw(void *arg); -static inline int HYPERVISOR_platform_op(struct xen_platform_op *op) -{ - op->interface_version = XENPF_INTERFACE_VERSION; - return HYPERVISOR_platform_op_raw(op); -} -int HYPERVISOR_multicall(struct multicall_entry *calls, uint32_t nr); - -static inline int -HYPERVISOR_suspend(unsigned long start_info_mfn) -{ - struct sched_shutdown r = { .reason = SHUTDOWN_suspend }; - - /* start_info_mfn is unused on ARM */ - return HYPERVISOR_sched_op(SCHEDOP_shutdown, &r); -} - -static inline void -MULTI_update_va_mapping(struct multicall_entry *mcl, unsigned long va, - unsigned int new_val, unsigned long flags) -{ - BUG(); -} - -static inline void -MULTI_mmu_update(struct multicall_entry *mcl, struct mmu_update *req, - int count, int *success_count, domid_t domid) -{ - BUG(); -} - -#endif /* _ASM_ARM_XEN_HYPERCALL_H */ +#include <xen/arm/hypercall.h> diff --git a/arch/arm/include/asm/xen/hypervisor.h b/arch/arm/include/asm/xen/hypervisor.h index 9525151..d6e7709 100644 --- a/arch/arm/include/asm/xen/hypervisor.h +++ b/arch/arm/include/asm/xen/hypervisor.h @@ -1,39 +1 @@ -#ifndef _ASM_ARM_XEN_HYPERVISOR_H -#define _ASM_ARM_XEN_HYPERVISOR_H - -#include <linux/init.h> - -extern struct shared_info *HYPERVISOR_shared_info; -extern struct start_info *xen_start_info; - -/* Lazy mode for batching updates / context switch */ -enum paravirt_lazy_mode { - PARAVIRT_LAZY_NONE, - PARAVIRT_LAZY_MMU, - PARAVIRT_LAZY_CPU, -}; - -static inline enum paravirt_lazy_mode paravirt_get_lazy_mode(void) -{ - return PARAVIRT_LAZY_NONE; -} - -extern struct dma_map_ops *xen_dma_ops; - -#ifdef CONFIG_XEN -void __init xen_early_init(void); -#else -static inline void xen_early_init(void) { return; } -#endif - -#ifdef CONFIG_HOTPLUG_CPU -static inline void xen_arch_register_cpu(int num) -{ -} - -static inline void xen_arch_unregister_cpu(int num) -{ -} -#endif - -#endif /* _ASM_ARM_XEN_HYPERVISOR_H */ +#include <xen/arm/hypervisor.h> diff --git a/arch/arm/include/asm/xen/interface.h b/arch/arm/include/asm/xen/interface.h index 75d5968..88c0d75 100644 --- a/arch/arm/include/asm/xen/interface.h +++ b/arch/arm/include/asm/xen/interface.h @@ -1,85 +1 @@ -/****************************************************************************** - * Guest OS interface to ARM Xen. - * - * Stefano Stabellini <stefano.stabellini@eu.citrix.com>, Citrix, 2012 - */ - -#ifndef _ASM_ARM_XEN_INTERFACE_H -#define _ASM_ARM_XEN_INTERFACE_H - -#include <linux/types.h> - -#define uint64_aligned_t uint64_t __attribute__((aligned(8))) - -#define __DEFINE_GUEST_HANDLE(name, type) \ - typedef struct { union { type *p; uint64_aligned_t q; }; } \ - __guest_handle_ ## name - -#define DEFINE_GUEST_HANDLE_STRUCT(name) \ - __DEFINE_GUEST_HANDLE(name, struct name) -#define DEFINE_GUEST_HANDLE(name) __DEFINE_GUEST_HANDLE(name, name) -#define GUEST_HANDLE(name) __guest_handle_ ## name - -#define set_xen_guest_handle(hnd, val) \ - do { \ - if (sizeof(hnd) == 8) \ - *(uint64_t *)&(hnd) = 0; \ - (hnd).p = val; \ - } while (0) - -#define __HYPERVISOR_platform_op_raw __HYPERVISOR_platform_op - -#ifndef __ASSEMBLY__ -/* Explicitly size integers that represent pfns in the interface with - * Xen so that we can have one ABI that works for 32 and 64 bit guests. - * Note that this means that the xen_pfn_t type may be capable of - * representing pfn's which the guest cannot represent in its own pfn - * type. However since pfn space is controlled by the guest this is - * fine since it simply wouldn't be able to create any sure pfns in - * the first place. - */ -typedef uint64_t xen_pfn_t; -#define PRI_xen_pfn "llx" -typedef uint64_t xen_ulong_t; -#define PRI_xen_ulong "llx" -typedef int64_t xen_long_t; -#define PRI_xen_long "llx" -/* Guest handles for primitive C types. */ -__DEFINE_GUEST_HANDLE(uchar, unsigned char); -__DEFINE_GUEST_HANDLE(uint, unsigned int); -DEFINE_GUEST_HANDLE(char); -DEFINE_GUEST_HANDLE(int); -DEFINE_GUEST_HANDLE(void); -DEFINE_GUEST_HANDLE(uint64_t); -DEFINE_GUEST_HANDLE(uint32_t); -DEFINE_GUEST_HANDLE(xen_pfn_t); -DEFINE_GUEST_HANDLE(xen_ulong_t); - -/* Maximum number of virtual CPUs in multi-processor guests. */ -#define MAX_VIRT_CPUS 1 - -struct arch_vcpu_info { }; -struct arch_shared_info { }; - -/* TODO: Move pvclock definitions some place arch independent */ -struct pvclock_vcpu_time_info { - u32 version; - u32 pad0; - u64 tsc_timestamp; - u64 system_time; - u32 tsc_to_system_mul; - s8 tsc_shift; - u8 flags; - u8 pad[2]; -} __attribute__((__packed__)); /* 32 bytes */ - -/* It is OK to have a 12 bytes struct with no padding because it is packed */ -struct pvclock_wall_clock { - u32 version; - u32 sec; - u32 nsec; - u32 sec_hi; -} __attribute__((__packed__)); -#endif - -#endif /* _ASM_ARM_XEN_INTERFACE_H */ +#include <xen/arm/interface.h> diff --git a/arch/arm/include/asm/xen/page-coherent.h b/arch/arm/include/asm/xen/page-coherent.h index 95ce6ac..b3ef061 100644 --- a/arch/arm/include/asm/xen/page-coherent.h +++ b/arch/arm/include/asm/xen/page-coherent.h @@ -1,98 +1 @@ -#ifndef _ASM_ARM_XEN_PAGE_COHERENT_H -#define _ASM_ARM_XEN_PAGE_COHERENT_H - -#include <asm/page.h> -#include <linux/dma-mapping.h> - -void __xen_dma_map_page(struct device *hwdev, struct page *page, - dma_addr_t dev_addr, unsigned long offset, size_t size, - enum dma_data_direction dir, unsigned long attrs); -void __xen_dma_unmap_page(struct device *hwdev, dma_addr_t handle, - size_t size, enum dma_data_direction dir, - unsigned long attrs); -void __xen_dma_sync_single_for_cpu(struct device *hwdev, - dma_addr_t handle, size_t size, enum dma_data_direction dir); - -void __xen_dma_sync_single_for_device(struct device *hwdev, - dma_addr_t handle, size_t size, enum dma_data_direction dir); - -static inline void *xen_alloc_coherent_pages(struct device *hwdev, size_t size, - dma_addr_t *dma_handle, gfp_t flags, unsigned long attrs) -{ - return __generic_dma_ops(hwdev)->alloc(hwdev, size, dma_handle, flags, attrs); -} - -static inline void xen_free_coherent_pages(struct device *hwdev, size_t size, - void *cpu_addr, dma_addr_t dma_handle, unsigned long attrs) -{ - __generic_dma_ops(hwdev)->free(hwdev, size, cpu_addr, dma_handle, attrs); -} - -static inline void xen_dma_map_page(struct device *hwdev, struct page *page, - dma_addr_t dev_addr, unsigned long offset, size_t size, - enum dma_data_direction dir, unsigned long attrs) -{ - unsigned long page_pfn = page_to_xen_pfn(page); - unsigned long dev_pfn = XEN_PFN_DOWN(dev_addr); - unsigned long compound_pages = - (1<<compound_order(page)) * XEN_PFN_PER_PAGE; - bool local = (page_pfn <= dev_pfn) && - (dev_pfn - page_pfn < compound_pages); - - /* - * Dom0 is mapped 1:1, while the Linux page can span across - * multiple Xen pages, it's not possible for it to contain a - * mix of local and foreign Xen pages. So if the first xen_pfn - * == mfn the page is local otherwise it's a foreign page - * grant-mapped in dom0. If the page is local we can safely - * call the native dma_ops function, otherwise we call the xen - * specific function. - */ - if (local) - __generic_dma_ops(hwdev)->map_page(hwdev, page, offset, size, dir, attrs); - else - __xen_dma_map_page(hwdev, page, dev_addr, offset, size, dir, attrs); -} - -static inline void xen_dma_unmap_page(struct device *hwdev, dma_addr_t handle, - size_t size, enum dma_data_direction dir, unsigned long attrs) -{ - unsigned long pfn = PFN_DOWN(handle); - /* - * Dom0 is mapped 1:1, while the Linux page can be spanned accross - * multiple Xen page, it's not possible to have a mix of local and - * foreign Xen page. Dom0 is mapped 1:1, so calling pfn_valid on a - * foreign mfn will always return false. If the page is local we can - * safely call the native dma_ops function, otherwise we call the xen - * specific function. - */ - if (pfn_valid(pfn)) { - if (__generic_dma_ops(hwdev)->unmap_page) - __generic_dma_ops(hwdev)->unmap_page(hwdev, handle, size, dir, attrs); - } else - __xen_dma_unmap_page(hwdev, handle, size, dir, attrs); -} - -static inline void xen_dma_sync_single_for_cpu(struct device *hwdev, - dma_addr_t handle, size_t size, enum dma_data_direction dir) -{ - unsigned long pfn = PFN_DOWN(handle); - if (pfn_valid(pfn)) { - if (__generic_dma_ops(hwdev)->sync_single_for_cpu) - __generic_dma_ops(hwdev)->sync_single_for_cpu(hwdev, handle, size, dir); - } else - __xen_dma_sync_single_for_cpu(hwdev, handle, size, dir); -} - -static inline void xen_dma_sync_single_for_device(struct device *hwdev, - dma_addr_t handle, size_t size, enum dma_data_direction dir) -{ - unsigned long pfn = PFN_DOWN(handle); - if (pfn_valid(pfn)) { - if (__generic_dma_ops(hwdev)->sync_single_for_device) - __generic_dma_ops(hwdev)->sync_single_for_device(hwdev, handle, size, dir); - } else - __xen_dma_sync_single_for_device(hwdev, handle, size, dir); -} - -#endif /* _ASM_ARM_XEN_PAGE_COHERENT_H */ +#include <xen/arm/page-coherent.h> diff --git a/arch/arm/include/asm/xen/page.h b/arch/arm/include/asm/xen/page.h index 415dbc6..31bbc80 100644 --- a/arch/arm/include/asm/xen/page.h +++ b/arch/arm/include/asm/xen/page.h @@ -1,122 +1 @@ -#ifndef _ASM_ARM_XEN_PAGE_H -#define _ASM_ARM_XEN_PAGE_H - -#include <asm/page.h> -#include <asm/pgtable.h> - -#include <linux/pfn.h> -#include <linux/types.h> -#include <linux/dma-mapping.h> - -#include <xen/xen.h> -#include <xen/interface/grant_table.h> - -#define phys_to_machine_mapping_valid(pfn) (1) - -/* Xen machine address */ -typedef struct xmaddr { - phys_addr_t maddr; -} xmaddr_t; - -/* Xen pseudo-physical address */ -typedef struct xpaddr { - phys_addr_t paddr; -} xpaddr_t; - -#define XMADDR(x) ((xmaddr_t) { .maddr = (x) }) -#define XPADDR(x) ((xpaddr_t) { .paddr = (x) }) - -#define INVALID_P2M_ENTRY (~0UL) - -/* - * The pseudo-physical frame (pfn) used in all the helpers is always based - * on Xen page granularity (i.e 4KB). - * - * A Linux page may be split across multiple non-contiguous Xen page so we - * have to keep track with frame based on 4KB page granularity. - * - * PV drivers should never make a direct usage of those helpers (particularly - * pfn_to_gfn and gfn_to_pfn). - */ - -unsigned long __pfn_to_mfn(unsigned long pfn); -extern struct rb_root phys_to_mach; - -/* Pseudo-physical <-> Guest conversion */ -static inline unsigned long pfn_to_gfn(unsigned long pfn) -{ - return pfn; -} - -static inline unsigned long gfn_to_pfn(unsigned long gfn) -{ - return gfn; -} - -/* Pseudo-physical <-> BUS conversion */ -static inline unsigned long pfn_to_bfn(unsigned long pfn) -{ - unsigned long mfn; - - if (phys_to_mach.rb_node != NULL) { - mfn = __pfn_to_mfn(pfn); - if (mfn != INVALID_P2M_ENTRY) - return mfn; - } - - return pfn; -} - -static inline unsigned long bfn_to_pfn(unsigned long bfn) -{ - return bfn; -} - -#define bfn_to_local_pfn(bfn) bfn_to_pfn(bfn) - -/* VIRT <-> GUEST conversion */ -#define virt_to_gfn(v) (pfn_to_gfn(virt_to_phys(v) >> XEN_PAGE_SHIFT)) -#define gfn_to_virt(m) (__va(gfn_to_pfn(m) << XEN_PAGE_SHIFT)) - -/* Only used in PV code. But ARM guests are always HVM. */ -static inline xmaddr_t arbitrary_virt_to_machine(void *vaddr) -{ - BUG(); -} - -/* TODO: this shouldn't be here but it is because the frontend drivers - * are using it (its rolled in headers) even though we won't hit the code path. - * So for right now just punt with this. - */ -static inline pte_t *lookup_address(unsigned long address, unsigned int *level) -{ - BUG(); - return NULL; -} - -extern int set_foreign_p2m_mapping(struct gnttab_map_grant_ref *map_ops, - struct gnttab_map_grant_ref *kmap_ops, - struct page **pages, unsigned int count); - -extern int clear_foreign_p2m_mapping(struct gnttab_unmap_grant_ref *unmap_ops, - struct gnttab_unmap_grant_ref *kunmap_ops, - struct page **pages, unsigned int count); - -bool __set_phys_to_machine(unsigned long pfn, unsigned long mfn); -bool __set_phys_to_machine_multi(unsigned long pfn, unsigned long mfn, - unsigned long nr_pages); - -static inline bool set_phys_to_machine(unsigned long pfn, unsigned long mfn) -{ - return __set_phys_to_machine(pfn, mfn); -} - -#define xen_remap(cookie, size) ioremap_cache((cookie), (size)) -#define xen_unmap(cookie) iounmap((cookie)) - -bool xen_arch_need_swiotlb(struct device *dev, - phys_addr_t phys, - dma_addr_t dev_addr); -unsigned long xen_get_swiotlb_free_pages(unsigned int order); - -#endif /* _ASM_ARM_XEN_PAGE_H */ +#include <xen/arm/page.h> diff --git a/arch/arm/include/uapi/asm/kvm.h b/arch/arm/include/uapi/asm/kvm.h index b38c10c..af05f8e 100644 --- a/arch/arm/include/uapi/asm/kvm.h +++ b/arch/arm/include/uapi/asm/kvm.h @@ -87,9 +87,11 @@ struct kvm_regs { /* Supported VGICv3 address types */ #define KVM_VGIC_V3_ADDR_TYPE_DIST 2 #define KVM_VGIC_V3_ADDR_TYPE_REDIST 3 +#define KVM_VGIC_ITS_ADDR_TYPE 4 #define KVM_VGIC_V3_DIST_SIZE SZ_64K #define KVM_VGIC_V3_REDIST_SIZE (2 * SZ_64K) +#define KVM_VGIC_V3_ITS_SIZE (2 * SZ_64K) #define KVM_ARM_VCPU_POWER_OFF 0 /* CPU is started in OFF state */ #define KVM_ARM_VCPU_PSCI_0_2 1 /* CPU uses PSCI v0.2 */ diff --git a/arch/arm/kvm/Kconfig b/arch/arm/kvm/Kconfig index 3e1cd04..90d0176 100644 --- a/arch/arm/kvm/Kconfig +++ b/arch/arm/kvm/Kconfig @@ -34,6 +34,7 @@ config KVM select HAVE_KVM_IRQFD select HAVE_KVM_IRQCHIP select HAVE_KVM_IRQ_ROUTING + select HAVE_KVM_MSI depends on ARM_VIRT_EXT && ARM_LPAE && ARM_ARCH_TIMER ---help--- Support hosting virtualized guest machines. diff --git a/arch/arm/kvm/Makefile b/arch/arm/kvm/Makefile index f19842e..d571243 100644 --- a/arch/arm/kvm/Makefile +++ b/arch/arm/kvm/Makefile @@ -32,5 +32,6 @@ obj-y += $(KVM)/arm/vgic/vgic-mmio.o obj-y += $(KVM)/arm/vgic/vgic-mmio-v2.o obj-y += $(KVM)/arm/vgic/vgic-mmio-v3.o obj-y += $(KVM)/arm/vgic/vgic-kvm-device.o +obj-y += $(KVM)/arm/vgic/vgic-its.o obj-y += $(KVM)/irqchip.o obj-y += $(KVM)/arm/arch_timer.o diff --git a/arch/arm/kvm/arm.c b/arch/arm/kvm/arm.c index 19b5f5c..8f92efa 100644 --- a/arch/arm/kvm/arm.c +++ b/arch/arm/kvm/arm.c @@ -221,6 +221,12 @@ int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext) case KVM_CAP_MAX_VCPUS: r = KVM_MAX_VCPUS; break; + case KVM_CAP_MSI_DEVID: + if (!kvm) + r = -EINVAL; + else + r = kvm->arch.vgic.msis_require_devid; + break; default: r = kvm_arch_dev_ioctl_check_extension(kvm, ext); break; diff --git a/arch/arm/xen/enlighten.c b/arch/arm/xen/enlighten.c index f193414..4986dc0 100644 --- a/arch/arm/xen/enlighten.c +++ b/arch/arm/xen/enlighten.c @@ -372,8 +372,7 @@ static int __init xen_guest_init(void) * for secondary CPUs as they are brought up. * For uniformity we use VCPUOP_register_vcpu_info even on cpu0. */ - xen_vcpu_info = __alloc_percpu(sizeof(struct vcpu_info), - sizeof(struct vcpu_info)); + xen_vcpu_info = alloc_percpu(struct vcpu_info); if (xen_vcpu_info == NULL) return -ENOMEM; diff --git a/arch/arm/xen/mm.c b/arch/arm/xen/mm.c index d062f08..bd62d94 100644 --- a/arch/arm/xen/mm.c +++ b/arch/arm/xen/mm.c @@ -186,7 +186,6 @@ struct dma_map_ops *xen_dma_ops; EXPORT_SYMBOL(xen_dma_ops); static struct dma_map_ops xen_swiotlb_dma_ops = { - .mapping_error = xen_swiotlb_dma_mapping_error, .alloc = xen_swiotlb_alloc_coherent, .free = xen_swiotlb_free_coherent, .sync_single_for_cpu = xen_swiotlb_sync_single_for_cpu, diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig index 657be7f..1117421 100644 --- a/arch/arm64/Kconfig +++ b/arch/arm64/Kconfig @@ -110,6 +110,7 @@ config ARM64 select POWER_SUPPLY select SPARSE_IRQ select SYSCTL_EXCEPTION_TRACE + select THREAD_INFO_IN_TASK help ARM 64-bit (AArch64) Linux support. @@ -239,6 +240,9 @@ config PGTABLE_LEVELS default 3 if ARM64_16K_PAGES && ARM64_VA_BITS_47 default 4 if !ARM64_64K_PAGES && ARM64_VA_BITS_48 +config ARCH_SUPPORTS_UPROBES + def_bool y + source "init/Kconfig" source "kernel/Kconfig.freezer" @@ -791,6 +795,14 @@ config SETEND_EMULATION If unsure, say Y endif +config ARM64_SW_TTBR0_PAN + bool "Emulate Privileged Access Never using TTBR0_EL1 switching" + help + Enabling this option prevents the kernel from accessing + user-space memory directly by pointing TTBR0_EL1 to a reserved + zeroed area and reserved ASID. The user access routines + restore the valid TTBR0_EL1 temporarily. + menu "ARMv8.1 architectural features" config ARM64_HW_AFDBM diff --git a/arch/arm64/Kconfig.debug b/arch/arm64/Kconfig.debug index b661fe7..d1ebd46 100644 --- a/arch/arm64/Kconfig.debug +++ b/arch/arm64/Kconfig.debug @@ -2,9 +2,13 @@ menu "Kernel hacking" source "lib/Kconfig.debug" -config ARM64_PTDUMP +config ARM64_PTDUMP_CORE + def_bool n + +config ARM64_PTDUMP_DEBUGFS bool "Export kernel pagetable layout to userspace via debugfs" depends on DEBUG_KERNEL + select ARM64_PTDUMP_CORE select DEBUG_FS help Say Y here if you want to show the kernel pagetable layout in a @@ -38,6 +42,35 @@ config ARM64_RANDOMIZE_TEXT_OFFSET of TEXT_OFFSET and platforms must not require a specific value. +config DEBUG_WX + bool "Warn on W+X mappings at boot" + select ARM64_PTDUMP_CORE + ---help--- + Generate a warning if any W+X mappings are found at boot. + + This is useful for discovering cases where the kernel is leaving + W+X mappings after applying NX, as such mappings are a security risk. + This check also includes UXN, which should be set on all kernel + mappings. + + Look for a message in dmesg output like this: + + arm64/mm: Checked W+X mappings: passed, no W+X pages found. + + or like this, if the check failed: + + arm64/mm: Checked W+X mappings: FAILED, <N> W+X pages found. + + Note that even if the check fails, your kernel is possibly + still fine, as W+X mappings are not a security hole in + themselves, what they do is that they make the exploitation + of other unfixed kernel bugs easier. + + There is no runtime or memory usage effect of this option + once the kernel has booted up - it's a one time check. + + If in doubt, say "Y". + config DEBUG_SET_MODULE_RONX bool "Set loadable kernel module data as NX and text as RO" depends on MODULES diff --git a/arch/arm64/Makefile b/arch/arm64/Makefile index 3635b86..b9a4a93 100644 --- a/arch/arm64/Makefile +++ b/arch/arm64/Makefile @@ -37,10 +37,16 @@ $(warning LSE atomics not supported by binutils) endif endif -KBUILD_CFLAGS += -mgeneral-regs-only $(lseinstr) +brokengasinst := $(call as-instr,1:\n.inst 0\n.rept . - 1b\n\nnop\n.endr\n,,-DCONFIG_BROKEN_GAS_INST=1) + +ifneq ($(brokengasinst),) +$(warning Detected assembler with broken .inst; disassembly will be unreliable) +endif + +KBUILD_CFLAGS += -mgeneral-regs-only $(lseinstr) $(brokengasinst) KBUILD_CFLAGS += -fno-asynchronous-unwind-tables KBUILD_CFLAGS += $(call cc-option, -mpc-relative-literal-loads) -KBUILD_AFLAGS += $(lseinstr) +KBUILD_AFLAGS += $(lseinstr) $(brokengasinst) ifeq ($(CONFIG_CPU_BIG_ENDIAN), y) KBUILD_CPPFLAGS += -mbig-endian diff --git a/arch/arm64/configs/defconfig b/arch/arm64/configs/defconfig index 6be0811..c3caadd 100644 --- a/arch/arm64/configs/defconfig +++ b/arch/arm64/configs/defconfig @@ -82,6 +82,7 @@ CONFIG_KEXEC=y # CONFIG_CORE_DUMP_DEFAULT_ELF_HEADERS is not set CONFIG_COMPAT=y CONFIG_CPU_IDLE=y +CONFIG_HIBERNATION=y CONFIG_ARM_CPUIDLE=y CONFIG_CPU_FREQ=y CONFIG_CPUFREQ_DT=y diff --git a/arch/arm64/include/asm/Kbuild b/arch/arm64/include/asm/Kbuild index b4ab238..8365a84 100644 --- a/arch/arm64/include/asm/Kbuild +++ b/arch/arm64/include/asm/Kbuild @@ -1,7 +1,6 @@ generic-y += bugs.h generic-y += clkdev.h generic-y += cputime.h -generic-y += current.h generic-y += delay.h generic-y += div64.h generic-y += dma.h diff --git a/arch/arm64/include/asm/assembler.h b/arch/arm64/include/asm/assembler.h index 28bfe61..446f6c4 100644 --- a/arch/arm64/include/asm/assembler.h +++ b/arch/arm64/include/asm/assembler.h @@ -41,6 +41,15 @@ msr daifclr, #2 .endm + .macro save_and_disable_irq, flags + mrs \flags, daif + msr daifset, #2 + .endm + + .macro restore_irq, flags + msr daif, \flags + .endm + /* * Enable and disable debug exceptions. */ @@ -202,14 +211,25 @@ lr .req x30 // link register .endm /* + * @dst: Result of per_cpu(sym, smp_processor_id()) * @sym: The name of the per-cpu variable - * @reg: Result of per_cpu(sym, smp_processor_id()) * @tmp: scratch register */ - .macro this_cpu_ptr, sym, reg, tmp - adr_l \reg, \sym + .macro adr_this_cpu, dst, sym, tmp + adr_l \dst, \sym mrs \tmp, tpidr_el1 - add \reg, \reg, \tmp + add \dst, \dst, \tmp + .endm + + /* + * @dst: Result of READ_ONCE(per_cpu(sym, smp_processor_id())) + * @sym: The name of the per-cpu variable + * @tmp: scratch register + */ + .macro ldr_this_cpu dst, sym, tmp + adr_l \dst, \sym + mrs \tmp, tpidr_el1 + ldr \dst, [\dst, \tmp] .endm /* @@ -395,4 +415,24 @@ alternative_endif movk \reg, :abs_g0_nc:\val .endm +/* + * Return the current thread_info. + */ + .macro get_thread_info, rd + mrs \rd, sp_el0 + .endm + +/* + * Errata workaround post TTBR0_EL1 update. + */ + .macro post_ttbr0_update_workaround +#ifdef CONFIG_CAVIUM_ERRATUM_27456 +alternative_if ARM64_WORKAROUND_CAVIUM_27456 + ic iallu + dsb nsh + isb +alternative_else_nop_endif +#endif + .endm + #endif /* __ASM_ASSEMBLER_H */ diff --git a/arch/arm64/include/asm/cacheflush.h b/arch/arm64/include/asm/cacheflush.h index 2e5fb97..5a2a6ee 100644 --- a/arch/arm64/include/asm/cacheflush.h +++ b/arch/arm64/include/asm/cacheflush.h @@ -65,12 +65,12 @@ * - kaddr - page address * - size - region size */ -extern void flush_cache_range(struct vm_area_struct *vma, unsigned long start, unsigned long end); extern void flush_icache_range(unsigned long start, unsigned long end); extern void __flush_dcache_area(void *addr, size_t len); extern void __clean_dcache_area_poc(void *addr, size_t len); extern void __clean_dcache_area_pou(void *addr, size_t len); extern long __flush_cache_user_range(unsigned long start, unsigned long end); +extern void sync_icache_aliases(void *kaddr, unsigned long len); static inline void flush_cache_mm(struct mm_struct *mm) { @@ -81,6 +81,11 @@ static inline void flush_cache_page(struct vm_area_struct *vma, { } +static inline void flush_cache_range(struct vm_area_struct *vma, + unsigned long start, unsigned long end) +{ +} + /* * Cache maintenance functions used by the DMA API. No to be used directly. */ diff --git a/arch/arm64/include/asm/cpucaps.h b/arch/arm64/include/asm/cpucaps.h index 87b4465..4174f09 100644 --- a/arch/arm64/include/asm/cpucaps.h +++ b/arch/arm64/include/asm/cpucaps.h @@ -34,7 +34,8 @@ #define ARM64_HAS_32BIT_EL0 13 #define ARM64_HYP_OFFSET_LOW 14 #define ARM64_MISMATCHED_CACHE_LINE_SIZE 15 +#define ARM64_HAS_NO_FPSIMD 16 -#define ARM64_NCAPS 16 +#define ARM64_NCAPS 17 #endif /* __ASM_CPUCAPS_H */ diff --git a/arch/arm64/include/asm/cpufeature.h b/arch/arm64/include/asm/cpufeature.h index 0bc0b1d..b4989df 100644 --- a/arch/arm64/include/asm/cpufeature.h +++ b/arch/arm64/include/asm/cpufeature.h @@ -9,8 +9,6 @@ #ifndef __ASM_CPUFEATURE_H #define __ASM_CPUFEATURE_H -#include <linux/jump_label.h> - #include <asm/cpucaps.h> #include <asm/hwcap.h> #include <asm/sysreg.h> @@ -27,6 +25,8 @@ #ifndef __ASSEMBLY__ +#include <linux/bug.h> +#include <linux/jump_label.h> #include <linux/kernel.h> /* CPU feature register tracking */ @@ -104,14 +104,19 @@ static inline bool cpu_have_feature(unsigned int num) return elf_hwcap & (1UL << num); } +/* System capability check for constant caps */ +static inline bool cpus_have_const_cap(int num) +{ + if (num >= ARM64_NCAPS) + return false; + return static_branch_unlikely(&cpu_hwcap_keys[num]); +} + static inline bool cpus_have_cap(unsigned int num) { if (num >= ARM64_NCAPS) return false; - if (__builtin_constant_p(num)) - return static_branch_unlikely(&cpu_hwcap_keys[num]); - else - return test_bit(num, cpu_hwcaps); + return test_bit(num, cpu_hwcaps); } static inline void cpus_set_cap(unsigned int num) @@ -200,7 +205,7 @@ static inline bool cpu_supports_mixed_endian_el0(void) static inline bool system_supports_32bit_el0(void) { - return cpus_have_cap(ARM64_HAS_32BIT_EL0); + return cpus_have_const_cap(ARM64_HAS_32BIT_EL0); } static inline bool system_supports_mixed_endian_el0(void) @@ -208,6 +213,17 @@ static inline bool system_supports_mixed_endian_el0(void) return id_aa64mmfr0_mixed_endian_el0(read_system_reg(SYS_ID_AA64MMFR0_EL1)); } +static inline bool system_supports_fpsimd(void) +{ + return !cpus_have_const_cap(ARM64_HAS_NO_FPSIMD); +} + +static inline bool system_uses_ttbr0_pan(void) +{ + return IS_ENABLED(CONFIG_ARM64_SW_TTBR0_PAN) && + !cpus_have_cap(ARM64_HAS_PAN); +} + #endif /* __ASSEMBLY__ */ #endif diff --git a/arch/arm64/include/asm/current.h b/arch/arm64/include/asm/current.h new file mode 100644 index 0000000..f2bcbe2 --- /dev/null +++ b/arch/arm64/include/asm/current.h @@ -0,0 +1,22 @@ +#ifndef __ASM_CURRENT_H +#define __ASM_CURRENT_H + +#include <linux/compiler.h> + +#include <asm/sysreg.h> + +#ifndef __ASSEMBLY__ + +struct task_struct; + +static __always_inline struct task_struct *get_current(void) +{ + return (struct task_struct *)read_sysreg(sp_el0); +} + +#define current get_current() + +#endif /* __ASSEMBLY__ */ + +#endif /* __ASM_CURRENT_H */ + diff --git a/arch/arm64/include/asm/debug-monitors.h b/arch/arm64/include/asm/debug-monitors.h index b71420a..a44cf52 100644 --- a/arch/arm64/include/asm/debug-monitors.h +++ b/arch/arm64/include/asm/debug-monitors.h @@ -68,6 +68,9 @@ #define BRK64_ESR_MASK 0xFFFF #define BRK64_ESR_KPROBES 0x0004 #define BRK64_OPCODE_KPROBES (AARCH64_BREAK_MON | (BRK64_ESR_KPROBES << 5)) +/* uprobes BRK opcodes with ESR encoding */ +#define BRK64_ESR_UPROBES 0x0005 +#define BRK64_OPCODE_UPROBES (AARCH64_BREAK_MON | (BRK64_ESR_UPROBES << 5)) /* AArch32 */ #define DBG_ESR_EVT_BKPT 0x4 diff --git a/arch/arm64/include/asm/efi.h b/arch/arm64/include/asm/efi.h index 771b3f0..0b6b163 100644 --- a/arch/arm64/include/asm/efi.h +++ b/arch/arm64/include/asm/efi.h @@ -1,6 +1,7 @@ #ifndef _ASM_EFI_H #define _ASM_EFI_H +#include <asm/cpufeature.h> #include <asm/io.h> #include <asm/mmu_context.h> #include <asm/neon.h> @@ -78,7 +79,30 @@ static inline void efifb_setup_from_dmi(struct screen_info *si, const char *opt) static inline void efi_set_pgd(struct mm_struct *mm) { - switch_mm(NULL, mm, NULL); + __switch_mm(mm); + + if (system_uses_ttbr0_pan()) { + if (mm != current->active_mm) { + /* + * Update the current thread's saved ttbr0 since it is + * restored as part of a return from exception. Set + * the hardware TTBR0_EL1 using cpu_switch_mm() + * directly to enable potential errata workarounds. + */ + update_saved_ttbr0(current, mm); + cpu_switch_mm(mm->pgd, mm); + } else { + /* + * Defer the switch to the current thread's TTBR0_EL1 + * until uaccess_enable(). Restore the current + * thread's saved ttbr0 corresponding to its active_mm + * (if different from init_mm). + */ + cpu_set_reserved_ttbr0(); + if (current->active_mm != &init_mm) + update_saved_ttbr0(current, current->active_mm); + } + } } void efi_virtmap_load(void); diff --git a/arch/arm64/include/asm/elf.h b/arch/arm64/include/asm/elf.h index a55384f..5d17004 100644 --- a/arch/arm64/include/asm/elf.h +++ b/arch/arm64/include/asm/elf.h @@ -138,7 +138,11 @@ typedef struct user_fpsimd_state elf_fpregset_t; */ #define ELF_PLAT_INIT(_r, load_addr) (_r)->regs[0] = 0 -#define SET_PERSONALITY(ex) clear_thread_flag(TIF_32BIT); +#define SET_PERSONALITY(ex) \ +({ \ + clear_bit(TIF_32BIT, ¤t->mm->context.flags); \ + clear_thread_flag(TIF_32BIT); \ +}) /* update AT_VECTOR_SIZE_ARCH if the number of NEW_AUX_ENT entries changes */ #define ARCH_DLINFO \ @@ -183,7 +187,11 @@ typedef compat_elf_greg_t compat_elf_gregset_t[COMPAT_ELF_NGREG]; ((x)->e_flags & EF_ARM_EABI_MASK)) #define compat_start_thread compat_start_thread -#define COMPAT_SET_PERSONALITY(ex) set_thread_flag(TIF_32BIT); +#define COMPAT_SET_PERSONALITY(ex) \ +({ \ + set_bit(TIF_32BIT, ¤t->mm->context.flags); \ + set_thread_flag(TIF_32BIT); \ + }) #define COMPAT_ARCH_DLINFO extern int aarch32_setup_vectors_page(struct linux_binprm *bprm, int uses_interp); diff --git a/arch/arm64/include/asm/futex.h b/arch/arm64/include/asm/futex.h index f2585cd..85c4a89 100644 --- a/arch/arm64/include/asm/futex.h +++ b/arch/arm64/include/asm/futex.h @@ -21,15 +21,12 @@ #include <linux/futex.h> #include <linux/uaccess.h> -#include <asm/alternative.h> -#include <asm/cpufeature.h> #include <asm/errno.h> -#include <asm/sysreg.h> #define __futex_atomic_op(insn, ret, oldval, uaddr, tmp, oparg) \ +do { \ + uaccess_enable(); \ asm volatile( \ - ALTERNATIVE("nop", SET_PSTATE_PAN(0), ARM64_HAS_PAN, \ - CONFIG_ARM64_PAN) \ " prfm pstl1strm, %2\n" \ "1: ldxr %w1, %2\n" \ insn "\n" \ @@ -44,11 +41,11 @@ " .popsection\n" \ _ASM_EXTABLE(1b, 4b) \ _ASM_EXTABLE(2b, 4b) \ - ALTERNATIVE("nop", SET_PSTATE_PAN(1), ARM64_HAS_PAN, \ - CONFIG_ARM64_PAN) \ : "=&r" (ret), "=&r" (oldval), "+Q" (*uaddr), "=&r" (tmp) \ : "r" (oparg), "Ir" (-EFAULT) \ - : "memory") + : "memory"); \ + uaccess_disable(); \ +} while (0) static inline int futex_atomic_op_inuser (int encoded_op, u32 __user *uaddr) @@ -118,8 +115,8 @@ futex_atomic_cmpxchg_inatomic(u32 *uval, u32 __user *uaddr, if (!access_ok(VERIFY_WRITE, uaddr, sizeof(u32))) return -EFAULT; + uaccess_enable(); asm volatile("// futex_atomic_cmpxchg_inatomic\n" -ALTERNATIVE("nop", SET_PSTATE_PAN(0), ARM64_HAS_PAN, CONFIG_ARM64_PAN) " prfm pstl1strm, %2\n" "1: ldxr %w1, %2\n" " sub %w3, %w1, %w4\n" @@ -134,10 +131,10 @@ ALTERNATIVE("nop", SET_PSTATE_PAN(0), ARM64_HAS_PAN, CONFIG_ARM64_PAN) " .popsection\n" _ASM_EXTABLE(1b, 4b) _ASM_EXTABLE(2b, 4b) -ALTERNATIVE("nop", SET_PSTATE_PAN(1), ARM64_HAS_PAN, CONFIG_ARM64_PAN) : "+r" (ret), "=&r" (val), "+Q" (*uaddr), "=&r" (tmp) : "r" (oldval), "r" (newval), "Ir" (-EFAULT) : "memory"); + uaccess_disable(); *uval = val; return ret; diff --git a/arch/arm64/include/asm/hw_breakpoint.h b/arch/arm64/include/asm/hw_breakpoint.h index 9510ace..b6b167a 100644 --- a/arch/arm64/include/asm/hw_breakpoint.h +++ b/arch/arm64/include/asm/hw_breakpoint.h @@ -77,7 +77,11 @@ static inline void decode_ctrl_reg(u32 reg, /* Lengths */ #define ARM_BREAKPOINT_LEN_1 0x1 #define ARM_BREAKPOINT_LEN_2 0x3 +#define ARM_BREAKPOINT_LEN_3 0x7 #define ARM_BREAKPOINT_LEN_4 0xf +#define ARM_BREAKPOINT_LEN_5 0x1f +#define ARM_BREAKPOINT_LEN_6 0x3f +#define ARM_BREAKPOINT_LEN_7 0x7f #define ARM_BREAKPOINT_LEN_8 0xff /* Kernel stepping */ @@ -119,7 +123,7 @@ struct perf_event; struct pmu; extern int arch_bp_generic_fields(struct arch_hw_breakpoint_ctrl ctrl, - int *gen_len, int *gen_type); + int *gen_len, int *gen_type, int *offset); extern int arch_check_bp_in_kernelspace(struct perf_event *bp); extern int arch_validate_hwbkpt_settings(struct perf_event *bp); extern int hw_breakpoint_exceptions_notify(struct notifier_block *unused, diff --git a/arch/arm64/include/asm/kernel-pgtable.h b/arch/arm64/include/asm/kernel-pgtable.h index 7e51d1b..7803343 100644 --- a/arch/arm64/include/asm/kernel-pgtable.h +++ b/arch/arm64/include/asm/kernel-pgtable.h @@ -19,6 +19,7 @@ #ifndef __ASM_KERNEL_PGTABLE_H #define __ASM_KERNEL_PGTABLE_H +#include <asm/pgtable.h> #include <asm/sparsemem.h> /* @@ -54,6 +55,12 @@ #define SWAPPER_DIR_SIZE (SWAPPER_PGTABLE_LEVELS * PAGE_SIZE) #define IDMAP_DIR_SIZE (IDMAP_PGTABLE_LEVELS * PAGE_SIZE) +#ifdef CONFIG_ARM64_SW_TTBR0_PAN +#define RESERVED_TTBR0_SIZE (PAGE_SIZE) +#else +#define RESERVED_TTBR0_SIZE (0) +#endif + /* Initial memory map size */ #if ARM64_SWAPPER_USES_SECTION_MAPS #define SWAPPER_BLOCK_SHIFT SECTION_SHIFT diff --git a/arch/arm64/include/asm/mmu.h b/arch/arm64/include/asm/mmu.h index 8d9fce0..4761941 100644 --- a/arch/arm64/include/asm/mmu.h +++ b/arch/arm64/include/asm/mmu.h @@ -19,6 +19,7 @@ typedef struct { atomic64_t id; void *vdso; + unsigned long flags; } mm_context_t; /* @@ -34,7 +35,7 @@ extern void __iomem *early_io_map(phys_addr_t phys, unsigned long virt); extern void init_mem_pgprot(void); extern void create_pgd_mapping(struct mm_struct *mm, phys_addr_t phys, unsigned long virt, phys_addr_t size, - pgprot_t prot, bool allow_block_mappings); + pgprot_t prot, bool page_mappings_only); extern void *fixmap_remap_fdt(phys_addr_t dt_phys); #endif diff --git a/arch/arm64/include/asm/mmu_context.h b/arch/arm64/include/asm/mmu_context.h index a501853..0363fe8 100644 --- a/arch/arm64/include/asm/mmu_context.h +++ b/arch/arm64/include/asm/mmu_context.h @@ -23,6 +23,7 @@ #include <linux/sched.h> #include <asm/cacheflush.h> +#include <asm/cpufeature.h> #include <asm/proc-fns.h> #include <asm-generic/mm_hooks.h> #include <asm/cputype.h> @@ -103,7 +104,7 @@ static inline void cpu_uninstall_idmap(void) local_flush_tlb_all(); cpu_set_default_tcr_t0sz(); - if (mm != &init_mm) + if (mm != &init_mm && !system_uses_ttbr0_pan()) cpu_switch_mm(mm->pgd, mm); } @@ -163,20 +164,26 @@ enter_lazy_tlb(struct mm_struct *mm, struct task_struct *tsk) { } -/* - * This is the actual mm switch as far as the scheduler - * is concerned. No registers are touched. We avoid - * calling the CPU specific function when the mm hasn't - * actually changed. - */ -static inline void -switch_mm(struct mm_struct *prev, struct mm_struct *next, - struct task_struct *tsk) +#ifdef CONFIG_ARM64_SW_TTBR0_PAN +static inline void update_saved_ttbr0(struct task_struct *tsk, + struct mm_struct *mm) { - unsigned int cpu = smp_processor_id(); + if (system_uses_ttbr0_pan()) { + BUG_ON(mm->pgd == swapper_pg_dir); + task_thread_info(tsk)->ttbr0 = + virt_to_phys(mm->pgd) | ASID(mm) << 48; + } +} +#else +static inline void update_saved_ttbr0(struct task_struct *tsk, + struct mm_struct *mm) +{ +} +#endif - if (prev == next) - return; +static inline void __switch_mm(struct mm_struct *next) +{ + unsigned int cpu = smp_processor_id(); /* * init_mm.pgd does not contain any user mappings and it is always @@ -190,8 +197,26 @@ switch_mm(struct mm_struct *prev, struct mm_struct *next, check_and_switch_context(next, cpu); } +static inline void +switch_mm(struct mm_struct *prev, struct mm_struct *next, + struct task_struct *tsk) +{ + if (prev != next) + __switch_mm(next); + + /* + * Update the saved TTBR0_EL1 of the scheduled-in task as the previous + * value may have not been initialised yet (activate_mm caller) or the + * ASID has changed since the last run (following the context switch + * of another thread of the same process). Avoid setting the reserved + * TTBR0_EL1 to swapper_pg_dir (init_mm; e.g. via idle_task_exit). + */ + if (next != &init_mm) + update_saved_ttbr0(tsk, next); +} + #define deactivate_mm(tsk,mm) do { } while (0) -#define activate_mm(prev,next) switch_mm(prev, next, NULL) +#define activate_mm(prev,next) switch_mm(prev, next, current) void verify_cpu_asid_bits(void); diff --git a/arch/arm64/include/asm/neon.h b/arch/arm64/include/asm/neon.h index 13ce4cc..ad4cdc9 100644 --- a/arch/arm64/include/asm/neon.h +++ b/arch/arm64/include/asm/neon.h @@ -9,8 +9,9 @@ */ #include <linux/types.h> +#include <asm/fpsimd.h> -#define cpu_has_neon() (1) +#define cpu_has_neon() system_supports_fpsimd() #define kernel_neon_begin() kernel_neon_begin_partial(32) diff --git a/arch/arm64/include/asm/opcodes.h b/arch/arm64/include/asm/opcodes.h deleted file mode 100644 index 123f45d..0000000 --- a/arch/arm64/include/asm/opcodes.h +++ /dev/null @@ -1,5 +0,0 @@ -#ifdef CONFIG_CPU_BIG_ENDIAN -#define CONFIG_CPU_ENDIAN_BE8 CONFIG_CPU_BIG_ENDIAN -#endif - -#include <../../arm/include/asm/opcodes.h> diff --git a/arch/arm64/include/asm/percpu.h b/arch/arm64/include/asm/percpu.h index 5394c84..3bd498e 100644 --- a/arch/arm64/include/asm/percpu.h +++ b/arch/arm64/include/asm/percpu.h @@ -16,6 +16,8 @@ #ifndef __ASM_PERCPU_H #define __ASM_PERCPU_H +#include <asm/stack_pointer.h> + static inline void set_my_cpu_offset(unsigned long off) { asm volatile("msr tpidr_el1, %0" :: "r" (off) : "memory"); @@ -101,16 +103,16 @@ static inline unsigned long __percpu_read(void *ptr, int size) switch (size) { case 1: - ret = ACCESS_ONCE(*(u8 *)ptr); + ret = READ_ONCE(*(u8 *)ptr); break; case 2: - ret = ACCESS_ONCE(*(u16 *)ptr); + ret = READ_ONCE(*(u16 *)ptr); break; case 4: - ret = ACCESS_ONCE(*(u32 *)ptr); + ret = READ_ONCE(*(u32 *)ptr); break; case 8: - ret = ACCESS_ONCE(*(u64 *)ptr); + ret = READ_ONCE(*(u64 *)ptr); break; default: BUILD_BUG(); @@ -123,16 +125,16 @@ static inline void __percpu_write(void *ptr, unsigned long val, int size) { switch (size) { case 1: - ACCESS_ONCE(*(u8 *)ptr) = (u8)val; + WRITE_ONCE(*(u8 *)ptr, (u8)val); break; case 2: - ACCESS_ONCE(*(u16 *)ptr) = (u16)val; + WRITE_ONCE(*(u16 *)ptr, (u16)val); break; case 4: - ACCESS_ONCE(*(u32 *)ptr) = (u32)val; + WRITE_ONCE(*(u32 *)ptr, (u32)val); break; case 8: - ACCESS_ONCE(*(u64 *)ptr) = (u64)val; + WRITE_ONCE(*(u64 *)ptr, (u64)val); break; default: BUILD_BUG(); diff --git a/arch/arm64/include/asm/perf_event.h b/arch/arm64/include/asm/perf_event.h index 38b6a2b..8d5cbec 100644 --- a/arch/arm64/include/asm/perf_event.h +++ b/arch/arm64/include/asm/perf_event.h @@ -17,6 +17,8 @@ #ifndef __ASM_PERF_EVENT_H #define __ASM_PERF_EVENT_H +#include <asm/stack_pointer.h> + #define ARMV8_PMU_MAX_COUNTERS 32 #define ARMV8_PMU_COUNTER_MASK (ARMV8_PMU_MAX_COUNTERS - 1) diff --git a/arch/arm64/include/asm/probes.h b/arch/arm64/include/asm/probes.h index 5af574d..6a5b289 100644 --- a/arch/arm64/include/asm/probes.h +++ b/arch/arm64/include/asm/probes.h @@ -15,21 +15,22 @@ #ifndef _ARM_PROBES_H #define _ARM_PROBES_H -#include <asm/opcodes.h> - -struct kprobe; -struct arch_specific_insn; - -typedef u32 kprobe_opcode_t; -typedef void (kprobes_handler_t) (u32 opcode, long addr, struct pt_regs *); +typedef u32 probe_opcode_t; +typedef void (probes_handler_t) (u32 opcode, long addr, struct pt_regs *); /* architecture specific copy of original instruction */ -struct arch_specific_insn { - kprobe_opcode_t *insn; +struct arch_probe_insn { + probe_opcode_t *insn; pstate_check_t *pstate_cc; - kprobes_handler_t *handler; + probes_handler_t *handler; /* restore address after step xol */ unsigned long restore; }; +#ifdef CONFIG_KPROBES +typedef u32 kprobe_opcode_t; +struct arch_specific_insn { + struct arch_probe_insn api; +}; +#endif #endif diff --git a/arch/arm64/include/asm/ptdump.h b/arch/arm64/include/asm/ptdump.h index 07b8ed0..6afd847 100644 --- a/arch/arm64/include/asm/ptdump.h +++ b/arch/arm64/include/asm/ptdump.h @@ -16,9 +16,10 @@ #ifndef __ASM_PTDUMP_H #define __ASM_PTDUMP_H -#ifdef CONFIG_ARM64_PTDUMP +#ifdef CONFIG_ARM64_PTDUMP_CORE #include <linux/mm_types.h> +#include <linux/seq_file.h> struct addr_marker { unsigned long start_address; @@ -29,16 +30,25 @@ struct ptdump_info { struct mm_struct *mm; const struct addr_marker *markers; unsigned long base_addr; - unsigned long max_addr; }; -int ptdump_register(struct ptdump_info *info, const char *name); - +void ptdump_walk_pgd(struct seq_file *s, struct ptdump_info *info); +#ifdef CONFIG_ARM64_PTDUMP_DEBUGFS +int ptdump_debugfs_register(struct ptdump_info *info, const char *name); #else -static inline int ptdump_register(struct ptdump_info *info, const char *name) +static inline int ptdump_debugfs_register(struct ptdump_info *info, + const char *name) { return 0; } -#endif /* CONFIG_ARM64_PTDUMP */ +#endif +void ptdump_check_wx(void); +#endif /* CONFIG_ARM64_PTDUMP_CORE */ + +#ifdef CONFIG_DEBUG_WX +#define debug_checkwx() ptdump_check_wx() +#else +#define debug_checkwx() do { } while (0) +#endif #endif /* __ASM_PTDUMP_H */ diff --git a/arch/arm64/include/asm/ptrace.h b/arch/arm64/include/asm/ptrace.h index ada08b5..513daf0 100644 --- a/arch/arm64/include/asm/ptrace.h +++ b/arch/arm64/include/asm/ptrace.h @@ -217,6 +217,14 @@ int valid_user_regs(struct user_pt_regs *regs, struct task_struct *task); #include <asm-generic/ptrace.h> +#define procedure_link_pointer(regs) ((regs)->regs[30]) + +static inline void procedure_link_pointer_set(struct pt_regs *regs, + unsigned long val) +{ + procedure_link_pointer(regs) = val; +} + #undef profile_pc extern unsigned long profile_pc(struct pt_regs *regs); diff --git a/arch/arm64/include/asm/smp.h b/arch/arm64/include/asm/smp.h index 0226447..d050d72 100644 --- a/arch/arm64/include/asm/smp.h +++ b/arch/arm64/include/asm/smp.h @@ -29,11 +29,22 @@ #ifndef __ASSEMBLY__ +#include <asm/percpu.h> + #include <linux/threads.h> #include <linux/cpumask.h> #include <linux/thread_info.h> -#define raw_smp_processor_id() (current_thread_info()->cpu) +DECLARE_PER_CPU_READ_MOSTLY(int, cpu_number); + +/* + * We don't use this_cpu_read(cpu_number) as that has implicit writes to + * preempt_count, and associated (compiler) barriers, that we'd like to avoid + * the expense of. If we're preemptible, the value can be stale at use anyway. + * And we can't use this_cpu_ptr() either, as that winds up recursing back + * here under CONFIG_DEBUG_PREEMPT=y. + */ +#define raw_smp_processor_id() (*raw_cpu_ptr(&cpu_number)) struct seq_file; @@ -73,6 +84,7 @@ asmlinkage void secondary_start_kernel(void); */ struct secondary_data { void *stack; + struct task_struct *task; long status; }; diff --git a/arch/arm64/include/asm/stack_pointer.h b/arch/arm64/include/asm/stack_pointer.h new file mode 100644 index 0000000..ffcdf74 --- /dev/null +++ b/arch/arm64/include/asm/stack_pointer.h @@ -0,0 +1,9 @@ +#ifndef __ASM_STACK_POINTER_H +#define __ASM_STACK_POINTER_H + +/* + * how to get the current stack pointer from C + */ +register unsigned long current_stack_pointer asm ("sp"); + +#endif /* __ASM_STACK_POINTER_H */ diff --git a/arch/arm64/include/asm/suspend.h b/arch/arm64/include/asm/suspend.h index b8a313f..de5600f 100644 --- a/arch/arm64/include/asm/suspend.h +++ b/arch/arm64/include/asm/suspend.h @@ -1,7 +1,7 @@ #ifndef __ASM_SUSPEND_H #define __ASM_SUSPEND_H -#define NR_CTX_REGS 10 +#define NR_CTX_REGS 12 #define NR_CALLEE_SAVED_REGS 12 /* diff --git a/arch/arm64/include/asm/sysreg.h b/arch/arm64/include/asm/sysreg.h index 6c80b36..98ae03f 100644 --- a/arch/arm64/include/asm/sysreg.h +++ b/arch/arm64/include/asm/sysreg.h @@ -22,8 +22,6 @@ #include <linux/stringify.h> -#include <asm/opcodes.h> - /* * ARMv8 ARM reserves the following encoding for system registers: * (Ref: ARMv8 ARM, Section: "System instruction class encoding overview", @@ -37,6 +35,33 @@ #define sys_reg(op0, op1, crn, crm, op2) \ ((((op0)&3)<<19)|((op1)<<16)|((crn)<<12)|((crm)<<8)|((op2)<<5)) +#ifndef CONFIG_BROKEN_GAS_INST + +#ifdef __ASSEMBLY__ +#define __emit_inst(x) .inst (x) +#else +#define __emit_inst(x) ".inst " __stringify((x)) "\n\t" +#endif + +#else /* CONFIG_BROKEN_GAS_INST */ + +#ifndef CONFIG_CPU_BIG_ENDIAN +#define __INSTR_BSWAP(x) (x) +#else /* CONFIG_CPU_BIG_ENDIAN */ +#define __INSTR_BSWAP(x) ((((x) << 24) & 0xff000000) | \ + (((x) << 8) & 0x00ff0000) | \ + (((x) >> 8) & 0x0000ff00) | \ + (((x) >> 24) & 0x000000ff)) +#endif /* CONFIG_CPU_BIG_ENDIAN */ + +#ifdef __ASSEMBLY__ +#define __emit_inst(x) .long __INSTR_BSWAP(x) +#else /* __ASSEMBLY__ */ +#define __emit_inst(x) ".long " __stringify(__INSTR_BSWAP(x)) "\n\t" +#endif /* __ASSEMBLY__ */ + +#endif /* CONFIG_BROKEN_GAS_INST */ + #define SYS_MIDR_EL1 sys_reg(3, 0, 0, 0, 0) #define SYS_MPIDR_EL1 sys_reg(3, 0, 0, 0, 5) #define SYS_REVIDR_EL1 sys_reg(3, 0, 0, 0, 6) @@ -81,10 +106,10 @@ #define REG_PSTATE_PAN_IMM sys_reg(0, 0, 4, 0, 4) #define REG_PSTATE_UAO_IMM sys_reg(0, 0, 4, 0, 3) -#define SET_PSTATE_PAN(x) __inst_arm(0xd5000000 | REG_PSTATE_PAN_IMM |\ - (!!x)<<8 | 0x1f) -#define SET_PSTATE_UAO(x) __inst_arm(0xd5000000 | REG_PSTATE_UAO_IMM |\ - (!!x)<<8 | 0x1f) +#define SET_PSTATE_PAN(x) __emit_inst(0xd5000000 | REG_PSTATE_PAN_IMM | \ + (!!x)<<8 | 0x1f) +#define SET_PSTATE_UAO(x) __emit_inst(0xd5000000 | REG_PSTATE_UAO_IMM | \ + (!!x)<<8 | 0x1f) /* Common SCTLR_ELx flags. */ #define SCTLR_ELx_EE (1 << 25) @@ -228,11 +253,11 @@ .equ .L__reg_num_xzr, 31 .macro mrs_s, rt, sreg - .inst 0xd5200000|(\sreg)|(.L__reg_num_\rt) + __emit_inst(0xd5200000|(\sreg)|(.L__reg_num_\rt)) .endm .macro msr_s, sreg, rt - .inst 0xd5000000|(\sreg)|(.L__reg_num_\rt) + __emit_inst(0xd5000000|(\sreg)|(.L__reg_num_\rt)) .endm #else @@ -246,11 +271,11 @@ asm( " .equ .L__reg_num_xzr, 31\n" "\n" " .macro mrs_s, rt, sreg\n" -" .inst 0xd5200000|(\\sreg)|(.L__reg_num_\\rt)\n" + __emit_inst(0xd5200000|(\\sreg)|(.L__reg_num_\\rt)) " .endm\n" "\n" " .macro msr_s, sreg, rt\n" -" .inst 0xd5000000|(\\sreg)|(.L__reg_num_\\rt)\n" + __emit_inst(0xd5000000|(\\sreg)|(.L__reg_num_\\rt)) " .endm\n" ); diff --git a/arch/arm64/include/asm/thread_info.h b/arch/arm64/include/asm/thread_info.h index e9ea5a6..46c3b93 100644 --- a/arch/arm64/include/asm/thread_info.h +++ b/arch/arm64/include/asm/thread_info.h @@ -36,58 +36,31 @@ struct task_struct; +#include <asm/stack_pointer.h> #include <asm/types.h> typedef unsigned long mm_segment_t; /* * low level task data that entry.S needs immediate access to. - * __switch_to() assumes cpu_context follows immediately after cpu_domain. */ struct thread_info { unsigned long flags; /* low level flags */ mm_segment_t addr_limit; /* address limit */ - struct task_struct *task; /* main task structure */ +#ifdef CONFIG_ARM64_SW_TTBR0_PAN + u64 ttbr0; /* saved TTBR0_EL1 */ +#endif int preempt_count; /* 0 => preemptable, <0 => bug */ - int cpu; /* cpu */ }; #define INIT_THREAD_INFO(tsk) \ { \ - .task = &tsk, \ - .flags = 0, \ .preempt_count = INIT_PREEMPT_COUNT, \ .addr_limit = KERNEL_DS, \ } -#define init_thread_info (init_thread_union.thread_info) #define init_stack (init_thread_union.stack) -/* - * how to get the current stack pointer from C - */ -register unsigned long current_stack_pointer asm ("sp"); - -/* - * how to get the thread information struct from C - */ -static inline struct thread_info *current_thread_info(void) __attribute_const__; - -/* - * struct thread_info can be accessed directly via sp_el0. - * - * We don't use read_sysreg() as we want the compiler to cache the value where - * possible. - */ -static inline struct thread_info *current_thread_info(void) -{ - unsigned long sp_el0; - - asm ("mrs %0, sp_el0" : "=r" (sp_el0)); - - return (struct thread_info *)sp_el0; -} - #define thread_saved_pc(tsk) \ ((unsigned long)(tsk->thread.cpu_context.pc)) #define thread_saved_sp(tsk) \ @@ -112,6 +85,7 @@ static inline struct thread_info *current_thread_info(void) #define TIF_NEED_RESCHED 1 #define TIF_NOTIFY_RESUME 2 /* callback before returning to user */ #define TIF_FOREIGN_FPSTATE 3 /* CPU's FP state is not current's */ +#define TIF_UPROBE 4 /* uprobe breakpoint or singlestep */ #define TIF_NOHZ 7 #define TIF_SYSCALL_TRACE 8 #define TIF_SYSCALL_AUDIT 9 @@ -132,10 +106,12 @@ static inline struct thread_info *current_thread_info(void) #define _TIF_SYSCALL_AUDIT (1 << TIF_SYSCALL_AUDIT) #define _TIF_SYSCALL_TRACEPOINT (1 << TIF_SYSCALL_TRACEPOINT) #define _TIF_SECCOMP (1 << TIF_SECCOMP) +#define _TIF_UPROBE (1 << TIF_UPROBE) #define _TIF_32BIT (1 << TIF_32BIT) #define _TIF_WORK_MASK (_TIF_NEED_RESCHED | _TIF_SIGPENDING | \ - _TIF_NOTIFY_RESUME | _TIF_FOREIGN_FPSTATE) + _TIF_NOTIFY_RESUME | _TIF_FOREIGN_FPSTATE | \ + _TIF_UPROBE) #define _TIF_SYSCALL_WORK (_TIF_SYSCALL_TRACE | _TIF_SYSCALL_AUDIT | \ _TIF_SYSCALL_TRACEPOINT | _TIF_SECCOMP | \ diff --git a/arch/arm64/include/asm/uaccess.h b/arch/arm64/include/asm/uaccess.h index 55d0adb..d26750c 100644 --- a/arch/arm64/include/asm/uaccess.h +++ b/arch/arm64/include/asm/uaccess.h @@ -18,6 +18,12 @@ #ifndef __ASM_UACCESS_H #define __ASM_UACCESS_H +#include <asm/alternative.h> +#include <asm/kernel-pgtable.h> +#include <asm/sysreg.h> + +#ifndef __ASSEMBLY__ + /* * User space memory access functions */ @@ -26,10 +32,8 @@ #include <linux/string.h> #include <linux/thread_info.h> -#include <asm/alternative.h> #include <asm/cpufeature.h> #include <asm/ptrace.h> -#include <asm/sysreg.h> #include <asm/errno.h> #include <asm/memory.h> #include <asm/compiler.h> @@ -120,6 +124,99 @@ static inline void set_fs(mm_segment_t fs) " .popsection\n" /* + * User access enabling/disabling. + */ +#ifdef CONFIG_ARM64_SW_TTBR0_PAN +static inline void __uaccess_ttbr0_disable(void) +{ + unsigned long ttbr; + + /* reserved_ttbr0 placed at the end of swapper_pg_dir */ + ttbr = read_sysreg(ttbr1_el1) + SWAPPER_DIR_SIZE; + write_sysreg(ttbr, ttbr0_el1); + isb(); +} + +static inline void __uaccess_ttbr0_enable(void) +{ + unsigned long flags; + + /* + * Disable interrupts to avoid preemption between reading the 'ttbr0' + * variable and the MSR. A context switch could trigger an ASID + * roll-over and an update of 'ttbr0'. + */ + local_irq_save(flags); + write_sysreg(current_thread_info()->ttbr0, ttbr0_el1); + isb(); + local_irq_restore(flags); +} + +static inline bool uaccess_ttbr0_disable(void) +{ + if (!system_uses_ttbr0_pan()) + return false; + __uaccess_ttbr0_disable(); + return true; +} + +static inline bool uaccess_ttbr0_enable(void) +{ + if (!system_uses_ttbr0_pan()) + return false; + __uaccess_ttbr0_enable(); + return true; +} +#else +static inline bool uaccess_ttbr0_disable(void) +{ + return false; +} + +static inline bool uaccess_ttbr0_enable(void) +{ + return false; +} +#endif + +#define __uaccess_disable(alt) \ +do { \ + if (!uaccess_ttbr0_disable()) \ + asm(ALTERNATIVE("nop", SET_PSTATE_PAN(1), alt, \ + CONFIG_ARM64_PAN)); \ +} while (0) + +#define __uaccess_enable(alt) \ +do { \ + if (!uaccess_ttbr0_enable()) \ + asm(ALTERNATIVE("nop", SET_PSTATE_PAN(0), alt, \ + CONFIG_ARM64_PAN)); \ +} while (0) + +static inline void uaccess_disable(void) +{ + __uaccess_disable(ARM64_HAS_PAN); +} + +static inline void uaccess_enable(void) +{ + __uaccess_enable(ARM64_HAS_PAN); +} + +/* + * These functions are no-ops when UAO is present. + */ +static inline void uaccess_disable_not_uao(void) +{ + __uaccess_disable(ARM64_ALT_PAN_NOT_UAO); +} + +static inline void uaccess_enable_not_uao(void) +{ + __uaccess_enable(ARM64_ALT_PAN_NOT_UAO); +} + +/* * The "__xxx" versions of the user access functions do not verify the address * space - it must have been done previously with a separate "access_ok()" * call. @@ -146,8 +243,7 @@ static inline void set_fs(mm_segment_t fs) do { \ unsigned long __gu_val; \ __chk_user_ptr(ptr); \ - asm(ALTERNATIVE("nop", SET_PSTATE_PAN(0), ARM64_ALT_PAN_NOT_UAO,\ - CONFIG_ARM64_PAN)); \ + uaccess_enable_not_uao(); \ switch (sizeof(*(ptr))) { \ case 1: \ __get_user_asm("ldrb", "ldtrb", "%w", __gu_val, (ptr), \ @@ -168,9 +264,8 @@ do { \ default: \ BUILD_BUG(); \ } \ + uaccess_disable_not_uao(); \ (x) = (__force __typeof__(*(ptr)))__gu_val; \ - asm(ALTERNATIVE("nop", SET_PSTATE_PAN(1), ARM64_ALT_PAN_NOT_UAO,\ - CONFIG_ARM64_PAN)); \ } while (0) #define __get_user(x, ptr) \ @@ -215,8 +310,7 @@ do { \ do { \ __typeof__(*(ptr)) __pu_val = (x); \ __chk_user_ptr(ptr); \ - asm(ALTERNATIVE("nop", SET_PSTATE_PAN(0), ARM64_ALT_PAN_NOT_UAO,\ - CONFIG_ARM64_PAN)); \ + uaccess_enable_not_uao(); \ switch (sizeof(*(ptr))) { \ case 1: \ __put_user_asm("strb", "sttrb", "%w", __pu_val, (ptr), \ @@ -237,8 +331,7 @@ do { \ default: \ BUILD_BUG(); \ } \ - asm(ALTERNATIVE("nop", SET_PSTATE_PAN(1), ARM64_ALT_PAN_NOT_UAO,\ - CONFIG_ARM64_PAN)); \ + uaccess_disable_not_uao(); \ } while (0) #define __put_user(x, ptr) \ @@ -331,4 +424,66 @@ extern long strncpy_from_user(char *dest, const char __user *src, long count); extern __must_check long strlen_user(const char __user *str); extern __must_check long strnlen_user(const char __user *str, long n); +#else /* __ASSEMBLY__ */ + +#include <asm/assembler.h> + +/* + * User access enabling/disabling macros. + */ +#ifdef CONFIG_ARM64_SW_TTBR0_PAN + .macro __uaccess_ttbr0_disable, tmp1 + mrs \tmp1, ttbr1_el1 // swapper_pg_dir + add \tmp1, \tmp1, #SWAPPER_DIR_SIZE // reserved_ttbr0 at the end of swapper_pg_dir + msr ttbr0_el1, \tmp1 // set reserved TTBR0_EL1 + isb + .endm + + .macro __uaccess_ttbr0_enable, tmp1 + get_thread_info \tmp1 + ldr \tmp1, [\tmp1, #TSK_TI_TTBR0] // load saved TTBR0_EL1 + msr ttbr0_el1, \tmp1 // set the non-PAN TTBR0_EL1 + isb + .endm + + .macro uaccess_ttbr0_disable, tmp1 +alternative_if_not ARM64_HAS_PAN + __uaccess_ttbr0_disable \tmp1 +alternative_else_nop_endif + .endm + + .macro uaccess_ttbr0_enable, tmp1, tmp2 +alternative_if_not ARM64_HAS_PAN + save_and_disable_irq \tmp2 // avoid preemption + __uaccess_ttbr0_enable \tmp1 + restore_irq \tmp2 +alternative_else_nop_endif + .endm +#else + .macro uaccess_ttbr0_disable, tmp1 + .endm + + .macro uaccess_ttbr0_enable, tmp1, tmp2 + .endm +#endif + +/* + * These macros are no-ops when UAO is present. + */ + .macro uaccess_disable_not_uao, tmp1 + uaccess_ttbr0_disable \tmp1 +alternative_if ARM64_ALT_PAN_NOT_UAO + SET_PSTATE_PAN(1) +alternative_else_nop_endif + .endm + + .macro uaccess_enable_not_uao, tmp1, tmp2 + uaccess_ttbr0_enable \tmp1, \tmp2 +alternative_if ARM64_ALT_PAN_NOT_UAO + SET_PSTATE_PAN(0) +alternative_else_nop_endif + .endm + +#endif /* __ASSEMBLY__ */ + #endif /* __ASM_UACCESS_H */ diff --git a/arch/arm64/include/asm/uprobes.h b/arch/arm64/include/asm/uprobes.h new file mode 100644 index 0000000..8d00407 --- /dev/null +++ b/arch/arm64/include/asm/uprobes.h @@ -0,0 +1,36 @@ +/* + * Copyright (C) 2014-2016 Pratyush Anand <panand@redhat.com> + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +#ifndef _ASM_UPROBES_H +#define _ASM_UPROBES_H + +#include <asm/debug-monitors.h> +#include <asm/insn.h> +#include <asm/probes.h> + +#define MAX_UINSN_BYTES AARCH64_INSN_SIZE + +#define UPROBE_SWBP_INSN BRK64_OPCODE_UPROBES +#define UPROBE_SWBP_INSN_SIZE AARCH64_INSN_SIZE +#define UPROBE_XOL_SLOT_BYTES MAX_UINSN_BYTES + +typedef u32 uprobe_opcode_t; + +struct arch_uprobe_task { +}; + +struct arch_uprobe { + union { + u8 insn[MAX_UINSN_BYTES]; + u8 ixol[MAX_UINSN_BYTES]; + }; + struct arch_probe_insn api; + bool simulate; +}; + +#endif diff --git a/arch/arm64/include/asm/xen/hypercall.h b/arch/arm64/include/asm/xen/hypercall.h index 74b0c42..3522cba 100644 --- a/arch/arm64/include/asm/xen/hypercall.h +++ b/arch/arm64/include/asm/xen/hypercall.h @@ -1 +1 @@ -#include <../../arm/include/asm/xen/hypercall.h> +#include <xen/arm/hypercall.h> diff --git a/arch/arm64/include/asm/xen/hypervisor.h b/arch/arm64/include/asm/xen/hypervisor.h index f263da8..d6e7709 100644 --- a/arch/arm64/include/asm/xen/hypervisor.h +++ b/arch/arm64/include/asm/xen/hypervisor.h @@ -1 +1 @@ -#include <../../arm/include/asm/xen/hypervisor.h> +#include <xen/arm/hypervisor.h> diff --git a/arch/arm64/include/asm/xen/interface.h b/arch/arm64/include/asm/xen/interface.h index 44457ae..88c0d75 100644 --- a/arch/arm64/include/asm/xen/interface.h +++ b/arch/arm64/include/asm/xen/interface.h @@ -1 +1 @@ -#include <../../arm/include/asm/xen/interface.h> +#include <xen/arm/interface.h> diff --git a/arch/arm64/include/asm/xen/page-coherent.h b/arch/arm64/include/asm/xen/page-coherent.h index 2052102..b3ef061 100644 --- a/arch/arm64/include/asm/xen/page-coherent.h +++ b/arch/arm64/include/asm/xen/page-coherent.h @@ -1 +1 @@ -#include <../../arm/include/asm/xen/page-coherent.h> +#include <xen/arm/page-coherent.h> diff --git a/arch/arm64/include/asm/xen/page.h b/arch/arm64/include/asm/xen/page.h index bed87ec..31bbc80 100644 --- a/arch/arm64/include/asm/xen/page.h +++ b/arch/arm64/include/asm/xen/page.h @@ -1 +1 @@ -#include <../../arm/include/asm/xen/page.h> +#include <xen/arm/page.h> diff --git a/arch/arm64/kernel/armv8_deprecated.c b/arch/arm64/kernel/armv8_deprecated.c index b0988bb..04de188 100644 --- a/arch/arm64/kernel/armv8_deprecated.c +++ b/arch/arm64/kernel/armv8_deprecated.c @@ -14,10 +14,8 @@ #include <linux/slab.h> #include <linux/sysctl.h> -#include <asm/alternative.h> #include <asm/cpufeature.h> #include <asm/insn.h> -#include <asm/opcodes.h> #include <asm/sysreg.h> #include <asm/system_misc.h> #include <asm/traps.h> @@ -285,10 +283,10 @@ static void __init register_insn_emulation_sysctl(struct ctl_table *table) #define __SWP_LL_SC_LOOPS 4 #define __user_swpX_asm(data, addr, res, temp, temp2, B) \ +do { \ + uaccess_enable(); \ __asm__ __volatile__( \ " mov %w3, %w7\n" \ - ALTERNATIVE("nop", SET_PSTATE_PAN(0), ARM64_HAS_PAN, \ - CONFIG_ARM64_PAN) \ "0: ldxr"B" %w2, [%4]\n" \ "1: stxr"B" %w0, %w1, [%4]\n" \ " cbz %w0, 2f\n" \ @@ -306,12 +304,12 @@ static void __init register_insn_emulation_sysctl(struct ctl_table *table) " .popsection" \ _ASM_EXTABLE(0b, 4b) \ _ASM_EXTABLE(1b, 4b) \ - ALTERNATIVE("nop", SET_PSTATE_PAN(1), ARM64_HAS_PAN, \ - CONFIG_ARM64_PAN) \ : "=&r" (res), "+r" (data), "=&r" (temp), "=&r" (temp2) \ : "r" (addr), "i" (-EAGAIN), "i" (-EFAULT), \ "i" (__SWP_LL_SC_LOOPS) \ - : "memory") + : "memory"); \ + uaccess_disable(); \ +} while (0) #define __user_swp_asm(data, addr, res, temp, temp2) \ __user_swpX_asm(data, addr, res, temp, temp2, "") @@ -352,6 +350,10 @@ static int emulate_swpX(unsigned int address, unsigned int *data, return res; } +#define ARM_OPCODE_CONDTEST_FAIL 0 +#define ARM_OPCODE_CONDTEST_PASS 1 +#define ARM_OPCODE_CONDTEST_UNCOND 2 + #define ARM_OPCODE_CONDITION_UNCOND 0xf static unsigned int __kprobes aarch32_check_condition(u32 opcode, u32 psr) diff --git a/arch/arm64/kernel/asm-offsets.c b/arch/arm64/kernel/asm-offsets.c index 4a2f0f0..bc049af 100644 --- a/arch/arm64/kernel/asm-offsets.c +++ b/arch/arm64/kernel/asm-offsets.c @@ -36,11 +36,13 @@ int main(void) { DEFINE(TSK_ACTIVE_MM, offsetof(struct task_struct, active_mm)); BLANK(); - DEFINE(TI_FLAGS, offsetof(struct thread_info, flags)); - DEFINE(TI_PREEMPT, offsetof(struct thread_info, preempt_count)); - DEFINE(TI_ADDR_LIMIT, offsetof(struct thread_info, addr_limit)); - DEFINE(TI_TASK, offsetof(struct thread_info, task)); - DEFINE(TI_CPU, offsetof(struct thread_info, cpu)); + DEFINE(TSK_TI_FLAGS, offsetof(struct task_struct, thread_info.flags)); + DEFINE(TSK_TI_PREEMPT, offsetof(struct task_struct, thread_info.preempt_count)); + DEFINE(TSK_TI_ADDR_LIMIT, offsetof(struct task_struct, thread_info.addr_limit)); +#ifdef CONFIG_ARM64_SW_TTBR0_PAN + DEFINE(TSK_TI_TTBR0, offsetof(struct task_struct, thread_info.ttbr0)); +#endif + DEFINE(TSK_STACK, offsetof(struct task_struct, stack)); BLANK(); DEFINE(THREAD_CPU_CONTEXT, offsetof(struct task_struct, thread.cpu_context)); BLANK(); @@ -123,6 +125,7 @@ int main(void) DEFINE(TZ_DSTTIME, offsetof(struct timezone, tz_dsttime)); BLANK(); DEFINE(CPU_BOOT_STACK, offsetof(struct secondary_data, stack)); + DEFINE(CPU_BOOT_TASK, offsetof(struct secondary_data, task)); BLANK(); #ifdef CONFIG_KVM_ARM_HOST DEFINE(VCPU_CONTEXT, offsetof(struct kvm_vcpu, arch.ctxt)); diff --git a/arch/arm64/kernel/cpufeature.c b/arch/arm64/kernel/cpufeature.c index c02504e..fdf8f04 100644 --- a/arch/arm64/kernel/cpufeature.c +++ b/arch/arm64/kernel/cpufeature.c @@ -47,6 +47,7 @@ unsigned int compat_elf_hwcap2 __read_mostly; #endif DECLARE_BITMAP(cpu_hwcaps, ARM64_NCAPS); +EXPORT_SYMBOL(cpu_hwcaps); DEFINE_STATIC_KEY_ARRAY_FALSE(cpu_hwcap_keys, ARM64_NCAPS); EXPORT_SYMBOL(cpu_hwcap_keys); @@ -746,6 +747,14 @@ static bool hyp_offset_low(const struct arm64_cpu_capabilities *entry, return idmap_addr > GENMASK(VA_BITS - 2, 0) && !is_kernel_in_hyp_mode(); } +static bool has_no_fpsimd(const struct arm64_cpu_capabilities *entry, int __unused) +{ + u64 pfr0 = read_system_reg(SYS_ID_AA64PFR0_EL1); + + return cpuid_feature_extract_signed_field(pfr0, + ID_AA64PFR0_FP_SHIFT) < 0; +} + static const struct arm64_cpu_capabilities arm64_features[] = { { .desc = "GIC system register CPU interface", @@ -829,6 +838,13 @@ static const struct arm64_cpu_capabilities arm64_features[] = { .def_scope = SCOPE_SYSTEM, .matches = hyp_offset_low, }, + { + /* FP/SIMD is not implemented */ + .capability = ARM64_HAS_NO_FPSIMD, + .def_scope = SCOPE_SYSTEM, + .min_field_value = 0, + .matches = has_no_fpsimd, + }, {}, }; @@ -1102,5 +1118,5 @@ void __init setup_cpu_features(void) static bool __maybe_unused cpufeature_pan_not_uao(const struct arm64_cpu_capabilities *entry, int __unused) { - return (cpus_have_cap(ARM64_HAS_PAN) && !cpus_have_cap(ARM64_HAS_UAO)); + return (cpus_have_const_cap(ARM64_HAS_PAN) && !cpus_have_const_cap(ARM64_HAS_UAO)); } diff --git a/arch/arm64/kernel/debug-monitors.c b/arch/arm64/kernel/debug-monitors.c index 73ae90e..605df76 100644 --- a/arch/arm64/kernel/debug-monitors.c +++ b/arch/arm64/kernel/debug-monitors.c @@ -226,6 +226,8 @@ static void send_user_sigtrap(int si_code) static int single_step_handler(unsigned long addr, unsigned int esr, struct pt_regs *regs) { + bool handler_found = false; + /* * If we are stepping a pending breakpoint, call the hw_breakpoint * handler first. @@ -233,7 +235,14 @@ static int single_step_handler(unsigned long addr, unsigned int esr, if (!reinstall_suspended_bps(regs)) return 0; - if (user_mode(regs)) { +#ifdef CONFIG_KPROBES + if (kprobe_single_step_handler(regs, esr) == DBG_HOOK_HANDLED) + handler_found = true; +#endif + if (!handler_found && call_step_hook(regs, esr) == DBG_HOOK_HANDLED) + handler_found = true; + + if (!handler_found && user_mode(regs)) { send_user_sigtrap(TRAP_TRACE); /* @@ -243,15 +252,8 @@ static int single_step_handler(unsigned long addr, unsigned int esr, * to the active-not-pending state). */ user_rewind_single_step(current); - } else { -#ifdef CONFIG_KPROBES - if (kprobe_single_step_handler(regs, esr) == DBG_HOOK_HANDLED) - return 0; -#endif - if (call_step_hook(regs, esr) == DBG_HOOK_HANDLED) - return 0; - - pr_warning("Unexpected kernel single-step exception at EL1\n"); + } else if (!handler_found) { + pr_warn("Unexpected kernel single-step exception at EL1\n"); /* * Re-enable stepping since we know that we will be * returning to regs. @@ -304,16 +306,20 @@ NOKPROBE_SYMBOL(call_break_hook); static int brk_handler(unsigned long addr, unsigned int esr, struct pt_regs *regs) { - if (user_mode(regs)) { - send_user_sigtrap(TRAP_BRKPT); - } + bool handler_found = false; + #ifdef CONFIG_KPROBES - else if ((esr & BRK64_ESR_MASK) == BRK64_ESR_KPROBES) { - if (kprobe_breakpoint_handler(regs, esr) != DBG_HOOK_HANDLED) - return -EFAULT; + if ((esr & BRK64_ESR_MASK) == BRK64_ESR_KPROBES) { + if (kprobe_breakpoint_handler(regs, esr) == DBG_HOOK_HANDLED) + handler_found = true; } #endif - else if (call_break_hook(regs, esr) != DBG_HOOK_HANDLED) { + if (!handler_found && call_break_hook(regs, esr) == DBG_HOOK_HANDLED) + handler_found = true; + + if (!handler_found && user_mode(regs)) { + send_user_sigtrap(TRAP_BRKPT); + } else if (!handler_found) { pr_warn("Unexpected kernel BRK exception at EL1\n"); return -EFAULT; } diff --git a/arch/arm64/kernel/efi.c b/arch/arm64/kernel/efi.c index ba9bee3..5d17f37 100644 --- a/arch/arm64/kernel/efi.c +++ b/arch/arm64/kernel/efi.c @@ -62,8 +62,8 @@ struct screen_info screen_info __section(.data); int __init efi_create_mapping(struct mm_struct *mm, efi_memory_desc_t *md) { pteval_t prot_val = create_mapping_protection(md); - bool allow_block_mappings = (md->type != EFI_RUNTIME_SERVICES_CODE && - md->type != EFI_RUNTIME_SERVICES_DATA); + bool page_mappings_only = (md->type == EFI_RUNTIME_SERVICES_CODE || + md->type == EFI_RUNTIME_SERVICES_DATA); if (!PAGE_ALIGNED(md->phys_addr) || !PAGE_ALIGNED(md->num_pages << EFI_PAGE_SHIFT)) { @@ -76,12 +76,12 @@ int __init efi_create_mapping(struct mm_struct *mm, efi_memory_desc_t *md) * from the MMU routines. So avoid block mappings altogether in * that case. */ - allow_block_mappings = false; + page_mappings_only = true; } create_pgd_mapping(mm, md->phys_addr, md->virt_addr, md->num_pages << EFI_PAGE_SHIFT, - __pgprot(prot_val | PTE_NG), allow_block_mappings); + __pgprot(prot_val | PTE_NG), page_mappings_only); return 0; } diff --git a/arch/arm64/kernel/entry.S b/arch/arm64/kernel/entry.S index 223d54a..4f0d763 100644 --- a/arch/arm64/kernel/entry.S +++ b/arch/arm64/kernel/entry.S @@ -29,7 +29,9 @@ #include <asm/esr.h> #include <asm/irq.h> #include <asm/memory.h> +#include <asm/ptrace.h> #include <asm/thread_info.h> +#include <asm/uaccess.h> #include <asm/unistd.h> /* @@ -90,9 +92,8 @@ .if \el == 0 mrs x21, sp_el0 - mov tsk, sp - and tsk, tsk, #~(THREAD_SIZE - 1) // Ensure MDSCR_EL1.SS is clear, - ldr x19, [tsk, #TI_FLAGS] // since we can unmask debug + ldr_this_cpu tsk, __entry_task, x20 // Ensure MDSCR_EL1.SS is clear, + ldr x19, [tsk, #TSK_TI_FLAGS] // since we can unmask debug disable_step_tsk x19, x20 // exceptions when scheduling. mov x29, xzr // fp pointed to user-space @@ -100,15 +101,41 @@ add x21, sp, #S_FRAME_SIZE get_thread_info tsk /* Save the task's original addr_limit and set USER_DS (TASK_SIZE_64) */ - ldr x20, [tsk, #TI_ADDR_LIMIT] + ldr x20, [tsk, #TSK_TI_ADDR_LIMIT] str x20, [sp, #S_ORIG_ADDR_LIMIT] mov x20, #TASK_SIZE_64 - str x20, [tsk, #TI_ADDR_LIMIT] + str x20, [tsk, #TSK_TI_ADDR_LIMIT] /* No need to reset PSTATE.UAO, hardware's already set it to 0 for us */ .endif /* \el == 0 */ mrs x22, elr_el1 mrs x23, spsr_el1 stp lr, x21, [sp, #S_LR] + +#ifdef CONFIG_ARM64_SW_TTBR0_PAN + /* + * Set the TTBR0 PAN bit in SPSR. When the exception is taken from + * EL0, there is no need to check the state of TTBR0_EL1 since + * accesses are always enabled. + * Note that the meaning of this bit differs from the ARMv8.1 PAN + * feature as all TTBR0_EL1 accesses are disabled, not just those to + * user mappings. + */ +alternative_if ARM64_HAS_PAN + b 1f // skip TTBR0 PAN +alternative_else_nop_endif + + .if \el != 0 + mrs x21, ttbr0_el1 + tst x21, #0xffff << 48 // Check for the reserved ASID + orr x23, x23, #PSR_PAN_BIT // Set the emulated PAN in the saved SPSR + b.eq 1f // TTBR0 access already disabled + and x23, x23, #~PSR_PAN_BIT // Clear the emulated PAN in the saved SPSR + .endif + + __uaccess_ttbr0_disable x21 +1: +#endif + stp x22, x23, [sp, #S_PC] /* @@ -139,7 +166,7 @@ .if \el != 0 /* Restore the task's original addr_limit. */ ldr x20, [sp, #S_ORIG_ADDR_LIMIT] - str x20, [tsk, #TI_ADDR_LIMIT] + str x20, [tsk, #TSK_TI_ADDR_LIMIT] /* No need to restore UAO, it will be restored from SPSR_EL1 */ .endif @@ -147,6 +174,40 @@ ldp x21, x22, [sp, #S_PC] // load ELR, SPSR .if \el == 0 ct_user_enter + .endif + +#ifdef CONFIG_ARM64_SW_TTBR0_PAN + /* + * Restore access to TTBR0_EL1. If returning to EL0, no need for SPSR + * PAN bit checking. + */ +alternative_if ARM64_HAS_PAN + b 2f // skip TTBR0 PAN +alternative_else_nop_endif + + .if \el != 0 + tbnz x22, #22, 1f // Skip re-enabling TTBR0 access if the PSR_PAN_BIT is set + .endif + + __uaccess_ttbr0_enable x0 + + .if \el == 0 + /* + * Enable errata workarounds only if returning to user. The only + * workaround currently required for TTBR0_EL1 changes are for the + * Cavium erratum 27456 (broadcast TLBI instructions may cause I-cache + * corruption). + */ + post_ttbr0_update_workaround + .endif +1: + .if \el != 0 + and x22, x22, #~PSR_PAN_BIT // ARMv8.0 CPUs do not understand this bit + .endif +2: +#endif + + .if \el == 0 ldr x23, [sp, #S_SP] // load return stack pointer msr sp_el0, x23 #ifdef CONFIG_ARM64_ERRATUM_845719 @@ -162,6 +223,7 @@ alternative_if ARM64_WORKAROUND_845719 alternative_else_nop_endif #endif .endif + msr elr_el1, x21 // set up the return data msr spsr_el1, x22 ldp x0, x1, [sp, #16 * 0] @@ -184,23 +246,20 @@ alternative_else_nop_endif eret // return to kernel .endm - .macro get_thread_info, rd - mrs \rd, sp_el0 - .endm - .macro irq_stack_entry mov x19, sp // preserve the original sp /* - * Compare sp with the current thread_info, if the top - * ~(THREAD_SIZE - 1) bits match, we are on a task stack, and - * should switch to the irq stack. + * Compare sp with the base of the task stack. + * If the top ~(THREAD_SIZE - 1) bits match, we are on a task stack, + * and should switch to the irq stack. */ - and x25, x19, #~(THREAD_SIZE - 1) - cmp x25, tsk - b.ne 9998f + ldr x25, [tsk, TSK_STACK] + eor x25, x25, x19 + and x25, x25, #~(THREAD_SIZE - 1) + cbnz x25, 9998f - this_cpu_ptr irq_stack, x25, x26 + adr_this_cpu x25, irq_stack, x26 mov x26, #IRQ_STACK_START_SP add x26, x25, x26 @@ -427,9 +486,9 @@ el1_irq: irq_handler #ifdef CONFIG_PREEMPT - ldr w24, [tsk, #TI_PREEMPT] // get preempt count + ldr w24, [tsk, #TSK_TI_PREEMPT] // get preempt count cbnz w24, 1f // preempt count != 0 - ldr x0, [tsk, #TI_FLAGS] // get flags + ldr x0, [tsk, #TSK_TI_FLAGS] // get flags tbz x0, #TIF_NEED_RESCHED, 1f // needs rescheduling? bl el1_preempt 1: @@ -444,7 +503,7 @@ ENDPROC(el1_irq) el1_preempt: mov x24, lr 1: bl preempt_schedule_irq // irq en/disable is done inside - ldr x0, [tsk, #TI_FLAGS] // get new tasks TI_FLAGS + ldr x0, [tsk, #TSK_TI_FLAGS] // get new tasks TI_FLAGS tbnz x0, #TIF_NEED_RESCHED, 1b // needs rescheduling? ret x24 #endif @@ -674,8 +733,7 @@ ENTRY(cpu_switch_to) ldp x29, x9, [x8], #16 ldr lr, [x8] mov sp, x9 - and x9, x9, #~(THREAD_SIZE - 1) - msr sp_el0, x9 + msr sp_el0, x1 ret ENDPROC(cpu_switch_to) @@ -686,7 +744,7 @@ ENDPROC(cpu_switch_to) ret_fast_syscall: disable_irq // disable interrupts str x0, [sp, #S_X0] // returned x0 - ldr x1, [tsk, #TI_FLAGS] // re-check for syscall tracing + ldr x1, [tsk, #TSK_TI_FLAGS] // re-check for syscall tracing and x2, x1, #_TIF_SYSCALL_WORK cbnz x2, ret_fast_syscall_trace and x2, x1, #_TIF_WORK_MASK @@ -706,14 +764,14 @@ work_pending: #ifdef CONFIG_TRACE_IRQFLAGS bl trace_hardirqs_on // enabled while in userspace #endif - ldr x1, [tsk, #TI_FLAGS] // re-check for single-step + ldr x1, [tsk, #TSK_TI_FLAGS] // re-check for single-step b finish_ret_to_user /* * "slow" syscall return path. */ ret_to_user: disable_irq // disable interrupts - ldr x1, [tsk, #TI_FLAGS] + ldr x1, [tsk, #TSK_TI_FLAGS] and x2, x1, #_TIF_WORK_MASK cbnz x2, work_pending finish_ret_to_user: @@ -746,7 +804,7 @@ el0_svc_naked: // compat entry point enable_dbg_and_irq ct_user_exit 1 - ldr x16, [tsk, #TI_FLAGS] // check for syscall hooks + ldr x16, [tsk, #TSK_TI_FLAGS] // check for syscall hooks tst x16, #_TIF_SYSCALL_WORK b.ne __sys_trace cmp scno, sc_nr // check upper syscall limit diff --git a/arch/arm64/kernel/fpsimd.c b/arch/arm64/kernel/fpsimd.c index 394c61d..b883f1f 100644 --- a/arch/arm64/kernel/fpsimd.c +++ b/arch/arm64/kernel/fpsimd.c @@ -127,6 +127,8 @@ void do_fpsimd_exc(unsigned int esr, struct pt_regs *regs) void fpsimd_thread_switch(struct task_struct *next) { + if (!system_supports_fpsimd()) + return; /* * Save the current FPSIMD state to memory, but only if whatever is in * the registers is in fact the most recent userland FPSIMD state of @@ -157,6 +159,8 @@ void fpsimd_thread_switch(struct task_struct *next) void fpsimd_flush_thread(void) { + if (!system_supports_fpsimd()) + return; memset(¤t->thread.fpsimd_state, 0, sizeof(struct fpsimd_state)); fpsimd_flush_task_state(current); set_thread_flag(TIF_FOREIGN_FPSTATE); @@ -168,6 +172,8 @@ void fpsimd_flush_thread(void) */ void fpsimd_preserve_current_state(void) { + if (!system_supports_fpsimd()) + return; preempt_disable(); if (!test_thread_flag(TIF_FOREIGN_FPSTATE)) fpsimd_save_state(¤t->thread.fpsimd_state); @@ -181,6 +187,8 @@ void fpsimd_preserve_current_state(void) */ void fpsimd_restore_current_state(void) { + if (!system_supports_fpsimd()) + return; preempt_disable(); if (test_and_clear_thread_flag(TIF_FOREIGN_FPSTATE)) { struct fpsimd_state *st = ¤t->thread.fpsimd_state; @@ -199,6 +207,8 @@ void fpsimd_restore_current_state(void) */ void fpsimd_update_current_state(struct fpsimd_state *state) { + if (!system_supports_fpsimd()) + return; preempt_disable(); fpsimd_load_state(state); if (test_and_clear_thread_flag(TIF_FOREIGN_FPSTATE)) { @@ -228,6 +238,8 @@ static DEFINE_PER_CPU(struct fpsimd_partial_state, softirq_fpsimdstate); */ void kernel_neon_begin_partial(u32 num_regs) { + if (WARN_ON(!system_supports_fpsimd())) + return; if (in_interrupt()) { struct fpsimd_partial_state *s = this_cpu_ptr( in_irq() ? &hardirq_fpsimdstate : &softirq_fpsimdstate); @@ -252,6 +264,8 @@ EXPORT_SYMBOL(kernel_neon_begin_partial); void kernel_neon_end(void) { + if (!system_supports_fpsimd()) + return; if (in_interrupt()) { struct fpsimd_partial_state *s = this_cpu_ptr( in_irq() ? &hardirq_fpsimdstate : &softirq_fpsimdstate); diff --git a/arch/arm64/kernel/head.S b/arch/arm64/kernel/head.S index 332e331..4b1abac 100644 --- a/arch/arm64/kernel/head.S +++ b/arch/arm64/kernel/head.S @@ -326,14 +326,14 @@ __create_page_tables: * dirty cache lines being evicted. */ adrp x0, idmap_pg_dir - adrp x1, swapper_pg_dir + SWAPPER_DIR_SIZE + adrp x1, swapper_pg_dir + SWAPPER_DIR_SIZE + RESERVED_TTBR0_SIZE bl __inval_cache_range /* * Clear the idmap and swapper page tables. */ adrp x0, idmap_pg_dir - adrp x6, swapper_pg_dir + SWAPPER_DIR_SIZE + adrp x6, swapper_pg_dir + SWAPPER_DIR_SIZE + RESERVED_TTBR0_SIZE 1: stp xzr, xzr, [x0], #16 stp xzr, xzr, [x0], #16 stp xzr, xzr, [x0], #16 @@ -412,7 +412,7 @@ __create_page_tables: * tables again to remove any speculatively loaded cache lines. */ adrp x0, idmap_pg_dir - adrp x1, swapper_pg_dir + SWAPPER_DIR_SIZE + adrp x1, swapper_pg_dir + SWAPPER_DIR_SIZE + RESERVED_TTBR0_SIZE dmb sy bl __inval_cache_range @@ -428,7 +428,8 @@ ENDPROC(__create_page_tables) __primary_switched: adrp x4, init_thread_union add sp, x4, #THREAD_SIZE - msr sp_el0, x4 // Save thread_info + adr_l x5, init_task + msr sp_el0, x5 // Save thread_info adr_l x8, vectors // load VBAR_EL1 with virtual msr vbar_el1, x8 // vector table address @@ -524,10 +525,21 @@ set_hcr: msr hcr_el2, x0 isb - /* Generic timers. */ + /* + * Allow Non-secure EL1 and EL0 to access physical timer and counter. + * This is not necessary for VHE, since the host kernel runs in EL2, + * and EL0 accesses are configured in the later stage of boot process. + * Note that when HCR_EL2.E2H == 1, CNTHCTL_EL2 has the same bit layout + * as CNTKCTL_EL1, and CNTKCTL_EL1 accessing instructions are redefined + * to access CNTHCTL_EL2. This allows the kernel designed to run at EL1 + * to transparently mess with the EL0 bits via CNTKCTL_EL1 access in + * EL2. + */ + cbnz x2, 1f mrs x0, cnthctl_el2 orr x0, x0, #3 // Enable EL1 physical timers msr cnthctl_el2, x0 +1: msr cntvoff_el2, xzr // Clear virtual offset #ifdef CONFIG_ARM_GIC_V3 @@ -699,10 +711,10 @@ __secondary_switched: isb adr_l x0, secondary_data - ldr x0, [x0, #CPU_BOOT_STACK] // get secondary_data.stack - mov sp, x0 - and x0, x0, #~(THREAD_SIZE - 1) - msr sp_el0, x0 // save thread_info + ldr x1, [x0, #CPU_BOOT_STACK] // get secondary_data.stack + mov sp, x1 + ldr x2, [x0, #CPU_BOOT_TASK] + msr sp_el0, x2 mov x29, #0 b secondary_start_kernel ENDPROC(__secondary_switched) diff --git a/arch/arm64/kernel/hw_breakpoint.c b/arch/arm64/kernel/hw_breakpoint.c index 948b731..1b3c747 100644 --- a/arch/arm64/kernel/hw_breakpoint.c +++ b/arch/arm64/kernel/hw_breakpoint.c @@ -317,9 +317,21 @@ static int get_hbp_len(u8 hbp_len) case ARM_BREAKPOINT_LEN_2: len_in_bytes = 2; break; + case ARM_BREAKPOINT_LEN_3: + len_in_bytes = 3; + break; case ARM_BREAKPOINT_LEN_4: len_in_bytes = 4; break; + case ARM_BREAKPOINT_LEN_5: + len_in_bytes = 5; + break; + case ARM_BREAKPOINT_LEN_6: + len_in_bytes = 6; + break; + case ARM_BREAKPOINT_LEN_7: + len_in_bytes = 7; + break; case ARM_BREAKPOINT_LEN_8: len_in_bytes = 8; break; @@ -349,7 +361,7 @@ int arch_check_bp_in_kernelspace(struct perf_event *bp) * to generic breakpoint descriptions. */ int arch_bp_generic_fields(struct arch_hw_breakpoint_ctrl ctrl, - int *gen_len, int *gen_type) + int *gen_len, int *gen_type, int *offset) { /* Type */ switch (ctrl.type) { @@ -369,17 +381,33 @@ int arch_bp_generic_fields(struct arch_hw_breakpoint_ctrl ctrl, return -EINVAL; } + if (!ctrl.len) + return -EINVAL; + *offset = __ffs(ctrl.len); + /* Len */ - switch (ctrl.len) { + switch (ctrl.len >> *offset) { case ARM_BREAKPOINT_LEN_1: *gen_len = HW_BREAKPOINT_LEN_1; break; case ARM_BREAKPOINT_LEN_2: *gen_len = HW_BREAKPOINT_LEN_2; break; + case ARM_BREAKPOINT_LEN_3: + *gen_len = HW_BREAKPOINT_LEN_3; + break; case ARM_BREAKPOINT_LEN_4: *gen_len = HW_BREAKPOINT_LEN_4; break; + case ARM_BREAKPOINT_LEN_5: + *gen_len = HW_BREAKPOINT_LEN_5; + break; + case ARM_BREAKPOINT_LEN_6: + *gen_len = HW_BREAKPOINT_LEN_6; + break; + case ARM_BREAKPOINT_LEN_7: + *gen_len = HW_BREAKPOINT_LEN_7; + break; case ARM_BREAKPOINT_LEN_8: *gen_len = HW_BREAKPOINT_LEN_8; break; @@ -423,9 +451,21 @@ static int arch_build_bp_info(struct perf_event *bp) case HW_BREAKPOINT_LEN_2: info->ctrl.len = ARM_BREAKPOINT_LEN_2; break; + case HW_BREAKPOINT_LEN_3: + info->ctrl.len = ARM_BREAKPOINT_LEN_3; + break; case HW_BREAKPOINT_LEN_4: info->ctrl.len = ARM_BREAKPOINT_LEN_4; break; + case HW_BREAKPOINT_LEN_5: + info->ctrl.len = ARM_BREAKPOINT_LEN_5; + break; + case HW_BREAKPOINT_LEN_6: + info->ctrl.len = ARM_BREAKPOINT_LEN_6; + break; + case HW_BREAKPOINT_LEN_7: + info->ctrl.len = ARM_BREAKPOINT_LEN_7; + break; case HW_BREAKPOINT_LEN_8: info->ctrl.len = ARM_BREAKPOINT_LEN_8; break; @@ -517,18 +557,17 @@ int arch_validate_hwbkpt_settings(struct perf_event *bp) default: return -EINVAL; } - - info->address &= ~alignment_mask; - info->ctrl.len <<= offset; } else { if (info->ctrl.type == ARM_BREAKPOINT_EXECUTE) alignment_mask = 0x3; else alignment_mask = 0x7; - if (info->address & alignment_mask) - return -EINVAL; + offset = info->address & alignment_mask; } + info->address &= ~alignment_mask; + info->ctrl.len <<= offset; + /* * Disallow per-task kernel breakpoints since these would * complicate the stepping code. @@ -661,12 +700,47 @@ unlock: } NOKPROBE_SYMBOL(breakpoint_handler); +/* + * Arm64 hardware does not always report a watchpoint hit address that matches + * one of the watchpoints set. It can also report an address "near" the + * watchpoint if a single instruction access both watched and unwatched + * addresses. There is no straight-forward way, short of disassembling the + * offending instruction, to map that address back to the watchpoint. This + * function computes the distance of the memory access from the watchpoint as a + * heuristic for the likelyhood that a given access triggered the watchpoint. + * + * See Section D2.10.5 "Determining the memory location that caused a Watchpoint + * exception" of ARMv8 Architecture Reference Manual for details. + * + * The function returns the distance of the address from the bytes watched by + * the watchpoint. In case of an exact match, it returns 0. + */ +static u64 get_distance_from_watchpoint(unsigned long addr, u64 val, + struct arch_hw_breakpoint_ctrl *ctrl) +{ + u64 wp_low, wp_high; + u32 lens, lene; + + lens = __ffs(ctrl->len); + lene = __fls(ctrl->len); + + wp_low = val + lens; + wp_high = val + lene; + if (addr < wp_low) + return wp_low - addr; + else if (addr > wp_high) + return addr - wp_high; + else + return 0; +} + static int watchpoint_handler(unsigned long addr, unsigned int esr, struct pt_regs *regs) { - int i, step = 0, *kernel_step, access; + int i, step = 0, *kernel_step, access, closest_match = 0; + u64 min_dist = -1, dist; u32 ctrl_reg; - u64 val, alignment_mask; + u64 val; struct perf_event *wp, **slots; struct debug_info *debug_info; struct arch_hw_breakpoint *info; @@ -675,35 +749,15 @@ static int watchpoint_handler(unsigned long addr, unsigned int esr, slots = this_cpu_ptr(wp_on_reg); debug_info = ¤t->thread.debug; + /* + * Find all watchpoints that match the reported address. If no exact + * match is found. Attribute the hit to the closest watchpoint. + */ + rcu_read_lock(); for (i = 0; i < core_num_wrps; ++i) { - rcu_read_lock(); - wp = slots[i]; - if (wp == NULL) - goto unlock; - - info = counter_arch_bp(wp); - /* AArch32 watchpoints are either 4 or 8 bytes aligned. */ - if (is_compat_task()) { - if (info->ctrl.len == ARM_BREAKPOINT_LEN_8) - alignment_mask = 0x7; - else - alignment_mask = 0x3; - } else { - alignment_mask = 0x7; - } - - /* Check if the watchpoint value matches. */ - val = read_wb_reg(AARCH64_DBG_REG_WVR, i); - if (val != (addr & ~alignment_mask)) - goto unlock; - - /* Possible match, check the byte address select to confirm. */ - ctrl_reg = read_wb_reg(AARCH64_DBG_REG_WCR, i); - decode_ctrl_reg(ctrl_reg, &ctrl); - if (!((1 << (addr & alignment_mask)) & ctrl.len)) - goto unlock; + continue; /* * Check that the access type matches. @@ -712,18 +766,41 @@ static int watchpoint_handler(unsigned long addr, unsigned int esr, access = (esr & AARCH64_ESR_ACCESS_MASK) ? HW_BREAKPOINT_W : HW_BREAKPOINT_R; if (!(access & hw_breakpoint_type(wp))) - goto unlock; + continue; + /* Check if the watchpoint value and byte select match. */ + val = read_wb_reg(AARCH64_DBG_REG_WVR, i); + ctrl_reg = read_wb_reg(AARCH64_DBG_REG_WCR, i); + decode_ctrl_reg(ctrl_reg, &ctrl); + dist = get_distance_from_watchpoint(addr, val, &ctrl); + if (dist < min_dist) { + min_dist = dist; + closest_match = i; + } + /* Is this an exact match? */ + if (dist != 0) + continue; + + info = counter_arch_bp(wp); info->trigger = addr; perf_bp_event(wp, regs); /* Do we need to handle the stepping? */ if (is_default_overflow_handler(wp)) step = 1; + } + if (min_dist > 0 && min_dist != -1) { + /* No exact match found. */ + wp = slots[closest_match]; + info = counter_arch_bp(wp); + info->trigger = addr; + perf_bp_event(wp, regs); -unlock: - rcu_read_unlock(); + /* Do we need to handle the stepping? */ + if (is_default_overflow_handler(wp)) + step = 1; } + rcu_read_unlock(); if (!step) return 0; diff --git a/arch/arm64/kernel/insn.c b/arch/arm64/kernel/insn.c index 6f2ac4f..94b62c1 100644 --- a/arch/arm64/kernel/insn.c +++ b/arch/arm64/kernel/insn.c @@ -30,7 +30,6 @@ #include <asm/cacheflush.h> #include <asm/debug-monitors.h> #include <asm/fixmap.h> -#include <asm/opcodes.h> #include <asm/insn.h> #define AARCH64_INSN_SF_BIT BIT(31) diff --git a/arch/arm64/kernel/kgdb.c b/arch/arm64/kernel/kgdb.c index e017a94..d217c9e 100644 --- a/arch/arm64/kernel/kgdb.c +++ b/arch/arm64/kernel/kgdb.c @@ -247,6 +247,9 @@ NOKPROBE_SYMBOL(kgdb_compiled_brk_fn); static int kgdb_step_brk_fn(struct pt_regs *regs, unsigned int esr) { + if (!kgdb_single_step) + return DBG_HOOK_ERROR; + kgdb_handle_exception(1, SIGTRAP, 0, regs); return 0; } diff --git a/arch/arm64/kernel/probes/Makefile b/arch/arm64/kernel/probes/Makefile index ce06312..89b6df6 100644 --- a/arch/arm64/kernel/probes/Makefile +++ b/arch/arm64/kernel/probes/Makefile @@ -1,3 +1,5 @@ obj-$(CONFIG_KPROBES) += kprobes.o decode-insn.o \ kprobes_trampoline.o \ simulate-insn.o +obj-$(CONFIG_UPROBES) += uprobes.o decode-insn.o \ + simulate-insn.o diff --git a/arch/arm64/kernel/probes/decode-insn.c b/arch/arm64/kernel/probes/decode-insn.c index d1731bf..6bf6657 100644 --- a/arch/arm64/kernel/probes/decode-insn.c +++ b/arch/arm64/kernel/probes/decode-insn.c @@ -17,7 +17,6 @@ #include <linux/kprobes.h> #include <linux/module.h> #include <linux/kallsyms.h> -#include <asm/kprobes.h> #include <asm/insn.h> #include <asm/sections.h> @@ -78,8 +77,8 @@ static bool __kprobes aarch64_insn_is_steppable(u32 insn) * INSN_GOOD If instruction is supported and uses instruction slot, * INSN_GOOD_NO_SLOT If instruction is supported but doesn't use its slot. */ -static enum kprobe_insn __kprobes -arm_probe_decode_insn(kprobe_opcode_t insn, struct arch_specific_insn *asi) +enum probe_insn __kprobes +arm_probe_decode_insn(probe_opcode_t insn, struct arch_probe_insn *api) { /* * Instructions reading or modifying the PC won't work from the XOL @@ -89,26 +88,26 @@ arm_probe_decode_insn(kprobe_opcode_t insn, struct arch_specific_insn *asi) return INSN_GOOD; if (aarch64_insn_is_bcond(insn)) { - asi->handler = simulate_b_cond; + api->handler = simulate_b_cond; } else if (aarch64_insn_is_cbz(insn) || aarch64_insn_is_cbnz(insn)) { - asi->handler = simulate_cbz_cbnz; + api->handler = simulate_cbz_cbnz; } else if (aarch64_insn_is_tbz(insn) || aarch64_insn_is_tbnz(insn)) { - asi->handler = simulate_tbz_tbnz; + api->handler = simulate_tbz_tbnz; } else if (aarch64_insn_is_adr_adrp(insn)) { - asi->handler = simulate_adr_adrp; + api->handler = simulate_adr_adrp; } else if (aarch64_insn_is_b(insn) || aarch64_insn_is_bl(insn)) { - asi->handler = simulate_b_bl; + api->handler = simulate_b_bl; } else if (aarch64_insn_is_br(insn) || aarch64_insn_is_blr(insn) || aarch64_insn_is_ret(insn)) { - asi->handler = simulate_br_blr_ret; + api->handler = simulate_br_blr_ret; } else if (aarch64_insn_is_ldr_lit(insn)) { - asi->handler = simulate_ldr_literal; + api->handler = simulate_ldr_literal; } else if (aarch64_insn_is_ldrsw_lit(insn)) { - asi->handler = simulate_ldrsw_literal; + api->handler = simulate_ldrsw_literal; } else { /* * Instruction cannot be stepped out-of-line and we don't @@ -120,6 +119,7 @@ arm_probe_decode_insn(kprobe_opcode_t insn, struct arch_specific_insn *asi) return INSN_GOOD_NO_SLOT; } +#ifdef CONFIG_KPROBES static bool __kprobes is_probed_address_atomic(kprobe_opcode_t *scan_start, kprobe_opcode_t *scan_end) { @@ -138,12 +138,12 @@ is_probed_address_atomic(kprobe_opcode_t *scan_start, kprobe_opcode_t *scan_end) return false; } -enum kprobe_insn __kprobes +enum probe_insn __kprobes arm_kprobe_decode_insn(kprobe_opcode_t *addr, struct arch_specific_insn *asi) { - enum kprobe_insn decoded; - kprobe_opcode_t insn = le32_to_cpu(*addr); - kprobe_opcode_t *scan_end = NULL; + enum probe_insn decoded; + probe_opcode_t insn = le32_to_cpu(*addr); + probe_opcode_t *scan_end = NULL; unsigned long size = 0, offset = 0; /* @@ -162,7 +162,7 @@ arm_kprobe_decode_insn(kprobe_opcode_t *addr, struct arch_specific_insn *asi) else scan_end = addr - MAX_ATOMIC_CONTEXT_SIZE; } - decoded = arm_probe_decode_insn(insn, asi); + decoded = arm_probe_decode_insn(insn, &asi->api); if (decoded != INSN_REJECTED && scan_end) if (is_probed_address_atomic(addr - 1, scan_end)) @@ -170,3 +170,4 @@ arm_kprobe_decode_insn(kprobe_opcode_t *addr, struct arch_specific_insn *asi) return decoded; } +#endif diff --git a/arch/arm64/kernel/probes/decode-insn.h b/arch/arm64/kernel/probes/decode-insn.h index d438289..76d3f31 100644 --- a/arch/arm64/kernel/probes/decode-insn.h +++ b/arch/arm64/kernel/probes/decode-insn.h @@ -23,13 +23,17 @@ */ #define MAX_ATOMIC_CONTEXT_SIZE (128 / sizeof(kprobe_opcode_t)) -enum kprobe_insn { +enum probe_insn { INSN_REJECTED, INSN_GOOD_NO_SLOT, INSN_GOOD, }; -enum kprobe_insn __kprobes +#ifdef CONFIG_KPROBES +enum probe_insn __kprobes arm_kprobe_decode_insn(kprobe_opcode_t *addr, struct arch_specific_insn *asi); +#endif +enum probe_insn __kprobes +arm_probe_decode_insn(probe_opcode_t insn, struct arch_probe_insn *asi); #endif /* _ARM_KERNEL_KPROBES_ARM64_H */ diff --git a/arch/arm64/kernel/probes/kprobes.c b/arch/arm64/kernel/probes/kprobes.c index f5077ea..1decd2b 100644 --- a/arch/arm64/kernel/probes/kprobes.c +++ b/arch/arm64/kernel/probes/kprobes.c @@ -44,31 +44,31 @@ post_kprobe_handler(struct kprobe_ctlblk *, struct pt_regs *); static void __kprobes arch_prepare_ss_slot(struct kprobe *p) { /* prepare insn slot */ - p->ainsn.insn[0] = cpu_to_le32(p->opcode); + p->ainsn.api.insn[0] = cpu_to_le32(p->opcode); - flush_icache_range((uintptr_t) (p->ainsn.insn), - (uintptr_t) (p->ainsn.insn) + + flush_icache_range((uintptr_t) (p->ainsn.api.insn), + (uintptr_t) (p->ainsn.api.insn) + MAX_INSN_SIZE * sizeof(kprobe_opcode_t)); /* * Needs restoring of return address after stepping xol. */ - p->ainsn.restore = (unsigned long) p->addr + + p->ainsn.api.restore = (unsigned long) p->addr + sizeof(kprobe_opcode_t); } static void __kprobes arch_prepare_simulate(struct kprobe *p) { /* This instructions is not executed xol. No need to adjust the PC */ - p->ainsn.restore = 0; + p->ainsn.api.restore = 0; } static void __kprobes arch_simulate_insn(struct kprobe *p, struct pt_regs *regs) { struct kprobe_ctlblk *kcb = get_kprobe_ctlblk(); - if (p->ainsn.handler) - p->ainsn.handler((u32)p->opcode, (long)p->addr, regs); + if (p->ainsn.api.handler) + p->ainsn.api.handler((u32)p->opcode, (long)p->addr, regs); /* single step simulated, now go for post processing */ post_kprobe_handler(kcb, regs); @@ -98,18 +98,18 @@ int __kprobes arch_prepare_kprobe(struct kprobe *p) return -EINVAL; case INSN_GOOD_NO_SLOT: /* insn need simulation */ - p->ainsn.insn = NULL; + p->ainsn.api.insn = NULL; break; case INSN_GOOD: /* instruction uses slot */ - p->ainsn.insn = get_insn_slot(); - if (!p->ainsn.insn) + p->ainsn.api.insn = get_insn_slot(); + if (!p->ainsn.api.insn) return -ENOMEM; break; }; /* prepare the instruction */ - if (p->ainsn.insn) + if (p->ainsn.api.insn) arch_prepare_ss_slot(p); else arch_prepare_simulate(p); @@ -142,9 +142,9 @@ void __kprobes arch_disarm_kprobe(struct kprobe *p) void __kprobes arch_remove_kprobe(struct kprobe *p) { - if (p->ainsn.insn) { - free_insn_slot(p->ainsn.insn, 0); - p->ainsn.insn = NULL; + if (p->ainsn.api.insn) { + free_insn_slot(p->ainsn.api.insn, 0); + p->ainsn.api.insn = NULL; } } @@ -244,9 +244,9 @@ static void __kprobes setup_singlestep(struct kprobe *p, } - if (p->ainsn.insn) { + if (p->ainsn.api.insn) { /* prepare for single stepping */ - slot = (unsigned long)p->ainsn.insn; + slot = (unsigned long)p->ainsn.api.insn; set_ss_context(kcb, slot); /* mark pending ss */ @@ -295,8 +295,8 @@ post_kprobe_handler(struct kprobe_ctlblk *kcb, struct pt_regs *regs) return; /* return addr restore if non-branching insn */ - if (cur->ainsn.restore != 0) - instruction_pointer_set(regs, cur->ainsn.restore); + if (cur->ainsn.api.restore != 0) + instruction_pointer_set(regs, cur->ainsn.api.restore); /* restore back original saved kprobe variables and continue */ if (kcb->kprobe_status == KPROBE_REENTER) { diff --git a/arch/arm64/kernel/probes/simulate-insn.c b/arch/arm64/kernel/probes/simulate-insn.c index 8977ce9..357d3ef 100644 --- a/arch/arm64/kernel/probes/simulate-insn.c +++ b/arch/arm64/kernel/probes/simulate-insn.c @@ -13,28 +13,26 @@ * General Public License for more details. */ +#include <linux/bitops.h> #include <linux/kernel.h> #include <linux/kprobes.h> #include "simulate-insn.h" -#define sign_extend(x, signbit) \ - ((x) | (0 - ((x) & (1 << (signbit))))) - #define bbl_displacement(insn) \ - sign_extend(((insn) & 0x3ffffff) << 2, 27) + sign_extend32(((insn) & 0x3ffffff) << 2, 27) #define bcond_displacement(insn) \ - sign_extend(((insn >> 5) & 0x7ffff) << 2, 20) + sign_extend32(((insn >> 5) & 0x7ffff) << 2, 20) #define cbz_displacement(insn) \ - sign_extend(((insn >> 5) & 0x7ffff) << 2, 20) + sign_extend32(((insn >> 5) & 0x7ffff) << 2, 20) #define tbz_displacement(insn) \ - sign_extend(((insn >> 5) & 0x3fff) << 2, 15) + sign_extend32(((insn >> 5) & 0x3fff) << 2, 15) #define ldr_displacement(insn) \ - sign_extend(((insn >> 5) & 0x7ffff) << 2, 20) + sign_extend32(((insn >> 5) & 0x7ffff) << 2, 20) static inline void set_x_reg(struct pt_regs *regs, int reg, u64 val) { @@ -106,7 +104,7 @@ simulate_adr_adrp(u32 opcode, long addr, struct pt_regs *regs) xn = opcode & 0x1f; imm = ((opcode >> 3) & 0x1ffffc) | ((opcode >> 29) & 0x3); - imm = sign_extend(imm, 20); + imm = sign_extend64(imm, 20); if (opcode & 0x80000000) val = (imm<<12) + (addr & 0xfffffffffffff000); else diff --git a/arch/arm64/kernel/probes/uprobes.c b/arch/arm64/kernel/probes/uprobes.c new file mode 100644 index 0000000..26c9985 --- /dev/null +++ b/arch/arm64/kernel/probes/uprobes.c @@ -0,0 +1,216 @@ +/* + * Copyright (C) 2014-2016 Pratyush Anand <panand@redhat.com> + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ +#include <linux/highmem.h> +#include <linux/ptrace.h> +#include <linux/uprobes.h> +#include <asm/cacheflush.h> + +#include "decode-insn.h" + +#define UPROBE_INV_FAULT_CODE UINT_MAX + +void arch_uprobe_copy_ixol(struct page *page, unsigned long vaddr, + void *src, unsigned long len) +{ + void *xol_page_kaddr = kmap_atomic(page); + void *dst = xol_page_kaddr + (vaddr & ~PAGE_MASK); + + /* Initialize the slot */ + memcpy(dst, src, len); + + /* flush caches (dcache/icache) */ + sync_icache_aliases(dst, len); + + kunmap_atomic(xol_page_kaddr); +} + +unsigned long uprobe_get_swbp_addr(struct pt_regs *regs) +{ + return instruction_pointer(regs); +} + +int arch_uprobe_analyze_insn(struct arch_uprobe *auprobe, struct mm_struct *mm, + unsigned long addr) +{ + probe_opcode_t insn; + + /* TODO: Currently we do not support AARCH32 instruction probing */ + if (test_bit(TIF_32BIT, &mm->context.flags)) + return -ENOTSUPP; + else if (!IS_ALIGNED(addr, AARCH64_INSN_SIZE)) + return -EINVAL; + + insn = *(probe_opcode_t *)(&auprobe->insn[0]); + + switch (arm_probe_decode_insn(insn, &auprobe->api)) { + case INSN_REJECTED: + return -EINVAL; + + case INSN_GOOD_NO_SLOT: + auprobe->simulate = true; + break; + + default: + break; + } + + return 0; +} + +int arch_uprobe_pre_xol(struct arch_uprobe *auprobe, struct pt_regs *regs) +{ + struct uprobe_task *utask = current->utask; + + /* Initialize with an invalid fault code to detect if ol insn trapped */ + current->thread.fault_code = UPROBE_INV_FAULT_CODE; + + /* Instruction points to execute ol */ + instruction_pointer_set(regs, utask->xol_vaddr); + + user_enable_single_step(current); + + return 0; +} + +int arch_uprobe_post_xol(struct arch_uprobe *auprobe, struct pt_regs *regs) +{ + struct uprobe_task *utask = current->utask; + + WARN_ON_ONCE(current->thread.fault_code != UPROBE_INV_FAULT_CODE); + + /* Instruction points to execute next to breakpoint address */ + instruction_pointer_set(regs, utask->vaddr + 4); + + user_disable_single_step(current); + + return 0; +} +bool arch_uprobe_xol_was_trapped(struct task_struct *t) +{ + /* + * Between arch_uprobe_pre_xol and arch_uprobe_post_xol, if an xol + * insn itself is trapped, then detect the case with the help of + * invalid fault code which is being set in arch_uprobe_pre_xol + */ + if (t->thread.fault_code != UPROBE_INV_FAULT_CODE) + return true; + + return false; +} + +bool arch_uprobe_skip_sstep(struct arch_uprobe *auprobe, struct pt_regs *regs) +{ + probe_opcode_t insn; + unsigned long addr; + + if (!auprobe->simulate) + return false; + + insn = *(probe_opcode_t *)(&auprobe->insn[0]); + addr = instruction_pointer(regs); + + if (auprobe->api.handler) + auprobe->api.handler(insn, addr, regs); + + return true; +} + +void arch_uprobe_abort_xol(struct arch_uprobe *auprobe, struct pt_regs *regs) +{ + struct uprobe_task *utask = current->utask; + + /* + * Task has received a fatal signal, so reset back to probbed + * address. + */ + instruction_pointer_set(regs, utask->vaddr); + + user_disable_single_step(current); +} + +bool arch_uretprobe_is_alive(struct return_instance *ret, enum rp_check ctx, + struct pt_regs *regs) +{ + /* + * If a simple branch instruction (B) was called for retprobed + * assembly label then return true even when regs->sp and ret->stack + * are same. It will ensure that cleanup and reporting of return + * instances corresponding to callee label is done when + * handle_trampoline for called function is executed. + */ + if (ctx == RP_CHECK_CHAIN_CALL) + return regs->sp <= ret->stack; + else + return regs->sp < ret->stack; +} + +unsigned long +arch_uretprobe_hijack_return_addr(unsigned long trampoline_vaddr, + struct pt_regs *regs) +{ + unsigned long orig_ret_vaddr; + + orig_ret_vaddr = procedure_link_pointer(regs); + /* Replace the return addr with trampoline addr */ + procedure_link_pointer_set(regs, trampoline_vaddr); + + return orig_ret_vaddr; +} + +int arch_uprobe_exception_notify(struct notifier_block *self, + unsigned long val, void *data) +{ + return NOTIFY_DONE; +} + +static int uprobe_breakpoint_handler(struct pt_regs *regs, + unsigned int esr) +{ + if (user_mode(regs) && uprobe_pre_sstep_notifier(regs)) + return DBG_HOOK_HANDLED; + + return DBG_HOOK_ERROR; +} + +static int uprobe_single_step_handler(struct pt_regs *regs, + unsigned int esr) +{ + struct uprobe_task *utask = current->utask; + + if (user_mode(regs)) { + WARN_ON(utask && + (instruction_pointer(regs) != utask->xol_vaddr + 4)); + + if (uprobe_post_sstep_notifier(regs)) + return DBG_HOOK_HANDLED; + } + + return DBG_HOOK_ERROR; +} + +/* uprobe breakpoint handler hook */ +static struct break_hook uprobes_break_hook = { + .esr_mask = BRK64_ESR_MASK, + .esr_val = BRK64_ESR_UPROBES, + .fn = uprobe_breakpoint_handler, +}; + +/* uprobe single step handler hook */ +static struct step_hook uprobes_step_hook = { + .fn = uprobe_single_step_handler, +}; + +static int __init arch_init_uprobes(void) +{ + register_break_hook(&uprobes_break_hook); + register_step_hook(&uprobes_step_hook); + + return 0; +} + +device_initcall(arch_init_uprobes); diff --git a/arch/arm64/kernel/process.c b/arch/arm64/kernel/process.c index 01753cd..a3a2816 100644 --- a/arch/arm64/kernel/process.c +++ b/arch/arm64/kernel/process.c @@ -45,6 +45,7 @@ #include <linux/personality.h> #include <linux/notifier.h> #include <trace/events/power.h> +#include <linux/percpu.h> #include <asm/alternative.h> #include <asm/compat.h> @@ -282,7 +283,7 @@ int copy_thread(unsigned long clone_flags, unsigned long stack_start, memset(childregs, 0, sizeof(struct pt_regs)); childregs->pstate = PSR_MODE_EL1h; if (IS_ENABLED(CONFIG_ARM64_UAO) && - cpus_have_cap(ARM64_HAS_UAO)) + cpus_have_const_cap(ARM64_HAS_UAO)) childregs->pstate |= PSR_UAO_BIT; p->thread.cpu_context.x19 = stack_start; p->thread.cpu_context.x20 = stk_sz; @@ -322,6 +323,20 @@ void uao_thread_switch(struct task_struct *next) } /* + * We store our current task in sp_el0, which is clobbered by userspace. Keep a + * shadow copy so that we can restore this upon entry from userspace. + * + * This is *only* for exception entry from EL0, and is not valid until we + * __switch_to() a user task. + */ +DEFINE_PER_CPU(struct task_struct *, __entry_task); + +static void entry_task_switch(struct task_struct *next) +{ + __this_cpu_write(__entry_task, next); +} + +/* * Thread switching. */ struct task_struct *__switch_to(struct task_struct *prev, @@ -333,6 +348,7 @@ struct task_struct *__switch_to(struct task_struct *prev, tls_thread_switch(next); hw_breakpoint_thread_switch(next); contextidr_thread_switch(next); + entry_task_switch(next); uao_thread_switch(next); /* @@ -350,27 +366,35 @@ struct task_struct *__switch_to(struct task_struct *prev, unsigned long get_wchan(struct task_struct *p) { struct stackframe frame; - unsigned long stack_page; + unsigned long stack_page, ret = 0; int count = 0; if (!p || p == current || p->state == TASK_RUNNING) return 0; + stack_page = (unsigned long)try_get_task_stack(p); + if (!stack_page) + return 0; + frame.fp = thread_saved_fp(p); frame.sp = thread_saved_sp(p); frame.pc = thread_saved_pc(p); #ifdef CONFIG_FUNCTION_GRAPH_TRACER frame.graph = p->curr_ret_stack; #endif - stack_page = (unsigned long)task_stack_page(p); do { if (frame.sp < stack_page || frame.sp >= stack_page + THREAD_SIZE || unwind_frame(p, &frame)) - return 0; - if (!in_sched_functions(frame.pc)) - return frame.pc; + goto out; + if (!in_sched_functions(frame.pc)) { + ret = frame.pc; + goto out; + } } while (count ++ < 16); - return 0; + +out: + put_task_stack(p); + return ret; } unsigned long arch_align_stack(unsigned long sp) diff --git a/arch/arm64/kernel/ptrace.c b/arch/arm64/kernel/ptrace.c index e0c81da..fc35e06 100644 --- a/arch/arm64/kernel/ptrace.c +++ b/arch/arm64/kernel/ptrace.c @@ -327,13 +327,13 @@ static int ptrace_hbp_fill_attr_ctrl(unsigned int note_type, struct arch_hw_breakpoint_ctrl ctrl, struct perf_event_attr *attr) { - int err, len, type, disabled = !ctrl.enabled; + int err, len, type, offset, disabled = !ctrl.enabled; attr->disabled = disabled; if (disabled) return 0; - err = arch_bp_generic_fields(ctrl, &len, &type); + err = arch_bp_generic_fields(ctrl, &len, &type, &offset); if (err) return err; @@ -352,6 +352,7 @@ static int ptrace_hbp_fill_attr_ctrl(unsigned int note_type, attr->bp_len = len; attr->bp_type = type; + attr->bp_addr += offset; return 0; } @@ -404,7 +405,7 @@ static int ptrace_hbp_get_addr(unsigned int note_type, if (IS_ERR(bp)) return PTR_ERR(bp); - *addr = bp ? bp->attr.bp_addr : 0; + *addr = bp ? counter_arch_bp(bp)->address : 0; return 0; } diff --git a/arch/arm64/kernel/return_address.c b/arch/arm64/kernel/return_address.c index 1718706..12a87f2 100644 --- a/arch/arm64/kernel/return_address.c +++ b/arch/arm64/kernel/return_address.c @@ -12,6 +12,7 @@ #include <linux/export.h> #include <linux/ftrace.h> +#include <asm/stack_pointer.h> #include <asm/stacktrace.h> struct return_address_data { diff --git a/arch/arm64/kernel/setup.c b/arch/arm64/kernel/setup.c index f534f49..a53f52a 100644 --- a/arch/arm64/kernel/setup.c +++ b/arch/arm64/kernel/setup.c @@ -291,6 +291,15 @@ void __init setup_arch(char **cmdline_p) smp_init_cpus(); smp_build_mpidr_hash(); +#ifdef CONFIG_ARM64_SW_TTBR0_PAN + /* + * Make sure init_thread_info.ttbr0 always generates translation + * faults in case uaccess_enable() is inadvertently called by the init + * thread. + */ + init_task.thread_info.ttbr0 = virt_to_phys(empty_zero_page); +#endif + #ifdef CONFIG_VT #if defined(CONFIG_VGA_CONSOLE) conswitchp = &vga_con; diff --git a/arch/arm64/kernel/signal.c b/arch/arm64/kernel/signal.c index 404dd67..c7b6de6 100644 --- a/arch/arm64/kernel/signal.c +++ b/arch/arm64/kernel/signal.c @@ -414,6 +414,9 @@ asmlinkage void do_notify_resume(struct pt_regs *regs, } else { local_irq_enable(); + if (thread_flags & _TIF_UPROBE) + uprobe_notify_resume(regs); + if (thread_flags & _TIF_SIGPENDING) do_signal(regs); diff --git a/arch/arm64/kernel/sleep.S b/arch/arm64/kernel/sleep.S index 1bec41b..df67652 100644 --- a/arch/arm64/kernel/sleep.S +++ b/arch/arm64/kernel/sleep.S @@ -125,9 +125,6 @@ ENTRY(_cpu_resume) /* load sp from context */ ldr x2, [x0, #CPU_CTX_SP] mov sp, x2 - /* save thread_info */ - and x2, x2, #~(THREAD_SIZE - 1) - msr sp_el0, x2 /* * cpu_do_resume expects x0 to contain context address pointer */ diff --git a/arch/arm64/kernel/smp.c b/arch/arm64/kernel/smp.c index 8507703..cb87234 100644 --- a/arch/arm64/kernel/smp.c +++ b/arch/arm64/kernel/smp.c @@ -58,6 +58,9 @@ #define CREATE_TRACE_POINTS #include <trace/events/ipi.h> +DEFINE_PER_CPU_READ_MOSTLY(int, cpu_number); +EXPORT_PER_CPU_SYMBOL(cpu_number); + /* * as from 2.5, kernels no longer have an init_tasks structure * so we need some other way of telling a new secondary core @@ -146,6 +149,7 @@ int __cpu_up(unsigned int cpu, struct task_struct *idle) * We need to tell the secondary core where to find its stack and the * page tables. */ + secondary_data.task = idle; secondary_data.stack = task_stack_page(idle) + THREAD_START_SP; update_cpu_boot_status(CPU_MMU_OFF); __flush_dcache_area(&secondary_data, sizeof(secondary_data)); @@ -170,6 +174,7 @@ int __cpu_up(unsigned int cpu, struct task_struct *idle) pr_err("CPU%u: failed to boot: %d\n", cpu, ret); } + secondary_data.task = NULL; secondary_data.stack = NULL; status = READ_ONCE(secondary_data.status); if (ret && status) { @@ -208,7 +213,10 @@ int __cpu_up(unsigned int cpu, struct task_struct *idle) asmlinkage void secondary_start_kernel(void) { struct mm_struct *mm = &init_mm; - unsigned int cpu = smp_processor_id(); + unsigned int cpu; + + cpu = task_cpu(current); + set_my_cpu_offset(per_cpu_offset(cpu)); /* * All kernel threads share the same mm context; grab a @@ -217,8 +225,6 @@ asmlinkage void secondary_start_kernel(void) atomic_inc(&mm->mm_count); current->active_mm = mm; - set_my_cpu_offset(per_cpu_offset(smp_processor_id())); - /* * TTBR0 is only used for the identity mapping at this stage. Make it * point to zero page to avoid speculatively fetching new entries. @@ -718,6 +724,8 @@ void __init smp_prepare_cpus(unsigned int max_cpus) */ for_each_possible_cpu(cpu) { + per_cpu(cpu_number, cpu) = cpu; + if (cpu == smp_processor_id()) continue; diff --git a/arch/arm64/kernel/stacktrace.c b/arch/arm64/kernel/stacktrace.c index c2efddf..8a552a3 100644 --- a/arch/arm64/kernel/stacktrace.c +++ b/arch/arm64/kernel/stacktrace.c @@ -22,6 +22,7 @@ #include <linux/stacktrace.h> #include <asm/irq.h> +#include <asm/stack_pointer.h> #include <asm/stacktrace.h> /* @@ -128,7 +129,6 @@ void notrace walk_stackframe(struct task_struct *tsk, struct stackframe *frame, break; } } -EXPORT_SYMBOL(walk_stackframe); #ifdef CONFIG_STACKTRACE struct stack_trace_data { @@ -181,6 +181,9 @@ void save_stack_trace_tsk(struct task_struct *tsk, struct stack_trace *trace) struct stack_trace_data data; struct stackframe frame; + if (!try_get_task_stack(tsk)) + return; + data.trace = trace; data.skip = trace->skip; @@ -202,6 +205,8 @@ void save_stack_trace_tsk(struct task_struct *tsk, struct stack_trace *trace) walk_stackframe(tsk, &frame, save_trace, &data); if (trace->nr_entries < trace->max_entries) trace->entries[trace->nr_entries++] = ULONG_MAX; + + put_task_stack(tsk); } void save_stack_trace(struct stack_trace *trace) diff --git a/arch/arm64/kernel/suspend.c b/arch/arm64/kernel/suspend.c index bb0cd78..1e3be90 100644 --- a/arch/arm64/kernel/suspend.c +++ b/arch/arm64/kernel/suspend.c @@ -47,12 +47,6 @@ void notrace __cpu_suspend_exit(void) cpu_uninstall_idmap(); /* - * Restore per-cpu offset before any kernel - * subsystem relying on it has a chance to run. - */ - set_my_cpu_offset(per_cpu_offset(cpu)); - - /* * PSTATE was not saved over suspend/resume, re-enable any detected * features that might not have been set correctly. */ diff --git a/arch/arm64/kernel/topology.c b/arch/arm64/kernel/topology.c index 694f6de..23e9e13 100644 --- a/arch/arm64/kernel/topology.c +++ b/arch/arm64/kernel/topology.c @@ -19,10 +19,226 @@ #include <linux/nodemask.h> #include <linux/of.h> #include <linux/sched.h> +#include <linux/slab.h> +#include <linux/string.h> +#include <linux/cpufreq.h> +#include <asm/cpu.h> #include <asm/cputype.h> #include <asm/topology.h> +static DEFINE_PER_CPU(unsigned long, cpu_scale) = SCHED_CAPACITY_SCALE; +static DEFINE_MUTEX(cpu_scale_mutex); + +unsigned long arch_scale_cpu_capacity(struct sched_domain *sd, int cpu) +{ + return per_cpu(cpu_scale, cpu); +} + +static void set_capacity_scale(unsigned int cpu, unsigned long capacity) +{ + per_cpu(cpu_scale, cpu) = capacity; +} + +#ifdef CONFIG_PROC_SYSCTL +static ssize_t cpu_capacity_show(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + struct cpu *cpu = container_of(dev, struct cpu, dev); + + return sprintf(buf, "%lu\n", + arch_scale_cpu_capacity(NULL, cpu->dev.id)); +} + +static ssize_t cpu_capacity_store(struct device *dev, + struct device_attribute *attr, + const char *buf, + size_t count) +{ + struct cpu *cpu = container_of(dev, struct cpu, dev); + int this_cpu = cpu->dev.id, i; + unsigned long new_capacity; + ssize_t ret; + + if (count) { + ret = kstrtoul(buf, 0, &new_capacity); + if (ret) + return ret; + if (new_capacity > SCHED_CAPACITY_SCALE) + return -EINVAL; + + mutex_lock(&cpu_scale_mutex); + for_each_cpu(i, &cpu_topology[this_cpu].core_sibling) + set_capacity_scale(i, new_capacity); + mutex_unlock(&cpu_scale_mutex); + } + + return count; +} + +static DEVICE_ATTR_RW(cpu_capacity); + +static int register_cpu_capacity_sysctl(void) +{ + int i; + struct device *cpu; + + for_each_possible_cpu(i) { + cpu = get_cpu_device(i); + if (!cpu) { + pr_err("%s: too early to get CPU%d device!\n", + __func__, i); + continue; + } + device_create_file(cpu, &dev_attr_cpu_capacity); + } + + return 0; +} +subsys_initcall(register_cpu_capacity_sysctl); +#endif + +static u32 capacity_scale; +static u32 *raw_capacity; +static bool cap_parsing_failed; + +static void __init parse_cpu_capacity(struct device_node *cpu_node, int cpu) +{ + int ret; + u32 cpu_capacity; + + if (cap_parsing_failed) + return; + + ret = of_property_read_u32(cpu_node, + "capacity-dmips-mhz", + &cpu_capacity); + if (!ret) { + if (!raw_capacity) { + raw_capacity = kcalloc(num_possible_cpus(), + sizeof(*raw_capacity), + GFP_KERNEL); + if (!raw_capacity) { + pr_err("cpu_capacity: failed to allocate memory for raw capacities\n"); + cap_parsing_failed = true; + return; + } + } + capacity_scale = max(cpu_capacity, capacity_scale); + raw_capacity[cpu] = cpu_capacity; + pr_debug("cpu_capacity: %s cpu_capacity=%u (raw)\n", + cpu_node->full_name, raw_capacity[cpu]); + } else { + if (raw_capacity) { + pr_err("cpu_capacity: missing %s raw capacity\n", + cpu_node->full_name); + pr_err("cpu_capacity: partial information: fallback to 1024 for all CPUs\n"); + } + cap_parsing_failed = true; + kfree(raw_capacity); + } +} + +static void normalize_cpu_capacity(void) +{ + u64 capacity; + int cpu; + + if (!raw_capacity || cap_parsing_failed) + return; + + pr_debug("cpu_capacity: capacity_scale=%u\n", capacity_scale); + mutex_lock(&cpu_scale_mutex); + for_each_possible_cpu(cpu) { + pr_debug("cpu_capacity: cpu=%d raw_capacity=%u\n", + cpu, raw_capacity[cpu]); + capacity = (raw_capacity[cpu] << SCHED_CAPACITY_SHIFT) + / capacity_scale; + set_capacity_scale(cpu, capacity); + pr_debug("cpu_capacity: CPU%d cpu_capacity=%lu\n", + cpu, arch_scale_cpu_capacity(NULL, cpu)); + } + mutex_unlock(&cpu_scale_mutex); +} + +#ifdef CONFIG_CPU_FREQ +static cpumask_var_t cpus_to_visit; +static bool cap_parsing_done; +static void parsing_done_workfn(struct work_struct *work); +static DECLARE_WORK(parsing_done_work, parsing_done_workfn); + +static int +init_cpu_capacity_callback(struct notifier_block *nb, + unsigned long val, + void *data) +{ + struct cpufreq_policy *policy = data; + int cpu; + + if (cap_parsing_failed || cap_parsing_done) + return 0; + + switch (val) { + case CPUFREQ_NOTIFY: + pr_debug("cpu_capacity: init cpu capacity for CPUs [%*pbl] (to_visit=%*pbl)\n", + cpumask_pr_args(policy->related_cpus), + cpumask_pr_args(cpus_to_visit)); + cpumask_andnot(cpus_to_visit, + cpus_to_visit, + policy->related_cpus); + for_each_cpu(cpu, policy->related_cpus) { + raw_capacity[cpu] = arch_scale_cpu_capacity(NULL, cpu) * + policy->cpuinfo.max_freq / 1000UL; + capacity_scale = max(raw_capacity[cpu], capacity_scale); + } + if (cpumask_empty(cpus_to_visit)) { + normalize_cpu_capacity(); + kfree(raw_capacity); + pr_debug("cpu_capacity: parsing done\n"); + cap_parsing_done = true; + schedule_work(&parsing_done_work); + } + } + return 0; +} + +static struct notifier_block init_cpu_capacity_notifier = { + .notifier_call = init_cpu_capacity_callback, +}; + +static int __init register_cpufreq_notifier(void) +{ + if (cap_parsing_failed) + return -EINVAL; + + if (!alloc_cpumask_var(&cpus_to_visit, GFP_KERNEL)) { + pr_err("cpu_capacity: failed to allocate memory for cpus_to_visit\n"); + return -ENOMEM; + } + cpumask_copy(cpus_to_visit, cpu_possible_mask); + + return cpufreq_register_notifier(&init_cpu_capacity_notifier, + CPUFREQ_POLICY_NOTIFIER); +} +core_initcall(register_cpufreq_notifier); + +static void parsing_done_workfn(struct work_struct *work) +{ + cpufreq_unregister_notifier(&init_cpu_capacity_notifier, + CPUFREQ_POLICY_NOTIFIER); +} + +#else +static int __init free_raw_capacity(void) +{ + kfree(raw_capacity); + + return 0; +} +core_initcall(free_raw_capacity); +#endif + static int __init get_cpu_for_node(struct device_node *node) { struct device_node *cpu_node; @@ -34,6 +250,7 @@ static int __init get_cpu_for_node(struct device_node *node) for_each_possible_cpu(cpu) { if (of_get_cpu_node(cpu, NULL) == cpu_node) { + parse_cpu_capacity(cpu_node, cpu); of_node_put(cpu_node); return cpu; } @@ -178,13 +395,17 @@ static int __init parse_dt_topology(void) * cluster with restricted subnodes. */ map = of_get_child_by_name(cn, "cpu-map"); - if (!map) + if (!map) { + cap_parsing_failed = true; goto out; + } ret = parse_cluster(map, 0); if (ret != 0) goto out_map; + normalize_cpu_capacity(); + /* * Check that all cores are in the topology; the SMP code will * only mark cores described in the DT as possible. diff --git a/arch/arm64/kernel/traps.c b/arch/arm64/kernel/traps.c index c9986b3..5b830be 100644 --- a/arch/arm64/kernel/traps.c +++ b/arch/arm64/kernel/traps.c @@ -38,6 +38,7 @@ #include <asm/esr.h> #include <asm/insn.h> #include <asm/traps.h> +#include <asm/stack_pointer.h> #include <asm/stacktrace.h> #include <asm/exception.h> #include <asm/system_misc.h> @@ -147,6 +148,9 @@ static void dump_backtrace(struct pt_regs *regs, struct task_struct *tsk) if (!tsk) tsk = current; + if (!try_get_task_stack(tsk)) + return; + /* * Switching between stacks is valid when tracing current and in * non-preemptible context. @@ -212,6 +216,8 @@ static void dump_backtrace(struct pt_regs *regs, struct task_struct *tsk) stack + sizeof(struct pt_regs)); } } + + put_task_stack(tsk); } void show_stack(struct task_struct *tsk, unsigned long *sp) @@ -227,10 +233,9 @@ void show_stack(struct task_struct *tsk, unsigned long *sp) #endif #define S_SMP " SMP" -static int __die(const char *str, int err, struct thread_info *thread, - struct pt_regs *regs) +static int __die(const char *str, int err, struct pt_regs *regs) { - struct task_struct *tsk = thread->task; + struct task_struct *tsk = current; static int die_counter; int ret; @@ -245,7 +250,8 @@ static int __die(const char *str, int err, struct thread_info *thread, print_modules(); __show_regs(regs); pr_emerg("Process %.*s (pid: %d, stack limit = 0x%p)\n", - TASK_COMM_LEN, tsk->comm, task_pid_nr(tsk), thread + 1); + TASK_COMM_LEN, tsk->comm, task_pid_nr(tsk), + end_of_stack(tsk)); if (!user_mode(regs)) { dump_mem(KERN_EMERG, "Stack: ", regs->sp, @@ -264,7 +270,6 @@ static DEFINE_RAW_SPINLOCK(die_lock); */ void die(const char *str, struct pt_regs *regs, int err) { - struct thread_info *thread = current_thread_info(); int ret; oops_enter(); @@ -272,9 +277,9 @@ void die(const char *str, struct pt_regs *regs, int err) raw_spin_lock_irq(&die_lock); console_verbose(); bust_spinlocks(1); - ret = __die(str, err, thread, regs); + ret = __die(str, err, regs); - if (regs && kexec_should_crash(thread->task)) + if (regs && kexec_should_crash(current)) crash_kexec(regs); bust_spinlocks(0); @@ -435,9 +440,10 @@ int cpu_enable_cache_maint_trap(void *__unused) } #define __user_cache_maint(insn, address, res) \ - if (untagged_addr(address) >= user_addr_max()) \ + if (untagged_addr(address) >= user_addr_max()) { \ res = -EFAULT; \ - else \ + } else { \ + uaccess_ttbr0_enable(); \ asm volatile ( \ "1: " insn ", %1\n" \ " mov %w0, #0\n" \ @@ -449,7 +455,9 @@ int cpu_enable_cache_maint_trap(void *__unused) " .popsection\n" \ _ASM_EXTABLE(1b, 3b) \ : "=r" (res) \ - : "r" (address), "i" (-EFAULT) ) + : "r" (address), "i" (-EFAULT)); \ + uaccess_ttbr0_disable(); \ + } static void user_cache_maint_handler(unsigned int esr, struct pt_regs *regs) { diff --git a/arch/arm64/kernel/vmlinux.lds.S b/arch/arm64/kernel/vmlinux.lds.S index 1105aab..b8deffa 100644 --- a/arch/arm64/kernel/vmlinux.lds.S +++ b/arch/arm64/kernel/vmlinux.lds.S @@ -216,6 +216,11 @@ SECTIONS swapper_pg_dir = .; . += SWAPPER_DIR_SIZE; +#ifdef CONFIG_ARM64_SW_TTBR0_PAN + reserved_ttbr0 = .; + . += RESERVED_TTBR0_SIZE; +#endif + _end = .; STABS_DEBUG diff --git a/arch/arm64/kvm/Kconfig b/arch/arm64/kvm/Kconfig index 6eaf12c..52cb7ad 100644 --- a/arch/arm64/kvm/Kconfig +++ b/arch/arm64/kvm/Kconfig @@ -16,9 +16,6 @@ menuconfig VIRTUALIZATION if VIRTUALIZATION -config KVM_ARM_VGIC_V3_ITS - bool - config KVM bool "Kernel-based Virtual Machine (KVM) support" depends on OF @@ -34,7 +31,6 @@ config KVM select KVM_VFIO select HAVE_KVM_EVENTFD select HAVE_KVM_IRQFD - select KVM_ARM_VGIC_V3_ITS select KVM_ARM_PMU if HW_PERF_EVENTS select HAVE_KVM_MSI select HAVE_KVM_IRQCHIP diff --git a/arch/arm64/kvm/handle_exit.c b/arch/arm64/kvm/handle_exit.c index a204adf..1bfe30d 100644 --- a/arch/arm64/kvm/handle_exit.c +++ b/arch/arm64/kvm/handle_exit.c @@ -57,6 +57,16 @@ static int handle_smc(struct kvm_vcpu *vcpu, struct kvm_run *run) return 1; } +/* + * Guest access to FP/ASIMD registers are routed to this handler only + * when the system doesn't support FP/ASIMD. + */ +static int handle_no_fpsimd(struct kvm_vcpu *vcpu, struct kvm_run *run) +{ + kvm_inject_undefined(vcpu); + return 1; +} + /** * kvm_handle_wfx - handle a wait-for-interrupts or wait-for-event * instruction executed by a guest @@ -144,6 +154,7 @@ static exit_handle_fn arm_exit_handlers[] = { [ESR_ELx_EC_BREAKPT_LOW]= kvm_handle_guest_debug, [ESR_ELx_EC_BKPT32] = kvm_handle_guest_debug, [ESR_ELx_EC_BRK64] = kvm_handle_guest_debug, + [ESR_ELx_EC_FP_ASIMD] = handle_no_fpsimd, }; static exit_handle_fn kvm_get_exit_handler(struct kvm_vcpu *vcpu) diff --git a/arch/arm64/kvm/hyp/hyp-entry.S b/arch/arm64/kvm/hyp/hyp-entry.S index 4e92399..5e9052f 100644 --- a/arch/arm64/kvm/hyp/hyp-entry.S +++ b/arch/arm64/kvm/hyp/hyp-entry.S @@ -106,9 +106,16 @@ el1_trap: * x0: ESR_EC */ - /* Guest accessed VFP/SIMD registers, save host, restore Guest */ + /* + * We trap the first access to the FP/SIMD to save the host context + * and restore the guest context lazily. + * If FP/SIMD is not implemented, handle the trap and inject an + * undefined instruction exception to the guest. + */ +alternative_if_not ARM64_HAS_NO_FPSIMD cmp x0, #ESR_ELx_EC_FP_ASIMD b.eq __fpsimd_guest_restore +alternative_else_nop_endif mrs x1, tpidr_el2 mov x0, #ARM_EXCEPTION_TRAP diff --git a/arch/arm64/kvm/hyp/switch.c b/arch/arm64/kvm/hyp/switch.c index 83037cd..75e83dd 100644 --- a/arch/arm64/kvm/hyp/switch.c +++ b/arch/arm64/kvm/hyp/switch.c @@ -21,6 +21,7 @@ #include <asm/kvm_asm.h> #include <asm/kvm_emulate.h> #include <asm/kvm_hyp.h> +#include <asm/fpsimd.h> static bool __hyp_text __fpsimd_enabled_nvhe(void) { @@ -76,16 +77,24 @@ static void __hyp_text __activate_traps(struct kvm_vcpu *vcpu) * traps are only taken to EL2 if the operation would not otherwise * trap to EL1. Therefore, always make sure that for 32-bit guests, * we set FPEXC.EN to prevent traps to EL1, when setting the TFP bit. + * If FP/ASIMD is not implemented, FPEXC is UNDEFINED and any access to + * it will cause an exception. */ val = vcpu->arch.hcr_el2; - if (!(val & HCR_RW)) { + if (!(val & HCR_RW) && system_supports_fpsimd()) { write_sysreg(1 << 30, fpexc32_el2); isb(); } write_sysreg(val, hcr_el2); /* Trap on AArch32 cp15 c15 accesses (EL1 or EL0) */ write_sysreg(1 << 15, hstr_el2); - /* Make sure we trap PMU access from EL0 to EL2 */ + /* + * Make sure we trap PMU access from EL0 to EL2. Also sanitize + * PMSELR_EL0 to make sure it never contains the cycle + * counter, which could make a PMXEVCNTR_EL0 access UNDEF at + * EL1 instead of being trapped to EL2. + */ + write_sysreg(0, pmselr_el0); write_sysreg(ARMV8_PMU_USERENR_MASK, pmuserenr_el0); write_sysreg(vcpu->arch.mdcr_el2, mdcr_el2); __activate_traps_arch()(); diff --git a/arch/arm64/kvm/reset.c b/arch/arm64/kvm/reset.c index 5bc4608..e95d4f6 100644 --- a/arch/arm64/kvm/reset.c +++ b/arch/arm64/kvm/reset.c @@ -86,12 +86,6 @@ int kvm_arch_dev_ioctl_check_extension(struct kvm *kvm, long ext) case KVM_CAP_VCPU_ATTRIBUTES: r = 1; break; - case KVM_CAP_MSI_DEVID: - if (!kvm) - r = -EINVAL; - else - r = kvm->arch.vgic.msis_require_devid; - break; default: r = 0; } diff --git a/arch/arm64/lib/clear_user.S b/arch/arm64/lib/clear_user.S index 5d1cad3..d7150e3 100644 --- a/arch/arm64/lib/clear_user.S +++ b/arch/arm64/lib/clear_user.S @@ -17,10 +17,7 @@ */ #include <linux/linkage.h> -#include <asm/alternative.h> -#include <asm/assembler.h> -#include <asm/cpufeature.h> -#include <asm/sysreg.h> +#include <asm/uaccess.h> .text @@ -33,8 +30,7 @@ * Alignment fixed up by hardware. */ ENTRY(__clear_user) -ALTERNATIVE("nop", __stringify(SET_PSTATE_PAN(0)), ARM64_ALT_PAN_NOT_UAO, \ - CONFIG_ARM64_PAN) + uaccess_enable_not_uao x2, x3 mov x2, x1 // save the size for fixup return subs x1, x1, #8 b.mi 2f @@ -54,8 +50,7 @@ uao_user_alternative 9f, strh, sttrh, wzr, x0, 2 b.mi 5f uao_user_alternative 9f, strb, sttrb, wzr, x0, 0 5: mov x0, #0 -ALTERNATIVE("nop", __stringify(SET_PSTATE_PAN(1)), ARM64_ALT_PAN_NOT_UAO, \ - CONFIG_ARM64_PAN) + uaccess_disable_not_uao x2 ret ENDPROC(__clear_user) diff --git a/arch/arm64/lib/copy_from_user.S b/arch/arm64/lib/copy_from_user.S index 4fd67ea..cfe1339 100644 --- a/arch/arm64/lib/copy_from_user.S +++ b/arch/arm64/lib/copy_from_user.S @@ -16,11 +16,8 @@ #include <linux/linkage.h> -#include <asm/alternative.h> -#include <asm/assembler.h> #include <asm/cache.h> -#include <asm/cpufeature.h> -#include <asm/sysreg.h> +#include <asm/uaccess.h> /* * Copy from user space to a kernel buffer (alignment handled by the hardware) @@ -67,12 +64,10 @@ end .req x5 ENTRY(__arch_copy_from_user) -ALTERNATIVE("nop", __stringify(SET_PSTATE_PAN(0)), ARM64_ALT_PAN_NOT_UAO, \ - CONFIG_ARM64_PAN) + uaccess_enable_not_uao x3, x4 add end, x0, x2 #include "copy_template.S" -ALTERNATIVE("nop", __stringify(SET_PSTATE_PAN(1)), ARM64_ALT_PAN_NOT_UAO, \ - CONFIG_ARM64_PAN) + uaccess_disable_not_uao x3 mov x0, #0 // Nothing to copy ret ENDPROC(__arch_copy_from_user) diff --git a/arch/arm64/lib/copy_in_user.S b/arch/arm64/lib/copy_in_user.S index f7292dd0..718b1c4 100644 --- a/arch/arm64/lib/copy_in_user.S +++ b/arch/arm64/lib/copy_in_user.S @@ -18,11 +18,8 @@ #include <linux/linkage.h> -#include <asm/alternative.h> -#include <asm/assembler.h> #include <asm/cache.h> -#include <asm/cpufeature.h> -#include <asm/sysreg.h> +#include <asm/uaccess.h> /* * Copy from user space to user space (alignment handled by the hardware) @@ -68,12 +65,10 @@ end .req x5 ENTRY(__copy_in_user) -ALTERNATIVE("nop", __stringify(SET_PSTATE_PAN(0)), ARM64_ALT_PAN_NOT_UAO, \ - CONFIG_ARM64_PAN) + uaccess_enable_not_uao x3, x4 add end, x0, x2 #include "copy_template.S" -ALTERNATIVE("nop", __stringify(SET_PSTATE_PAN(1)), ARM64_ALT_PAN_NOT_UAO, \ - CONFIG_ARM64_PAN) + uaccess_disable_not_uao x3 mov x0, #0 ret ENDPROC(__copy_in_user) diff --git a/arch/arm64/lib/copy_to_user.S b/arch/arm64/lib/copy_to_user.S index 7a7efe2..e99e31c 100644 --- a/arch/arm64/lib/copy_to_user.S +++ b/arch/arm64/lib/copy_to_user.S @@ -16,11 +16,8 @@ #include <linux/linkage.h> -#include <asm/alternative.h> -#include <asm/assembler.h> #include <asm/cache.h> -#include <asm/cpufeature.h> -#include <asm/sysreg.h> +#include <asm/uaccess.h> /* * Copy to user space from a kernel buffer (alignment handled by the hardware) @@ -66,12 +63,10 @@ end .req x5 ENTRY(__arch_copy_to_user) -ALTERNATIVE("nop", __stringify(SET_PSTATE_PAN(0)), ARM64_ALT_PAN_NOT_UAO, \ - CONFIG_ARM64_PAN) + uaccess_enable_not_uao x3, x4 add end, x0, x2 #include "copy_template.S" -ALTERNATIVE("nop", __stringify(SET_PSTATE_PAN(1)), ARM64_ALT_PAN_NOT_UAO, \ - CONFIG_ARM64_PAN) + uaccess_disable_not_uao x3 mov x0, #0 ret ENDPROC(__arch_copy_to_user) diff --git a/arch/arm64/mm/Makefile b/arch/arm64/mm/Makefile index 54bb209..e703fb9 100644 --- a/arch/arm64/mm/Makefile +++ b/arch/arm64/mm/Makefile @@ -3,7 +3,8 @@ obj-y := dma-mapping.o extable.o fault.o init.o \ ioremap.o mmap.o pgd.o mmu.o \ context.o proc.o pageattr.o obj-$(CONFIG_HUGETLB_PAGE) += hugetlbpage.o -obj-$(CONFIG_ARM64_PTDUMP) += dump.o +obj-$(CONFIG_ARM64_PTDUMP_CORE) += dump.o +obj-$(CONFIG_ARM64_PTDUMP_DEBUGFS) += ptdump_debugfs.o obj-$(CONFIG_NUMA) += numa.o obj-$(CONFIG_KASAN) += kasan_init.o diff --git a/arch/arm64/mm/cache.S b/arch/arm64/mm/cache.S index 58b5a90..da95769 100644 --- a/arch/arm64/mm/cache.S +++ b/arch/arm64/mm/cache.S @@ -23,6 +23,7 @@ #include <asm/assembler.h> #include <asm/cpufeature.h> #include <asm/alternative.h> +#include <asm/uaccess.h> /* * flush_icache_range(start,end) @@ -48,6 +49,7 @@ ENTRY(flush_icache_range) * - end - virtual end address of region */ ENTRY(__flush_cache_user_range) + uaccess_ttbr0_enable x2, x3 dcache_line_size x2, x3 sub x3, x2, #1 bic x4, x0, x3 @@ -69,10 +71,12 @@ USER(9f, ic ivau, x4 ) // invalidate I line PoU dsb ish isb mov x0, #0 +1: + uaccess_ttbr0_disable x1 ret 9: mov x0, #-EFAULT - ret + b 1b ENDPROC(flush_icache_range) ENDPROC(__flush_cache_user_range) diff --git a/arch/arm64/mm/context.c b/arch/arm64/mm/context.c index efcf1f7..4c63cb1 100644 --- a/arch/arm64/mm/context.c +++ b/arch/arm64/mm/context.c @@ -221,7 +221,12 @@ void check_and_switch_context(struct mm_struct *mm, unsigned int cpu) raw_spin_unlock_irqrestore(&cpu_asid_lock, flags); switch_mm_fastpath: - cpu_switch_mm(mm->pgd, mm); + /* + * Defer TTBR0_EL1 setting for user threads to uaccess_enable() when + * emulating PAN. + */ + if (!system_uses_ttbr0_pan()) + cpu_switch_mm(mm->pgd, mm); } static int asids_init(void) diff --git a/arch/arm64/mm/dma-mapping.c b/arch/arm64/mm/dma-mapping.c index 3f74d0d..aa6c8f8 100644 --- a/arch/arm64/mm/dma-mapping.c +++ b/arch/arm64/mm/dma-mapping.c @@ -938,11 +938,6 @@ static void __iommu_setup_dma_ops(struct device *dev, u64 dma_base, u64 size, void arch_teardown_dma_ops(struct device *dev) { - struct iommu_domain *domain = iommu_get_domain_for_dev(dev); - - if (WARN_ON(domain)) - iommu_detach_device(domain, dev); - dev->archdata.dma_ops = NULL; } diff --git a/arch/arm64/mm/dump.c b/arch/arm64/mm/dump.c index 9c3e75d..ca74a2a 100644 --- a/arch/arm64/mm/dump.c +++ b/arch/arm64/mm/dump.c @@ -50,6 +50,18 @@ static const struct addr_marker address_markers[] = { { -1, NULL }, }; +#define pt_dump_seq_printf(m, fmt, args...) \ +({ \ + if (m) \ + seq_printf(m, fmt, ##args); \ +}) + +#define pt_dump_seq_puts(m, fmt) \ +({ \ + if (m) \ + seq_printf(m, fmt); \ +}) + /* * The page dumper groups page table entries of the same type into a single * description. It uses pg_state to track the range information while @@ -62,6 +74,9 @@ struct pg_state { unsigned long start_address; unsigned level; u64 current_prot; + bool check_wx; + unsigned long wx_pages; + unsigned long uxn_pages; }; struct prot_bits { @@ -186,10 +201,39 @@ static void dump_prot(struct pg_state *st, const struct prot_bits *bits, s = bits->clear; if (s) - seq_printf(st->seq, " %s", s); + pt_dump_seq_printf(st->seq, " %s", s); } } +static void note_prot_uxn(struct pg_state *st, unsigned long addr) +{ + if (!st->check_wx) + return; + + if ((st->current_prot & PTE_UXN) == PTE_UXN) + return; + + WARN_ONCE(1, "arm64/mm: Found non-UXN mapping at address %p/%pS\n", + (void *)st->start_address, (void *)st->start_address); + + st->uxn_pages += (addr - st->start_address) / PAGE_SIZE; +} + +static void note_prot_wx(struct pg_state *st, unsigned long addr) +{ + if (!st->check_wx) + return; + if ((st->current_prot & PTE_RDONLY) == PTE_RDONLY) + return; + if ((st->current_prot & PTE_PXN) == PTE_PXN) + return; + + WARN_ONCE(1, "arm64/mm: Found insecure W+X mapping at address %p/%pS\n", + (void *)st->start_address, (void *)st->start_address); + + st->wx_pages += (addr - st->start_address) / PAGE_SIZE; +} + static void note_page(struct pg_state *st, unsigned long addr, unsigned level, u64 val) { @@ -200,14 +244,16 @@ static void note_page(struct pg_state *st, unsigned long addr, unsigned level, st->level = level; st->current_prot = prot; st->start_address = addr; - seq_printf(st->seq, "---[ %s ]---\n", st->marker->name); + pt_dump_seq_printf(st->seq, "---[ %s ]---\n", st->marker->name); } else if (prot != st->current_prot || level != st->level || addr >= st->marker[1].start_address) { const char *unit = units; unsigned long delta; if (st->current_prot) { - seq_printf(st->seq, "0x%016lx-0x%016lx ", + note_prot_uxn(st, addr); + note_prot_wx(st, addr); + pt_dump_seq_printf(st->seq, "0x%016lx-0x%016lx ", st->start_address, addr); delta = (addr - st->start_address) >> 10; @@ -215,17 +261,17 @@ static void note_page(struct pg_state *st, unsigned long addr, unsigned level, delta >>= 10; unit++; } - seq_printf(st->seq, "%9lu%c %s", delta, *unit, + pt_dump_seq_printf(st->seq, "%9lu%c %s", delta, *unit, pg_level[st->level].name); if (pg_level[st->level].bits) dump_prot(st, pg_level[st->level].bits, pg_level[st->level].num); - seq_puts(st->seq, "\n"); + pt_dump_seq_puts(st->seq, "\n"); } if (addr >= st->marker[1].start_address) { st->marker++; - seq_printf(st->seq, "---[ %s ]---\n", st->marker->name); + pt_dump_seq_printf(st->seq, "---[ %s ]---\n", st->marker->name); } st->start_address = addr; @@ -235,7 +281,7 @@ static void note_page(struct pg_state *st, unsigned long addr, unsigned level, if (addr >= st->marker[1].start_address) { st->marker++; - seq_printf(st->seq, "---[ %s ]---\n", st->marker->name); + pt_dump_seq_printf(st->seq, "---[ %s ]---\n", st->marker->name); } } @@ -304,9 +350,8 @@ static void walk_pgd(struct pg_state *st, struct mm_struct *mm, } } -static int ptdump_show(struct seq_file *m, void *v) +void ptdump_walk_pgd(struct seq_file *m, struct ptdump_info *info) { - struct ptdump_info *info = m->private; struct pg_state st = { .seq = m, .marker = info->markers, @@ -315,33 +360,16 @@ static int ptdump_show(struct seq_file *m, void *v) walk_pgd(&st, info->mm, info->base_addr); note_page(&st, 0, 0, 0); - return 0; } -static int ptdump_open(struct inode *inode, struct file *file) +static void ptdump_initialize(void) { - return single_open(file, ptdump_show, inode->i_private); -} - -static const struct file_operations ptdump_fops = { - .open = ptdump_open, - .read = seq_read, - .llseek = seq_lseek, - .release = single_release, -}; - -int ptdump_register(struct ptdump_info *info, const char *name) -{ - struct dentry *pe; unsigned i, j; for (i = 0; i < ARRAY_SIZE(pg_level); i++) if (pg_level[i].bits) for (j = 0; j < pg_level[i].num; j++) pg_level[i].mask |= pg_level[i].bits[j].mask; - - pe = debugfs_create_file(name, 0400, NULL, info, &ptdump_fops); - return pe ? 0 : -ENOMEM; } static struct ptdump_info kernel_ptdump_info = { @@ -350,8 +378,30 @@ static struct ptdump_info kernel_ptdump_info = { .base_addr = VA_START, }; +void ptdump_check_wx(void) +{ + struct pg_state st = { + .seq = NULL, + .marker = (struct addr_marker[]) { + { 0, NULL}, + { -1, NULL}, + }, + .check_wx = true, + }; + + walk_pgd(&st, &init_mm, 0); + note_page(&st, 0, 0, 0); + if (st.wx_pages || st.uxn_pages) + pr_warn("Checked W+X mappings: FAILED, %lu W+X pages found, %lu non-UXN pages found\n", + st.wx_pages, st.uxn_pages); + else + pr_info("Checked W+X mappings: passed, no W+X pages found\n"); +} + static int ptdump_init(void) { - return ptdump_register(&kernel_ptdump_info, "kernel_page_tables"); + ptdump_initialize(); + return ptdump_debugfs_register(&kernel_ptdump_info, + "kernel_page_tables"); } device_initcall(ptdump_init); diff --git a/arch/arm64/mm/fault.c b/arch/arm64/mm/fault.c index 0f87883..a78a5c4 100644 --- a/arch/arm64/mm/fault.c +++ b/arch/arm64/mm/fault.c @@ -269,13 +269,19 @@ out: return fault; } -static inline bool is_permission_fault(unsigned int esr) +static inline bool is_permission_fault(unsigned int esr, struct pt_regs *regs) { unsigned int ec = ESR_ELx_EC(esr); unsigned int fsc_type = esr & ESR_ELx_FSC_TYPE; - return (ec == ESR_ELx_EC_DABT_CUR && fsc_type == ESR_ELx_FSC_PERM) || - (ec == ESR_ELx_EC_IABT_CUR && fsc_type == ESR_ELx_FSC_PERM); + if (ec != ESR_ELx_EC_DABT_CUR && ec != ESR_ELx_EC_IABT_CUR) + return false; + + if (system_uses_ttbr0_pan()) + return fsc_type == ESR_ELx_FSC_FAULT && + (regs->pstate & PSR_PAN_BIT); + else + return fsc_type == ESR_ELx_FSC_PERM; } static bool is_el0_instruction_abort(unsigned int esr) @@ -315,7 +321,7 @@ static int __kprobes do_page_fault(unsigned long addr, unsigned int esr, mm_flags |= FAULT_FLAG_WRITE; } - if (is_permission_fault(esr) && (addr < USER_DS)) { + if (addr < USER_DS && is_permission_fault(esr, regs)) { /* regs->orig_addr_limit may be 0 if we entered from EL0 */ if (regs->orig_addr_limit == KERNEL_DS) die("Accessing user space memory with fs=KERNEL_DS", regs, esr); @@ -507,10 +513,10 @@ static const struct fault_info { { do_bad, SIGBUS, 0, "unknown 17" }, { do_bad, SIGBUS, 0, "unknown 18" }, { do_bad, SIGBUS, 0, "unknown 19" }, - { do_bad, SIGBUS, 0, "synchronous abort (translation table walk)" }, - { do_bad, SIGBUS, 0, "synchronous abort (translation table walk)" }, - { do_bad, SIGBUS, 0, "synchronous abort (translation table walk)" }, - { do_bad, SIGBUS, 0, "synchronous abort (translation table walk)" }, + { do_bad, SIGBUS, 0, "synchronous external abort (translation table walk)" }, + { do_bad, SIGBUS, 0, "synchronous external abort (translation table walk)" }, + { do_bad, SIGBUS, 0, "synchronous external abort (translation table walk)" }, + { do_bad, SIGBUS, 0, "synchronous external abort (translation table walk)" }, { do_bad, SIGBUS, 0, "synchronous parity error" }, { do_bad, SIGBUS, 0, "unknown 25" }, { do_bad, SIGBUS, 0, "unknown 26" }, diff --git a/arch/arm64/mm/flush.c b/arch/arm64/mm/flush.c index 8377329..554a255 100644 --- a/arch/arm64/mm/flush.c +++ b/arch/arm64/mm/flush.c @@ -25,14 +25,7 @@ #include <asm/cachetype.h> #include <asm/tlbflush.h> -void flush_cache_range(struct vm_area_struct *vma, unsigned long start, - unsigned long end) -{ - if (vma->vm_flags & VM_EXEC) - __flush_icache_all(); -} - -static void sync_icache_aliases(void *kaddr, unsigned long len) +void sync_icache_aliases(void *kaddr, unsigned long len) { unsigned long addr = (unsigned long)kaddr; diff --git a/arch/arm64/mm/hugetlbpage.c b/arch/arm64/mm/hugetlbpage.c index 2e49bd2..964b754 100644 --- a/arch/arm64/mm/hugetlbpage.c +++ b/arch/arm64/mm/hugetlbpage.c @@ -51,20 +51,8 @@ static int find_num_contig(struct mm_struct *mm, unsigned long addr, *pgsize = PAGE_SIZE; if (!pte_cont(pte)) return 1; - if (!pgd_present(*pgd)) { - VM_BUG_ON(!pgd_present(*pgd)); - return 1; - } pud = pud_offset(pgd, addr); - if (!pud_present(*pud)) { - VM_BUG_ON(!pud_present(*pud)); - return 1; - } pmd = pmd_offset(pud, addr); - if (!pmd_present(*pmd)) { - VM_BUG_ON(!pmd_present(*pmd)); - return 1; - } if ((pte_t *)pmd == ptep) { *pgsize = PMD_SIZE; return CONT_PMDS; @@ -212,7 +200,7 @@ pte_t huge_ptep_get_and_clear(struct mm_struct *mm, ncontig = find_num_contig(mm, addr, cpte, *cpte, &pgsize); /* save the 1st pte to return */ pte = ptep_get_and_clear(mm, addr, cpte); - for (i = 1; i < ncontig; ++i) { + for (i = 1, addr += pgsize; i < ncontig; ++i, addr += pgsize) { /* * If HW_AFDBM is enabled, then the HW could * turn on the dirty bit for any of the page @@ -250,7 +238,7 @@ int huge_ptep_set_access_flags(struct vm_area_struct *vma, pfn = pte_pfn(*cpte); ncontig = find_num_contig(vma->vm_mm, addr, cpte, *cpte, &pgsize); - for (i = 0; i < ncontig; ++i, ++cpte) { + for (i = 0; i < ncontig; ++i, ++cpte, addr += pgsize) { changed = ptep_set_access_flags(vma, addr, cpte, pfn_pte(pfn, hugeprot), @@ -273,7 +261,7 @@ void huge_ptep_set_wrprotect(struct mm_struct *mm, cpte = huge_pte_offset(mm, addr); ncontig = find_num_contig(mm, addr, cpte, *cpte, &pgsize); - for (i = 0; i < ncontig; ++i, ++cpte) + for (i = 0; i < ncontig; ++i, ++cpte, addr += pgsize) ptep_set_wrprotect(mm, addr, cpte); } else { ptep_set_wrprotect(mm, addr, ptep); @@ -291,7 +279,7 @@ void huge_ptep_clear_flush(struct vm_area_struct *vma, cpte = huge_pte_offset(vma->vm_mm, addr); ncontig = find_num_contig(vma->vm_mm, addr, cpte, *cpte, &pgsize); - for (i = 0; i < ncontig; ++i, ++cpte) + for (i = 0; i < ncontig; ++i, ++cpte, addr += pgsize) ptep_clear_flush(vma, addr, cpte); } else { ptep_clear_flush(vma, addr, ptep); @@ -323,7 +311,7 @@ __setup("hugepagesz=", setup_hugepagesz); static __init int add_default_hugepagesz(void) { if (size_to_hstate(CONT_PTES * PAGE_SIZE) == NULL) - hugetlb_add_hstate(CONT_PMD_SHIFT); + hugetlb_add_hstate(CONT_PTE_SHIFT); return 0; } arch_initcall(add_default_hugepagesz); diff --git a/arch/arm64/mm/mmu.c b/arch/arm64/mm/mmu.c index 05615a3..17243e4 100644 --- a/arch/arm64/mm/mmu.c +++ b/arch/arm64/mm/mmu.c @@ -28,8 +28,6 @@ #include <linux/memblock.h> #include <linux/fs.h> #include <linux/io.h> -#include <linux/slab.h> -#include <linux/stop_machine.h> #include <asm/barrier.h> #include <asm/cputype.h> @@ -42,6 +40,7 @@ #include <asm/tlb.h> #include <asm/memblock.h> #include <asm/mmu_context.h> +#include <asm/ptdump.h> u64 idmap_t0sz = TCR_T0SZ(VA_BITS); @@ -95,11 +94,24 @@ static phys_addr_t __init early_pgtable_alloc(void) return phys; } +static bool pgattr_change_is_safe(u64 old, u64 new) +{ + /* + * The following mapping attributes may be updated in live + * kernel mappings without the need for break-before-make. + */ + static const pteval_t mask = PTE_PXN | PTE_RDONLY | PTE_WRITE; + + return old == 0 || new == 0 || ((old ^ new) & ~mask) == 0; +} + static void alloc_init_pte(pmd_t *pmd, unsigned long addr, unsigned long end, unsigned long pfn, pgprot_t prot, - phys_addr_t (*pgtable_alloc)(void)) + phys_addr_t (*pgtable_alloc)(void), + bool page_mappings_only) { + pgprot_t __prot = prot; pte_t *pte; BUG_ON(pmd_sect(*pmd)); @@ -115,8 +127,28 @@ static void alloc_init_pte(pmd_t *pmd, unsigned long addr, pte = pte_set_fixmap_offset(pmd, addr); do { - set_pte(pte, pfn_pte(pfn, prot)); + pte_t old_pte = *pte; + + /* + * Set the contiguous bit for the subsequent group of PTEs if + * its size and alignment are appropriate. + */ + if (((addr | PFN_PHYS(pfn)) & ~CONT_PTE_MASK) == 0) { + if (end - addr >= CONT_PTE_SIZE && !page_mappings_only) + __prot = __pgprot(pgprot_val(prot) | PTE_CONT); + else + __prot = prot; + } + + set_pte(pte, pfn_pte(pfn, __prot)); pfn++; + + /* + * After the PTE entry has been populated once, we + * only allow updates to the permission attributes. + */ + BUG_ON(!pgattr_change_is_safe(pte_val(old_pte), pte_val(*pte))); + } while (pte++, addr += PAGE_SIZE, addr != end); pte_clear_fixmap(); @@ -125,8 +157,9 @@ static void alloc_init_pte(pmd_t *pmd, unsigned long addr, static void alloc_init_pmd(pud_t *pud, unsigned long addr, unsigned long end, phys_addr_t phys, pgprot_t prot, phys_addr_t (*pgtable_alloc)(void), - bool allow_block_mappings) + bool page_mappings_only) { + pgprot_t __prot = prot; pmd_t *pmd; unsigned long next; @@ -146,27 +179,39 @@ static void alloc_init_pmd(pud_t *pud, unsigned long addr, unsigned long end, pmd = pmd_set_fixmap_offset(pud, addr); do { + pmd_t old_pmd = *pmd; + next = pmd_addr_end(addr, end); + /* try section mapping first */ if (((addr | next | phys) & ~SECTION_MASK) == 0 && - allow_block_mappings) { - pmd_t old_pmd =*pmd; - pmd_set_huge(pmd, phys, prot); + !page_mappings_only) { /* - * Check for previous table entries created during - * boot (__create_page_tables) and flush them. + * Set the contiguous bit for the subsequent group of + * PMDs if its size and alignment are appropriate. */ - if (!pmd_none(old_pmd)) { - flush_tlb_all(); - if (pmd_table(old_pmd)) { - phys_addr_t table = pmd_page_paddr(old_pmd); - if (!WARN_ON_ONCE(slab_is_available())) - memblock_free(table, PAGE_SIZE); - } + if (((addr | phys) & ~CONT_PMD_MASK) == 0) { + if (end - addr >= CONT_PMD_SIZE) + __prot = __pgprot(pgprot_val(prot) | + PTE_CONT); + else + __prot = prot; } + pmd_set_huge(pmd, phys, __prot); + + /* + * After the PMD entry has been populated once, we + * only allow updates to the permission attributes. + */ + BUG_ON(!pgattr_change_is_safe(pmd_val(old_pmd), + pmd_val(*pmd))); } else { alloc_init_pte(pmd, addr, next, __phys_to_pfn(phys), - prot, pgtable_alloc); + prot, pgtable_alloc, + page_mappings_only); + + BUG_ON(pmd_val(old_pmd) != 0 && + pmd_val(old_pmd) != pmd_val(*pmd)); } phys += next - addr; } while (pmd++, addr = next, addr != end); @@ -189,7 +234,7 @@ static inline bool use_1G_block(unsigned long addr, unsigned long next, static void alloc_init_pud(pgd_t *pgd, unsigned long addr, unsigned long end, phys_addr_t phys, pgprot_t prot, phys_addr_t (*pgtable_alloc)(void), - bool allow_block_mappings) + bool page_mappings_only) { pud_t *pud; unsigned long next; @@ -204,33 +249,28 @@ static void alloc_init_pud(pgd_t *pgd, unsigned long addr, unsigned long end, pud = pud_set_fixmap_offset(pgd, addr); do { + pud_t old_pud = *pud; + next = pud_addr_end(addr, end); /* * For 4K granule only, attempt to put down a 1GB block */ - if (use_1G_block(addr, next, phys) && allow_block_mappings) { - pud_t old_pud = *pud; + if (use_1G_block(addr, next, phys) && !page_mappings_only) { pud_set_huge(pud, phys, prot); /* - * If we have an old value for a pud, it will - * be pointing to a pmd table that we no longer - * need (from swapper_pg_dir). - * - * Look up the old pmd table and free it. + * After the PUD entry has been populated once, we + * only allow updates to the permission attributes. */ - if (!pud_none(old_pud)) { - flush_tlb_all(); - if (pud_table(old_pud)) { - phys_addr_t table = pud_page_paddr(old_pud); - if (!WARN_ON_ONCE(slab_is_available())) - memblock_free(table, PAGE_SIZE); - } - } + BUG_ON(!pgattr_change_is_safe(pud_val(old_pud), + pud_val(*pud))); } else { alloc_init_pmd(pud, addr, next, phys, prot, - pgtable_alloc, allow_block_mappings); + pgtable_alloc, page_mappings_only); + + BUG_ON(pud_val(old_pud) != 0 && + pud_val(old_pud) != pud_val(*pud)); } phys += next - addr; } while (pud++, addr = next, addr != end); @@ -242,7 +282,7 @@ static void __create_pgd_mapping(pgd_t *pgdir, phys_addr_t phys, unsigned long virt, phys_addr_t size, pgprot_t prot, phys_addr_t (*pgtable_alloc)(void), - bool allow_block_mappings) + bool page_mappings_only) { unsigned long addr, length, end, next; pgd_t *pgd = pgd_offset_raw(pgdir, virt); @@ -262,7 +302,7 @@ static void __create_pgd_mapping(pgd_t *pgdir, phys_addr_t phys, do { next = pgd_addr_end(addr, end); alloc_init_pud(pgd, addr, next, phys, prot, pgtable_alloc, - allow_block_mappings); + page_mappings_only); phys += next - addr; } while (pgd++, addr = next, addr != end); } @@ -291,17 +331,17 @@ static void __init create_mapping_noalloc(phys_addr_t phys, unsigned long virt, &phys, virt); return; } - __create_pgd_mapping(init_mm.pgd, phys, virt, size, prot, NULL, true); + __create_pgd_mapping(init_mm.pgd, phys, virt, size, prot, NULL, false); } void __init create_pgd_mapping(struct mm_struct *mm, phys_addr_t phys, unsigned long virt, phys_addr_t size, - pgprot_t prot, bool allow_block_mappings) + pgprot_t prot, bool page_mappings_only) { BUG_ON(mm == &init_mm); __create_pgd_mapping(mm->pgd, phys, virt, size, prot, - pgd_pgtable_alloc, allow_block_mappings); + pgd_pgtable_alloc, page_mappings_only); } static void create_mapping_late(phys_addr_t phys, unsigned long virt, @@ -314,7 +354,7 @@ static void create_mapping_late(phys_addr_t phys, unsigned long virt, } __create_pgd_mapping(init_mm.pgd, phys, virt, size, prot, - NULL, !debug_pagealloc_enabled()); + NULL, debug_pagealloc_enabled()); } static void __init __map_memblock(pgd_t *pgd, phys_addr_t start, phys_addr_t end) @@ -332,7 +372,7 @@ static void __init __map_memblock(pgd_t *pgd, phys_addr_t start, phys_addr_t end __create_pgd_mapping(pgd, start, __phys_to_virt(start), end - start, PAGE_KERNEL, early_pgtable_alloc, - !debug_pagealloc_enabled()); + debug_pagealloc_enabled()); return; } @@ -345,13 +385,13 @@ static void __init __map_memblock(pgd_t *pgd, phys_addr_t start, phys_addr_t end __phys_to_virt(start), kernel_start - start, PAGE_KERNEL, early_pgtable_alloc, - !debug_pagealloc_enabled()); + debug_pagealloc_enabled()); if (kernel_end < end) __create_pgd_mapping(pgd, kernel_end, __phys_to_virt(kernel_end), end - kernel_end, PAGE_KERNEL, early_pgtable_alloc, - !debug_pagealloc_enabled()); + debug_pagealloc_enabled()); /* * Map the linear alias of the [_text, __init_begin) interval as @@ -361,7 +401,7 @@ static void __init __map_memblock(pgd_t *pgd, phys_addr_t start, phys_addr_t end */ __create_pgd_mapping(pgd, kernel_start, __phys_to_virt(kernel_start), kernel_end - kernel_start, PAGE_KERNEL_RO, - early_pgtable_alloc, !debug_pagealloc_enabled()); + early_pgtable_alloc, debug_pagealloc_enabled()); } static void __init map_mem(pgd_t *pgd) @@ -396,6 +436,11 @@ void mark_rodata_ro(void) section_size = (unsigned long)__init_begin - (unsigned long)__start_rodata; create_mapping_late(__pa(__start_rodata), (unsigned long)__start_rodata, section_size, PAGE_KERNEL_RO); + + /* flush the TLBs after updating live kernel mappings */ + flush_tlb_all(); + + debug_checkwx(); } static void __init map_kernel_segment(pgd_t *pgd, void *va_start, void *va_end, @@ -408,7 +453,7 @@ static void __init map_kernel_segment(pgd_t *pgd, void *va_start, void *va_end, BUG_ON(!PAGE_ALIGNED(size)); __create_pgd_mapping(pgd, pa_start, (unsigned long)va_start, size, prot, - early_pgtable_alloc, !debug_pagealloc_enabled()); + early_pgtable_alloc, debug_pagealloc_enabled()); vma->addr = va_start; vma->phys_addr = pa_start; diff --git a/arch/arm64/mm/proc.S b/arch/arm64/mm/proc.S index 352c73b..32682be 100644 --- a/arch/arm64/mm/proc.S +++ b/arch/arm64/mm/proc.S @@ -70,11 +70,14 @@ ENTRY(cpu_do_suspend) mrs x8, mdscr_el1 mrs x9, oslsr_el1 mrs x10, sctlr_el1 + mrs x11, tpidr_el1 + mrs x12, sp_el0 stp x2, x3, [x0] stp x4, xzr, [x0, #16] stp x5, x6, [x0, #32] stp x7, x8, [x0, #48] stp x9, x10, [x0, #64] + stp x11, x12, [x0, #80] ret ENDPROC(cpu_do_suspend) @@ -90,6 +93,7 @@ ENTRY(cpu_do_resume) ldp x6, x8, [x0, #32] ldp x9, x10, [x0, #48] ldp x11, x12, [x0, #64] + ldp x13, x14, [x0, #80] msr tpidr_el0, x2 msr tpidrro_el0, x3 msr contextidr_el1, x4 @@ -112,6 +116,8 @@ ENTRY(cpu_do_resume) msr mdscr_el1, x10 msr sctlr_el1, x12 + msr tpidr_el1, x13 + msr sp_el0, x14 /* * Restore oslsr_el1 by writing oslar_el1 */ @@ -136,11 +142,7 @@ ENTRY(cpu_do_switch_mm) bfi x0, x1, #48, #16 // set the ASID msr ttbr0_el1, x0 // set TTBR0 isb -alternative_if ARM64_WORKAROUND_CAVIUM_27456 - ic iallu - dsb nsh - isb -alternative_else_nop_endif + post_ttbr0_update_workaround ret ENDPROC(cpu_do_switch_mm) diff --git a/arch/arm64/mm/ptdump_debugfs.c b/arch/arm64/mm/ptdump_debugfs.c new file mode 100644 index 0000000..eee4d86 --- /dev/null +++ b/arch/arm64/mm/ptdump_debugfs.c @@ -0,0 +1,31 @@ +#include <linux/debugfs.h> +#include <linux/seq_file.h> + +#include <asm/ptdump.h> + +static int ptdump_show(struct seq_file *m, void *v) +{ + struct ptdump_info *info = m->private; + ptdump_walk_pgd(m, info); + return 0; +} + +static int ptdump_open(struct inode *inode, struct file *file) +{ + return single_open(file, ptdump_show, inode->i_private); +} + +static const struct file_operations ptdump_fops = { + .open = ptdump_open, + .read = seq_read, + .llseek = seq_lseek, + .release = single_release, +}; + +int ptdump_debugfs_register(struct ptdump_info *info, const char *name) +{ + struct dentry *pe; + pe = debugfs_create_file(name, 0400, NULL, info, &ptdump_fops); + return pe ? 0 : -ENOMEM; + +} diff --git a/arch/arm64/xen/hypercall.S b/arch/arm64/xen/hypercall.S index 329c802..b41aff2 100644 --- a/arch/arm64/xen/hypercall.S +++ b/arch/arm64/xen/hypercall.S @@ -49,6 +49,7 @@ #include <linux/linkage.h> #include <asm/assembler.h> +#include <asm/uaccess.h> #include <xen/interface/xen.h> @@ -91,6 +92,20 @@ ENTRY(privcmd_call) mov x2, x3 mov x3, x4 mov x4, x5 + /* + * Privcmd calls are issued by the userspace. The kernel needs to + * enable access to TTBR0_EL1 as the hypervisor would issue stage 1 + * translations to user memory via AT instructions. Since AT + * instructions are not affected by the PAN bit (ARMv8.1), we only + * need the explicit uaccess_enable/disable if the TTBR0 PAN emulation + * is enabled (it implies that hardware UAO and PAN disabled). + */ + uaccess_ttbr0_enable x6, x7 hvc XEN_IMM + + /* + * Disable userspace access from kernel once the hyp call completed. + */ + uaccess_ttbr0_disable x6 ret ENDPROC(privcmd_call); diff --git a/arch/powerpc/include/asm/book3s/64/mmu-hash.h b/arch/powerpc/include/asm/book3s/64/mmu-hash.h index e407af2..2e6a823 100644 --- a/arch/powerpc/include/asm/book3s/64/mmu-hash.h +++ b/arch/powerpc/include/asm/book3s/64/mmu-hash.h @@ -70,7 +70,9 @@ #define HPTE_V_SSIZE_SHIFT 62 #define HPTE_V_AVPN_SHIFT 7 +#define HPTE_V_COMMON_BITS ASM_CONST(0x000fffffffffffff) #define HPTE_V_AVPN ASM_CONST(0x3fffffffffffff80) +#define HPTE_V_AVPN_3_0 ASM_CONST(0x000fffffffffff80) #define HPTE_V_AVPN_VAL(x) (((x) & HPTE_V_AVPN) >> HPTE_V_AVPN_SHIFT) #define HPTE_V_COMPARE(x,y) (!(((x) ^ (y)) & 0xffffffffffffff80UL)) #define HPTE_V_BOLTED ASM_CONST(0x0000000000000010) @@ -80,14 +82,16 @@ #define HPTE_V_VALID ASM_CONST(0x0000000000000001) /* - * ISA 3.0 have a different HPTE format. + * ISA 3.0 has a different HPTE format. */ #define HPTE_R_3_0_SSIZE_SHIFT 58 +#define HPTE_R_3_0_SSIZE_MASK (3ull << HPTE_R_3_0_SSIZE_SHIFT) #define HPTE_R_PP0 ASM_CONST(0x8000000000000000) #define HPTE_R_TS ASM_CONST(0x4000000000000000) #define HPTE_R_KEY_HI ASM_CONST(0x3000000000000000) #define HPTE_R_RPN_SHIFT 12 #define HPTE_R_RPN ASM_CONST(0x0ffffffffffff000) +#define HPTE_R_RPN_3_0 ASM_CONST(0x01fffffffffff000) #define HPTE_R_PP ASM_CONST(0x0000000000000003) #define HPTE_R_PPP ASM_CONST(0x8000000000000003) #define HPTE_R_N ASM_CONST(0x0000000000000004) @@ -316,12 +320,43 @@ static inline unsigned long hpte_encode_avpn(unsigned long vpn, int psize, */ v = (vpn >> (23 - VPN_SHIFT)) & ~(mmu_psize_defs[psize].avpnm); v <<= HPTE_V_AVPN_SHIFT; - if (!cpu_has_feature(CPU_FTR_ARCH_300)) - v |= ((unsigned long) ssize) << HPTE_V_SSIZE_SHIFT; + v |= ((unsigned long) ssize) << HPTE_V_SSIZE_SHIFT; return v; } /* + * ISA v3.0 defines a new HPTE format, which differs from the old + * format in having smaller AVPN and ARPN fields, and the B field + * in the second dword instead of the first. + */ +static inline unsigned long hpte_old_to_new_v(unsigned long v) +{ + /* trim AVPN, drop B */ + return v & HPTE_V_COMMON_BITS; +} + +static inline unsigned long hpte_old_to_new_r(unsigned long v, unsigned long r) +{ + /* move B field from 1st to 2nd dword, trim ARPN */ + return (r & ~HPTE_R_3_0_SSIZE_MASK) | + (((v) >> HPTE_V_SSIZE_SHIFT) << HPTE_R_3_0_SSIZE_SHIFT); +} + +static inline unsigned long hpte_new_to_old_v(unsigned long v, unsigned long r) +{ + /* insert B field */ + return (v & HPTE_V_COMMON_BITS) | + ((r & HPTE_R_3_0_SSIZE_MASK) << + (HPTE_V_SSIZE_SHIFT - HPTE_R_3_0_SSIZE_SHIFT)); +} + +static inline unsigned long hpte_new_to_old_r(unsigned long r) +{ + /* clear out B field */ + return r & ~HPTE_R_3_0_SSIZE_MASK; +} + +/* * This function sets the AVPN and L fields of the HPTE appropriately * using the base page size and actual page size. */ @@ -341,12 +376,8 @@ static inline unsigned long hpte_encode_v(unsigned long vpn, int base_psize, * aligned for the requested page size */ static inline unsigned long hpte_encode_r(unsigned long pa, int base_psize, - int actual_psize, int ssize) + int actual_psize) { - - if (cpu_has_feature(CPU_FTR_ARCH_300)) - pa |= ((unsigned long) ssize) << HPTE_R_3_0_SSIZE_SHIFT; - /* A 4K page needs no special encoding */ if (actual_psize == MMU_PAGE_4K) return pa & HPTE_R_RPN; diff --git a/arch/powerpc/include/asm/kvm_asm.h b/arch/powerpc/include/asm/kvm_asm.h index 05cabed..09a802b 100644 --- a/arch/powerpc/include/asm/kvm_asm.h +++ b/arch/powerpc/include/asm/kvm_asm.h @@ -99,6 +99,7 @@ #define BOOK3S_INTERRUPT_H_EMUL_ASSIST 0xe40 #define BOOK3S_INTERRUPT_HMI 0xe60 #define BOOK3S_INTERRUPT_H_DOORBELL 0xe80 +#define BOOK3S_INTERRUPT_H_VIRT 0xea0 #define BOOK3S_INTERRUPT_PERFMON 0xf00 #define BOOK3S_INTERRUPT_ALTIVEC 0xf20 #define BOOK3S_INTERRUPT_VSX 0xf40 diff --git a/arch/powerpc/include/asm/kvm_host.h b/arch/powerpc/include/asm/kvm_host.h index 28350a2..e59b172 100644 --- a/arch/powerpc/include/asm/kvm_host.h +++ b/arch/powerpc/include/asm/kvm_host.h @@ -48,7 +48,7 @@ #ifdef CONFIG_KVM_MMIO #define KVM_COALESCED_MMIO_PAGE_OFFSET 1 #endif -#define KVM_HALT_POLL_NS_DEFAULT 500000 +#define KVM_HALT_POLL_NS_DEFAULT 10000 /* 10 us */ /* These values are internal and can be increased later */ #define KVM_NR_IRQCHIPS 1 @@ -244,8 +244,10 @@ struct kvm_arch_memory_slot { struct kvm_arch { unsigned int lpid; #ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE + unsigned int tlb_sets; unsigned long hpt_virt; struct revmap_entry *revmap; + atomic64_t mmio_update; unsigned int host_lpid; unsigned long host_lpcr; unsigned long sdr1; @@ -408,6 +410,24 @@ struct kvmppc_passthru_irqmap { #define KVMPPC_IRQ_MPIC 1 #define KVMPPC_IRQ_XICS 2 +#define MMIO_HPTE_CACHE_SIZE 4 + +struct mmio_hpte_cache_entry { + unsigned long hpte_v; + unsigned long hpte_r; + unsigned long rpte; + unsigned long pte_index; + unsigned long eaddr; + unsigned long slb_v; + long mmio_update; + unsigned int slb_base_pshift; +}; + +struct mmio_hpte_cache { + struct mmio_hpte_cache_entry entry[MMIO_HPTE_CACHE_SIZE]; + unsigned int index; +}; + struct openpic; struct kvm_vcpu_arch { @@ -498,6 +518,8 @@ struct kvm_vcpu_arch { ulong tcscr; ulong acop; ulong wort; + ulong tid; + ulong psscr; ulong shadow_srr1; #endif u32 vrsave; /* also USPRG0 */ @@ -546,6 +568,7 @@ struct kvm_vcpu_arch { u64 tfiar; u32 cr_tm; + u64 xer_tm; u64 lr_tm; u64 ctr_tm; u64 amr_tm; @@ -655,9 +678,11 @@ struct kvm_vcpu_arch { #ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE struct kvm_vcpu_arch_shared shregs; + struct mmio_hpte_cache mmio_cache; unsigned long pgfault_addr; long pgfault_index; unsigned long pgfault_hpte[2]; + struct mmio_hpte_cache_entry *pgfault_cache; struct task_struct *run_task; struct kvm_run *kvm_run; diff --git a/arch/powerpc/include/asm/kvm_ppc.h b/arch/powerpc/include/asm/kvm_ppc.h index f6e4964..2da67bf 100644 --- a/arch/powerpc/include/asm/kvm_ppc.h +++ b/arch/powerpc/include/asm/kvm_ppc.h @@ -483,9 +483,10 @@ extern void kvmppc_xics_set_mapped(struct kvm *kvm, unsigned long guest_irq, unsigned long host_irq); extern void kvmppc_xics_clr_mapped(struct kvm *kvm, unsigned long guest_irq, unsigned long host_irq); -extern long kvmppc_deliver_irq_passthru(struct kvm_vcpu *vcpu, u32 xirr, - struct kvmppc_irq_map *irq_map, - struct kvmppc_passthru_irqmap *pimap); +extern long kvmppc_deliver_irq_passthru(struct kvm_vcpu *vcpu, __be32 xirr, + struct kvmppc_irq_map *irq_map, + struct kvmppc_passthru_irqmap *pimap, + bool *again); extern int h_ipi_redirect; #else static inline struct kvmppc_passthru_irqmap *kvmppc_get_passthru_irqmap( @@ -510,6 +511,48 @@ static inline int kvmppc_xics_hcall(struct kvm_vcpu *vcpu, u32 cmd) #endif /* + * Prototypes for functions called only from assembler code. + * Having prototypes reduces sparse errors. + */ +long kvmppc_rm_h_put_tce(struct kvm_vcpu *vcpu, unsigned long liobn, + unsigned long ioba, unsigned long tce); +long kvmppc_rm_h_put_tce_indirect(struct kvm_vcpu *vcpu, + unsigned long liobn, unsigned long ioba, + unsigned long tce_list, unsigned long npages); +long kvmppc_rm_h_stuff_tce(struct kvm_vcpu *vcpu, + unsigned long liobn, unsigned long ioba, + unsigned long tce_value, unsigned long npages); +long int kvmppc_rm_h_confer(struct kvm_vcpu *vcpu, int target, + unsigned int yield_count); +long kvmppc_h_random(struct kvm_vcpu *vcpu); +void kvmhv_commence_exit(int trap); +long kvmppc_realmode_machine_check(struct kvm_vcpu *vcpu); +void kvmppc_subcore_enter_guest(void); +void kvmppc_subcore_exit_guest(void); +long kvmppc_realmode_hmi_handler(void); +long kvmppc_h_enter(struct kvm_vcpu *vcpu, unsigned long flags, + long pte_index, unsigned long pteh, unsigned long ptel); +long kvmppc_h_remove(struct kvm_vcpu *vcpu, unsigned long flags, + unsigned long pte_index, unsigned long avpn); +long kvmppc_h_bulk_remove(struct kvm_vcpu *vcpu); +long kvmppc_h_protect(struct kvm_vcpu *vcpu, unsigned long flags, + unsigned long pte_index, unsigned long avpn, + unsigned long va); +long kvmppc_h_read(struct kvm_vcpu *vcpu, unsigned long flags, + unsigned long pte_index); +long kvmppc_h_clear_ref(struct kvm_vcpu *vcpu, unsigned long flags, + unsigned long pte_index); +long kvmppc_h_clear_mod(struct kvm_vcpu *vcpu, unsigned long flags, + unsigned long pte_index); +long kvmppc_hpte_hv_fault(struct kvm_vcpu *vcpu, unsigned long addr, + unsigned long slb_v, unsigned int status, bool data); +unsigned long kvmppc_rm_h_xirr(struct kvm_vcpu *vcpu); +int kvmppc_rm_h_ipi(struct kvm_vcpu *vcpu, unsigned long server, + unsigned long mfrr); +int kvmppc_rm_h_cppr(struct kvm_vcpu *vcpu, unsigned long cppr); +int kvmppc_rm_h_eoi(struct kvm_vcpu *vcpu, unsigned long xirr); + +/* * Host-side operations we want to set up while running in real * mode in the guest operating on the xics. * Currently only VCPU wakeup is supported. diff --git a/arch/powerpc/include/asm/mmu.h b/arch/powerpc/include/asm/mmu.h index e311c25..8d14993 100644 --- a/arch/powerpc/include/asm/mmu.h +++ b/arch/powerpc/include/asm/mmu.h @@ -214,6 +214,11 @@ extern u64 ppc64_rma_size; /* Cleanup function used by kexec */ extern void mmu_cleanup_all(void); extern void radix__mmu_cleanup_all(void); + +/* Functions for creating and updating partition table on POWER9 */ +extern void mmu_partition_table_init(void); +extern void mmu_partition_table_set_entry(unsigned int lpid, unsigned long dw0, + unsigned long dw1); #endif /* CONFIG_PPC64 */ struct mm_struct; diff --git a/arch/powerpc/include/asm/opal.h b/arch/powerpc/include/asm/opal.h index e958b70..5c7db0f 100644 --- a/arch/powerpc/include/asm/opal.h +++ b/arch/powerpc/include/asm/opal.h @@ -220,9 +220,12 @@ int64_t opal_pci_set_power_state(uint64_t async_token, uint64_t id, int64_t opal_pci_poll2(uint64_t id, uint64_t data); int64_t opal_int_get_xirr(uint32_t *out_xirr, bool just_poll); +int64_t opal_rm_int_get_xirr(__be32 *out_xirr, bool just_poll); int64_t opal_int_set_cppr(uint8_t cppr); int64_t opal_int_eoi(uint32_t xirr); +int64_t opal_rm_int_eoi(uint32_t xirr); int64_t opal_int_set_mfrr(uint32_t cpu, uint8_t mfrr); +int64_t opal_rm_int_set_mfrr(uint32_t cpu, uint8_t mfrr); int64_t opal_pci_tce_kill(uint64_t phb_id, uint32_t kill_type, uint32_t pe_num, uint32_t tce_size, uint64_t dma_addr, uint32_t npages); diff --git a/arch/powerpc/include/asm/reg.h b/arch/powerpc/include/asm/reg.h index 9e1499f..04aa1ee 100644 --- a/arch/powerpc/include/asm/reg.h +++ b/arch/powerpc/include/asm/reg.h @@ -153,6 +153,8 @@ #define PSSCR_EC 0x00100000 /* Exit Criterion */ #define PSSCR_ESL 0x00200000 /* Enable State Loss */ #define PSSCR_SD 0x00400000 /* Status Disable */ +#define PSSCR_PLS 0xf000000000000000 /* Power-saving Level Status */ +#define PSSCR_GUEST_VIS 0xf0000000000003ff /* Guest-visible PSSCR fields */ /* Floating Point Status and Control Register (FPSCR) Fields */ #define FPSCR_FX 0x80000000 /* FPU exception summary */ @@ -236,6 +238,7 @@ #define SPRN_TEXASRU 0x83 /* '' '' '' Upper 32 */ #define TEXASR_FS __MASK(63-36) /* TEXASR Failure Summary */ #define SPRN_TFHAR 0x80 /* Transaction Failure Handler Addr */ +#define SPRN_TIDR 144 /* Thread ID register */ #define SPRN_CTRLF 0x088 #define SPRN_CTRLT 0x098 #define CTRL_CT 0xc0000000 /* current thread */ @@ -294,6 +297,7 @@ #define SPRN_HSRR1 0x13B /* Hypervisor Save/Restore 1 */ #define SPRN_LMRR 0x32D /* Load Monitor Region Register */ #define SPRN_LMSER 0x32E /* Load Monitor Section Enable Register */ +#define SPRN_ASDR 0x330 /* Access segment descriptor register */ #define SPRN_IC 0x350 /* Virtual Instruction Count */ #define SPRN_VTB 0x351 /* Virtual Time Base */ #define SPRN_LDBAR 0x352 /* LD Base Address Register */ @@ -305,6 +309,7 @@ /* HFSCR and FSCR bit numbers are the same */ #define FSCR_LM_LG 11 /* Enable Load Monitor Registers */ +#define FSCR_MSGP_LG 10 /* Enable MSGP */ #define FSCR_TAR_LG 8 /* Enable Target Address Register */ #define FSCR_EBB_LG 7 /* Enable Event Based Branching */ #define FSCR_TM_LG 5 /* Enable Transactional Memory */ @@ -320,6 +325,7 @@ #define FSCR_DSCR __MASK(FSCR_DSCR_LG) #define SPRN_HFSCR 0xbe /* HV=1 Facility Status & Control Register */ #define HFSCR_LM __MASK(FSCR_LM_LG) +#define HFSCR_MSGP __MASK(FSCR_MSGP_LG) #define HFSCR_TAR __MASK(FSCR_TAR_LG) #define HFSCR_EBB __MASK(FSCR_EBB_LG) #define HFSCR_TM __MASK(FSCR_TM_LG) @@ -358,6 +364,7 @@ #define LPCR_PECE_HVEE ASM_CONST(0x0000400000000000) /* P9 Wakeup on HV interrupts */ #define LPCR_MER ASM_CONST(0x0000000000000800) /* Mediated External Exception */ #define LPCR_MER_SH 11 +#define LPCR_GTSE ASM_CONST(0x0000000000000400) /* Guest Translation Shootdown Enable */ #define LPCR_TC ASM_CONST(0x0000000000000200) /* Translation control */ #define LPCR_LPES 0x0000000c #define LPCR_LPES0 ASM_CONST(0x0000000000000008) /* LPAR Env selector 0 */ @@ -378,6 +385,12 @@ #define PCR_VEC_DIS (1ul << (63-0)) /* Vec. disable (bit NA since POWER8) */ #define PCR_VSX_DIS (1ul << (63-1)) /* VSX disable (bit NA since POWER8) */ #define PCR_TM_DIS (1ul << (63-2)) /* Trans. memory disable (POWER8) */ +/* + * These bits are used in the function kvmppc_set_arch_compat() to specify and + * determine both the compatibility level which we want to emulate and the + * compatibility level which the host is capable of emulating. + */ +#define PCR_ARCH_207 0x8 /* Architecture 2.07 */ #define PCR_ARCH_206 0x4 /* Architecture 2.06 */ #define PCR_ARCH_205 0x2 /* Architecture 2.05 */ #define SPRN_HEIR 0x153 /* Hypervisor Emulated Instruction Register */ @@ -1219,6 +1232,7 @@ #define PVR_ARCH_206 0x0f000003 #define PVR_ARCH_206p 0x0f100003 #define PVR_ARCH_207 0x0f000004 +#define PVR_ARCH_300 0x0f000005 /* Macros for setting and retrieving special purpose registers */ #ifndef __ASSEMBLY__ diff --git a/arch/powerpc/include/uapi/asm/kvm.h b/arch/powerpc/include/uapi/asm/kvm.h index c93cf35..3603b6f 100644 --- a/arch/powerpc/include/uapi/asm/kvm.h +++ b/arch/powerpc/include/uapi/asm/kvm.h @@ -573,6 +573,10 @@ struct kvm_get_htab_header { #define KVM_REG_PPC_SPRG9 (KVM_REG_PPC | KVM_REG_SIZE_U64 | 0xba) #define KVM_REG_PPC_DBSR (KVM_REG_PPC | KVM_REG_SIZE_U32 | 0xbb) +/* POWER9 registers */ +#define KVM_REG_PPC_TIDR (KVM_REG_PPC | KVM_REG_SIZE_U64 | 0xbc) +#define KVM_REG_PPC_PSSCR (KVM_REG_PPC | KVM_REG_SIZE_U64 | 0xbd) + /* Transactional Memory checkpointed state: * This is all GPRs, all VSX regs and a subset of SPRs */ @@ -596,6 +600,7 @@ struct kvm_get_htab_header { #define KVM_REG_PPC_TM_VSCR (KVM_REG_PPC_TM | KVM_REG_SIZE_U32 | 0x67) #define KVM_REG_PPC_TM_DSCR (KVM_REG_PPC_TM | KVM_REG_SIZE_U64 | 0x68) #define KVM_REG_PPC_TM_TAR (KVM_REG_PPC_TM | KVM_REG_SIZE_U64 | 0x69) +#define KVM_REG_PPC_TM_XER (KVM_REG_PPC_TM | KVM_REG_SIZE_U64 | 0x6a) /* PPC64 eXternal Interrupt Controller Specification */ #define KVM_DEV_XICS_GRP_SOURCES 1 /* 64-bit source attributes */ diff --git a/arch/powerpc/kernel/asm-offsets.c b/arch/powerpc/kernel/asm-offsets.c index caec7bf..195a9fc 100644 --- a/arch/powerpc/kernel/asm-offsets.c +++ b/arch/powerpc/kernel/asm-offsets.c @@ -487,6 +487,7 @@ int main(void) /* book3s */ #ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE + DEFINE(KVM_TLB_SETS, offsetof(struct kvm, arch.tlb_sets)); DEFINE(KVM_SDR1, offsetof(struct kvm, arch.sdr1)); DEFINE(KVM_HOST_LPID, offsetof(struct kvm, arch.host_lpid)); DEFINE(KVM_HOST_LPCR, offsetof(struct kvm, arch.host_lpcr)); @@ -548,6 +549,8 @@ int main(void) DEFINE(VCPU_TCSCR, offsetof(struct kvm_vcpu, arch.tcscr)); DEFINE(VCPU_ACOP, offsetof(struct kvm_vcpu, arch.acop)); DEFINE(VCPU_WORT, offsetof(struct kvm_vcpu, arch.wort)); + DEFINE(VCPU_TID, offsetof(struct kvm_vcpu, arch.tid)); + DEFINE(VCPU_PSSCR, offsetof(struct kvm_vcpu, arch.psscr)); DEFINE(VCORE_ENTRY_EXIT, offsetof(struct kvmppc_vcore, entry_exit_map)); DEFINE(VCORE_IN_GUEST, offsetof(struct kvmppc_vcore, in_guest)); DEFINE(VCORE_NAPPING_THREADS, offsetof(struct kvmppc_vcore, napping_threads)); @@ -569,6 +572,7 @@ int main(void) DEFINE(VCPU_VRS_TM, offsetof(struct kvm_vcpu, arch.vr_tm.vr)); DEFINE(VCPU_VRSAVE_TM, offsetof(struct kvm_vcpu, arch.vrsave_tm)); DEFINE(VCPU_CR_TM, offsetof(struct kvm_vcpu, arch.cr_tm)); + DEFINE(VCPU_XER_TM, offsetof(struct kvm_vcpu, arch.xer_tm)); DEFINE(VCPU_LR_TM, offsetof(struct kvm_vcpu, arch.lr_tm)); DEFINE(VCPU_CTR_TM, offsetof(struct kvm_vcpu, arch.ctr_tm)); DEFINE(VCPU_AMR_TM, offsetof(struct kvm_vcpu, arch.amr_tm)); diff --git a/arch/powerpc/kernel/cpu_setup_power.S b/arch/powerpc/kernel/cpu_setup_power.S index 37c027c..f3e1f5d 100644 --- a/arch/powerpc/kernel/cpu_setup_power.S +++ b/arch/powerpc/kernel/cpu_setup_power.S @@ -174,7 +174,7 @@ __init_FSCR: __init_HFSCR: mfspr r3,SPRN_HFSCR ori r3,r3,HFSCR_TAR|HFSCR_TM|HFSCR_BHRB|HFSCR_PM|\ - HFSCR_DSCR|HFSCR_VECVSX|HFSCR_FP|HFSCR_EBB + HFSCR_DSCR|HFSCR_VECVSX|HFSCR_FP|HFSCR_EBB|HFSCR_MSGP mtspr SPRN_HFSCR,r3 blr diff --git a/arch/powerpc/kvm/book3s_64_mmu_hv.c b/arch/powerpc/kvm/book3s_64_mmu_hv.c index 05f09ae..b795dd1 100644 --- a/arch/powerpc/kvm/book3s_64_mmu_hv.c +++ b/arch/powerpc/kvm/book3s_64_mmu_hv.c @@ -88,6 +88,8 @@ long kvmppc_alloc_hpt(struct kvm *kvm, u32 *htab_orderp) /* 128 (2**7) bytes in each HPTEG */ kvm->arch.hpt_mask = (1ul << (order - 7)) - 1; + atomic64_set(&kvm->arch.mmio_update, 0); + /* Allocate reverse map array */ rev = vmalloc(sizeof(struct revmap_entry) * kvm->arch.hpt_npte); if (!rev) { @@ -255,7 +257,7 @@ static void kvmppc_mmu_book3s_64_hv_reset_msr(struct kvm_vcpu *vcpu) kvmppc_set_msr(vcpu, msr); } -long kvmppc_virtmode_do_h_enter(struct kvm *kvm, unsigned long flags, +static long kvmppc_virtmode_do_h_enter(struct kvm *kvm, unsigned long flags, long pte_index, unsigned long pteh, unsigned long ptel, unsigned long *pte_idx_ret) { @@ -312,7 +314,7 @@ static int kvmppc_mmu_book3s_64_hv_xlate(struct kvm_vcpu *vcpu, gva_t eaddr, struct kvmppc_slb *slbe; unsigned long slb_v; unsigned long pp, key; - unsigned long v, gr; + unsigned long v, orig_v, gr; __be64 *hptep; int index; int virtmode = vcpu->arch.shregs.msr & (data ? MSR_DR : MSR_IR); @@ -337,10 +339,12 @@ static int kvmppc_mmu_book3s_64_hv_xlate(struct kvm_vcpu *vcpu, gva_t eaddr, return -ENOENT; } hptep = (__be64 *)(kvm->arch.hpt_virt + (index << 4)); - v = be64_to_cpu(hptep[0]) & ~HPTE_V_HVLOCK; + v = orig_v = be64_to_cpu(hptep[0]) & ~HPTE_V_HVLOCK; + if (cpu_has_feature(CPU_FTR_ARCH_300)) + v = hpte_new_to_old_v(v, be64_to_cpu(hptep[1])); gr = kvm->arch.revmap[index].guest_rpte; - unlock_hpte(hptep, v); + unlock_hpte(hptep, orig_v); preempt_enable(); gpte->eaddr = eaddr; @@ -438,6 +442,7 @@ int kvmppc_book3s_hv_page_fault(struct kvm_run *run, struct kvm_vcpu *vcpu, { struct kvm *kvm = vcpu->kvm; unsigned long hpte[3], r; + unsigned long hnow_v, hnow_r; __be64 *hptep; unsigned long mmu_seq, psize, pte_size; unsigned long gpa_base, gfn_base; @@ -451,6 +456,7 @@ int kvmppc_book3s_hv_page_fault(struct kvm_run *run, struct kvm_vcpu *vcpu, unsigned int writing, write_ok; struct vm_area_struct *vma; unsigned long rcbits; + long mmio_update; /* * Real-mode code has already searched the HPT and found the @@ -460,6 +466,19 @@ int kvmppc_book3s_hv_page_fault(struct kvm_run *run, struct kvm_vcpu *vcpu, */ if (ea != vcpu->arch.pgfault_addr) return RESUME_GUEST; + + if (vcpu->arch.pgfault_cache) { + mmio_update = atomic64_read(&kvm->arch.mmio_update); + if (mmio_update == vcpu->arch.pgfault_cache->mmio_update) { + r = vcpu->arch.pgfault_cache->rpte; + psize = hpte_page_size(vcpu->arch.pgfault_hpte[0], r); + gpa_base = r & HPTE_R_RPN & ~(psize - 1); + gfn_base = gpa_base >> PAGE_SHIFT; + gpa = gpa_base | (ea & (psize - 1)); + return kvmppc_hv_emulate_mmio(run, vcpu, gpa, ea, + dsisr & DSISR_ISSTORE); + } + } index = vcpu->arch.pgfault_index; hptep = (__be64 *)(kvm->arch.hpt_virt + (index << 4)); rev = &kvm->arch.revmap[index]; @@ -472,6 +491,10 @@ int kvmppc_book3s_hv_page_fault(struct kvm_run *run, struct kvm_vcpu *vcpu, unlock_hpte(hptep, hpte[0]); preempt_enable(); + if (cpu_has_feature(CPU_FTR_ARCH_300)) { + hpte[0] = hpte_new_to_old_v(hpte[0], hpte[1]); + hpte[1] = hpte_new_to_old_r(hpte[1]); + } if (hpte[0] != vcpu->arch.pgfault_hpte[0] || hpte[1] != vcpu->arch.pgfault_hpte[1]) return RESUME_GUEST; @@ -575,16 +598,22 @@ int kvmppc_book3s_hv_page_fault(struct kvm_run *run, struct kvm_vcpu *vcpu, */ if (psize < PAGE_SIZE) psize = PAGE_SIZE; - r = (r & ~(HPTE_R_PP0 - psize)) | ((pfn << PAGE_SHIFT) & ~(psize - 1)); + r = (r & HPTE_R_KEY_HI) | (r & ~(HPTE_R_PP0 - psize)) | + ((pfn << PAGE_SHIFT) & ~(psize - 1)); if (hpte_is_writable(r) && !write_ok) r = hpte_make_readonly(r); ret = RESUME_GUEST; preempt_disable(); while (!try_lock_hpte(hptep, HPTE_V_HVLOCK)) cpu_relax(); - if ((be64_to_cpu(hptep[0]) & ~HPTE_V_HVLOCK) != hpte[0] || - be64_to_cpu(hptep[1]) != hpte[1] || - rev->guest_rpte != hpte[2]) + hnow_v = be64_to_cpu(hptep[0]); + hnow_r = be64_to_cpu(hptep[1]); + if (cpu_has_feature(CPU_FTR_ARCH_300)) { + hnow_v = hpte_new_to_old_v(hnow_v, hnow_r); + hnow_r = hpte_new_to_old_r(hnow_r); + } + if ((hnow_v & ~HPTE_V_HVLOCK) != hpte[0] || hnow_r != hpte[1] || + rev->guest_rpte != hpte[2]) /* HPTE has been changed under us; let the guest retry */ goto out_unlock; hpte[0] = (hpte[0] & ~HPTE_V_ABSENT) | HPTE_V_VALID; @@ -615,6 +644,10 @@ int kvmppc_book3s_hv_page_fault(struct kvm_run *run, struct kvm_vcpu *vcpu, kvmppc_add_revmap_chain(kvm, rev, rmap, index, 0); } + if (cpu_has_feature(CPU_FTR_ARCH_300)) { + r = hpte_old_to_new_r(hpte[0], r); + hpte[0] = hpte_old_to_new_v(hpte[0]); + } hptep[1] = cpu_to_be64(r); eieio(); __unlock_hpte(hptep, hpte[0]); @@ -758,6 +791,7 @@ static int kvm_unmap_rmapp(struct kvm *kvm, unsigned long *rmapp, hpte_rpn(ptel, psize) == gfn) { hptep[0] |= cpu_to_be64(HPTE_V_ABSENT); kvmppc_invalidate_hpte(kvm, hptep, i); + hptep[1] &= ~cpu_to_be64(HPTE_R_KEY_HI | HPTE_R_KEY_LO); /* Harvest R and C */ rcbits = be64_to_cpu(hptep[1]) & (HPTE_R_R | HPTE_R_C); *rmapp |= rcbits << KVMPPC_RMAP_RC_SHIFT; @@ -1165,7 +1199,7 @@ static long record_hpte(unsigned long flags, __be64 *hptp, unsigned long *hpte, struct revmap_entry *revp, int want_valid, int first_pass) { - unsigned long v, r; + unsigned long v, r, hr; unsigned long rcbits_unset; int ok = 1; int valid, dirty; @@ -1192,6 +1226,11 @@ static long record_hpte(unsigned long flags, __be64 *hptp, while (!try_lock_hpte(hptp, HPTE_V_HVLOCK)) cpu_relax(); v = be64_to_cpu(hptp[0]); + hr = be64_to_cpu(hptp[1]); + if (cpu_has_feature(CPU_FTR_ARCH_300)) { + v = hpte_new_to_old_v(v, hr); + hr = hpte_new_to_old_r(hr); + } /* re-evaluate valid and dirty from synchronized HPTE value */ valid = !!(v & HPTE_V_VALID); @@ -1199,8 +1238,8 @@ static long record_hpte(unsigned long flags, __be64 *hptp, /* Harvest R and C into guest view if necessary */ rcbits_unset = ~revp->guest_rpte & (HPTE_R_R | HPTE_R_C); - if (valid && (rcbits_unset & be64_to_cpu(hptp[1]))) { - revp->guest_rpte |= (be64_to_cpu(hptp[1]) & + if (valid && (rcbits_unset & hr)) { + revp->guest_rpte |= (hr & (HPTE_R_R | HPTE_R_C)) | HPTE_GR_MODIFIED; dirty = 1; } @@ -1608,7 +1647,7 @@ static ssize_t debugfs_htab_read(struct file *file, char __user *buf, return ret; } -ssize_t debugfs_htab_write(struct file *file, const char __user *buf, +static ssize_t debugfs_htab_write(struct file *file, const char __user *buf, size_t len, loff_t *ppos) { return -EACCES; diff --git a/arch/powerpc/kvm/book3s_64_vio_hv.c b/arch/powerpc/kvm/book3s_64_vio_hv.c index d461c44..e4c4ea9 100644 --- a/arch/powerpc/kvm/book3s_64_vio_hv.c +++ b/arch/powerpc/kvm/book3s_64_vio_hv.c @@ -39,7 +39,6 @@ #include <asm/udbg.h> #include <asm/iommu.h> #include <asm/tce.h> -#include <asm/iommu.h> #define TCES_PER_PAGE (PAGE_SIZE / sizeof(u64)) diff --git a/arch/powerpc/kvm/book3s_hv.c b/arch/powerpc/kvm/book3s_hv.c index 39ef1f4..8dcbe37 100644 --- a/arch/powerpc/kvm/book3s_hv.c +++ b/arch/powerpc/kvm/book3s_hv.c @@ -54,6 +54,9 @@ #include <asm/dbell.h> #include <asm/hmi.h> #include <asm/pnv-pci.h> +#include <asm/mmu.h> +#include <asm/opal.h> +#include <asm/xics.h> #include <linux/gfp.h> #include <linux/vmalloc.h> #include <linux/highmem.h> @@ -62,6 +65,7 @@ #include <linux/irqbypass.h> #include <linux/module.h> #include <linux/compiler.h> +#include <linux/of.h> #include "book3s.h" @@ -104,23 +108,6 @@ module_param_cb(h_ipi_redirect, &module_param_ops, &h_ipi_redirect, MODULE_PARM_DESC(h_ipi_redirect, "Redirect H_IPI wakeup to a free host core"); #endif -/* Maximum halt poll interval defaults to KVM_HALT_POLL_NS_DEFAULT */ -static unsigned int halt_poll_max_ns = KVM_HALT_POLL_NS_DEFAULT; -module_param(halt_poll_max_ns, uint, S_IRUGO | S_IWUSR); -MODULE_PARM_DESC(halt_poll_max_ns, "Maximum halt poll time in ns"); - -/* Factor by which the vcore halt poll interval is grown, default is to double - */ -static unsigned int halt_poll_ns_grow = 2; -module_param(halt_poll_ns_grow, int, S_IRUGO); -MODULE_PARM_DESC(halt_poll_ns_grow, "Factor halt poll time is grown by"); - -/* Factor by which the vcore halt poll interval is shrunk, default is to reset - */ -static unsigned int halt_poll_ns_shrink; -module_param(halt_poll_ns_shrink, int, S_IRUGO); -MODULE_PARM_DESC(halt_poll_ns_shrink, "Factor halt poll time is shrunk by"); - static void kvmppc_end_cede(struct kvm_vcpu *vcpu); static int kvmppc_hv_setup_htab_rma(struct kvm_vcpu *vcpu); @@ -146,12 +133,21 @@ static inline struct kvm_vcpu *next_runnable_thread(struct kvmppc_vcore *vc, static bool kvmppc_ipi_thread(int cpu) { + unsigned long msg = PPC_DBELL_TYPE(PPC_DBELL_SERVER); + + /* On POWER9 we can use msgsnd to IPI any cpu */ + if (cpu_has_feature(CPU_FTR_ARCH_300)) { + msg |= get_hard_smp_processor_id(cpu); + smp_mb(); + __asm__ __volatile__ (PPC_MSGSND(%0) : : "r" (msg)); + return true; + } + /* On POWER8 for IPIs to threads in the same core, use msgsnd */ if (cpu_has_feature(CPU_FTR_ARCH_207S)) { preempt_disable(); if (cpu_first_thread_sibling(cpu) == cpu_first_thread_sibling(smp_processor_id())) { - unsigned long msg = PPC_DBELL_TYPE(PPC_DBELL_SERVER); msg |= cpu_thread_in_core(cpu); smp_mb(); __asm__ __volatile__ (PPC_MSGSND(%0) : : "r" (msg)); @@ -162,8 +158,12 @@ static bool kvmppc_ipi_thread(int cpu) } #if defined(CONFIG_PPC_ICP_NATIVE) && defined(CONFIG_SMP) - if (cpu >= 0 && cpu < nr_cpu_ids && paca[cpu].kvm_hstate.xics_phys) { - xics_wake_cpu(cpu); + if (cpu >= 0 && cpu < nr_cpu_ids) { + if (paca[cpu].kvm_hstate.xics_phys) { + xics_wake_cpu(cpu); + return true; + } + opal_int_set_mfrr(get_hard_smp_processor_id(cpu), IPI_PRIORITY); return true; } #endif @@ -299,41 +299,54 @@ static void kvmppc_set_pvr_hv(struct kvm_vcpu *vcpu, u32 pvr) vcpu->arch.pvr = pvr; } +/* Dummy value used in computing PCR value below */ +#define PCR_ARCH_300 (PCR_ARCH_207 << 1) + static int kvmppc_set_arch_compat(struct kvm_vcpu *vcpu, u32 arch_compat) { - unsigned long pcr = 0; + unsigned long host_pcr_bit = 0, guest_pcr_bit = 0; struct kvmppc_vcore *vc = vcpu->arch.vcore; + /* We can (emulate) our own architecture version and anything older */ + if (cpu_has_feature(CPU_FTR_ARCH_300)) + host_pcr_bit = PCR_ARCH_300; + else if (cpu_has_feature(CPU_FTR_ARCH_207S)) + host_pcr_bit = PCR_ARCH_207; + else if (cpu_has_feature(CPU_FTR_ARCH_206)) + host_pcr_bit = PCR_ARCH_206; + else + host_pcr_bit = PCR_ARCH_205; + + /* Determine lowest PCR bit needed to run guest in given PVR level */ + guest_pcr_bit = host_pcr_bit; if (arch_compat) { switch (arch_compat) { case PVR_ARCH_205: - /* - * If an arch bit is set in PCR, all the defined - * higher-order arch bits also have to be set. - */ - pcr = PCR_ARCH_206 | PCR_ARCH_205; + guest_pcr_bit = PCR_ARCH_205; break; case PVR_ARCH_206: case PVR_ARCH_206p: - pcr = PCR_ARCH_206; + guest_pcr_bit = PCR_ARCH_206; break; case PVR_ARCH_207: + guest_pcr_bit = PCR_ARCH_207; + break; + case PVR_ARCH_300: + guest_pcr_bit = PCR_ARCH_300; break; default: return -EINVAL; } - - if (!cpu_has_feature(CPU_FTR_ARCH_207S)) { - /* POWER7 can't emulate POWER8 */ - if (!(pcr & PCR_ARCH_206)) - return -EINVAL; - pcr &= ~PCR_ARCH_206; - } } + /* Check requested PCR bits don't exceed our capabilities */ + if (guest_pcr_bit > host_pcr_bit) + return -EINVAL; + spin_lock(&vc->lock); vc->arch_compat = arch_compat; - vc->pcr = pcr; + /* Set all PCR bits for which guest_pcr_bit <= bit < host_pcr_bit */ + vc->pcr = host_pcr_bit - guest_pcr_bit; spin_unlock(&vc->lock); return 0; @@ -945,6 +958,7 @@ static int kvmppc_handle_exit_hv(struct kvm_run *run, struct kvm_vcpu *vcpu, break; case BOOK3S_INTERRUPT_EXTERNAL: case BOOK3S_INTERRUPT_H_DOORBELL: + case BOOK3S_INTERRUPT_H_VIRT: vcpu->stat.ext_intr_exits++; r = RESUME_GUEST; break; @@ -1229,6 +1243,12 @@ static int kvmppc_get_one_reg_hv(struct kvm_vcpu *vcpu, u64 id, case KVM_REG_PPC_WORT: *val = get_reg_val(id, vcpu->arch.wort); break; + case KVM_REG_PPC_TIDR: + *val = get_reg_val(id, vcpu->arch.tid); + break; + case KVM_REG_PPC_PSSCR: + *val = get_reg_val(id, vcpu->arch.psscr); + break; case KVM_REG_PPC_VPA_ADDR: spin_lock(&vcpu->arch.vpa_update_lock); *val = get_reg_val(id, vcpu->arch.vpa.next_gpa); @@ -1288,6 +1308,9 @@ static int kvmppc_get_one_reg_hv(struct kvm_vcpu *vcpu, u64 id, case KVM_REG_PPC_TM_CR: *val = get_reg_val(id, vcpu->arch.cr_tm); break; + case KVM_REG_PPC_TM_XER: + *val = get_reg_val(id, vcpu->arch.xer_tm); + break; case KVM_REG_PPC_TM_LR: *val = get_reg_val(id, vcpu->arch.lr_tm); break; @@ -1427,6 +1450,12 @@ static int kvmppc_set_one_reg_hv(struct kvm_vcpu *vcpu, u64 id, case KVM_REG_PPC_WORT: vcpu->arch.wort = set_reg_val(id, *val); break; + case KVM_REG_PPC_TIDR: + vcpu->arch.tid = set_reg_val(id, *val); + break; + case KVM_REG_PPC_PSSCR: + vcpu->arch.psscr = set_reg_val(id, *val) & PSSCR_GUEST_VIS; + break; case KVM_REG_PPC_VPA_ADDR: addr = set_reg_val(id, *val); r = -EINVAL; @@ -1498,6 +1527,9 @@ static int kvmppc_set_one_reg_hv(struct kvm_vcpu *vcpu, u64 id, case KVM_REG_PPC_TM_CR: vcpu->arch.cr_tm = set_reg_val(id, *val); break; + case KVM_REG_PPC_TM_XER: + vcpu->arch.xer_tm = set_reg_val(id, *val); + break; case KVM_REG_PPC_TM_LR: vcpu->arch.lr_tm = set_reg_val(id, *val); break; @@ -1540,6 +1572,20 @@ static int kvmppc_set_one_reg_hv(struct kvm_vcpu *vcpu, u64 id, return r; } +/* + * On POWER9, threads are independent and can be in different partitions. + * Therefore we consider each thread to be a subcore. + * There is a restriction that all threads have to be in the same + * MMU mode (radix or HPT), unfortunately, but since we only support + * HPT guests on a HPT host so far, that isn't an impediment yet. + */ +static int threads_per_vcore(void) +{ + if (cpu_has_feature(CPU_FTR_ARCH_300)) + return 1; + return threads_per_subcore; +} + static struct kvmppc_vcore *kvmppc_vcore_create(struct kvm *kvm, int core) { struct kvmppc_vcore *vcore; @@ -1554,7 +1600,7 @@ static struct kvmppc_vcore *kvmppc_vcore_create(struct kvm *kvm, int core) init_swait_queue_head(&vcore->wq); vcore->preempt_tb = TB_NIL; vcore->lpcr = kvm->arch.lpcr; - vcore->first_vcpuid = core * threads_per_subcore; + vcore->first_vcpuid = core * threads_per_vcore(); vcore->kvm = kvm; INIT_LIST_HEAD(&vcore->preempt_list); @@ -1717,7 +1763,7 @@ static struct kvm_vcpu *kvmppc_core_vcpu_create_hv(struct kvm *kvm, int core; struct kvmppc_vcore *vcore; - core = id / threads_per_subcore; + core = id / threads_per_vcore(); if (core >= KVM_MAX_VCORES) goto out; @@ -1935,7 +1981,10 @@ static void kvmppc_wait_for_nap(void) { int cpu = smp_processor_id(); int i, loops; + int n_threads = threads_per_vcore(); + if (n_threads <= 1) + return; for (loops = 0; loops < 1000000; ++loops) { /* * Check if all threads are finished. @@ -1943,17 +1992,17 @@ static void kvmppc_wait_for_nap(void) * and the thread clears it when finished, so we look * for any threads that still have a non-NULL vcore ptr. */ - for (i = 1; i < threads_per_subcore; ++i) + for (i = 1; i < n_threads; ++i) if (paca[cpu + i].kvm_hstate.kvm_vcore) break; - if (i == threads_per_subcore) { + if (i == n_threads) { HMT_medium(); return; } HMT_low(); } HMT_medium(); - for (i = 1; i < threads_per_subcore; ++i) + for (i = 1; i < n_threads; ++i) if (paca[cpu + i].kvm_hstate.kvm_vcore) pr_err("KVM: CPU %d seems to be stuck\n", cpu + i); } @@ -2019,7 +2068,7 @@ static void kvmppc_vcore_preempt(struct kvmppc_vcore *vc) vc->vcore_state = VCORE_PREEMPT; vc->pcpu = smp_processor_id(); - if (vc->num_threads < threads_per_subcore) { + if (vc->num_threads < threads_per_vcore()) { spin_lock(&lp->lock); list_add_tail(&vc->preempt_list, &lp->list); spin_unlock(&lp->lock); @@ -2123,8 +2172,7 @@ static bool can_dynamic_split(struct kvmppc_vcore *vc, struct core_info *cip) cip->subcore_threads[sub] = vc->num_threads; cip->subcore_vm[sub] = vc->kvm; init_master_vcore(vc); - list_del(&vc->preempt_list); - list_add_tail(&vc->preempt_list, &cip->vcs[sub]); + list_move_tail(&vc->preempt_list, &cip->vcs[sub]); return true; } @@ -2309,6 +2357,7 @@ static noinline void kvmppc_run_core(struct kvmppc_vcore *vc) unsigned long cmd_bit, stat_bit; int pcpu, thr; int target_threads; + int controlled_threads; /* * Remove from the list any threads that have a signal pending @@ -2327,11 +2376,18 @@ static noinline void kvmppc_run_core(struct kvmppc_vcore *vc) vc->preempt_tb = TB_NIL; /* + * Number of threads that we will be controlling: the same as + * the number of threads per subcore, except on POWER9, + * where it's 1 because the threads are (mostly) independent. + */ + controlled_threads = threads_per_vcore(); + + /* * Make sure we are running on primary threads, and that secondary * threads are offline. Also check if the number of threads in this * guest are greater than the current system threads per guest. */ - if ((threads_per_core > 1) && + if ((controlled_threads > 1) && ((vc->num_threads > threads_per_subcore) || !on_primary_thread())) { for_each_runnable_thread(i, vcpu, vc) { vcpu->arch.ret = -EBUSY; @@ -2347,7 +2403,7 @@ static noinline void kvmppc_run_core(struct kvmppc_vcore *vc) */ init_core_info(&core_info, vc); pcpu = smp_processor_id(); - target_threads = threads_per_subcore; + target_threads = controlled_threads; if (target_smt_mode && target_smt_mode < target_threads) target_threads = target_smt_mode; if (vc->num_threads < target_threads) @@ -2383,7 +2439,7 @@ static noinline void kvmppc_run_core(struct kvmppc_vcore *vc) smp_wmb(); } pcpu = smp_processor_id(); - for (thr = 0; thr < threads_per_subcore; ++thr) + for (thr = 0; thr < controlled_threads; ++thr) paca[pcpu + thr].kvm_hstate.kvm_split_mode = sip; /* Initiate micro-threading (split-core) if required */ @@ -2493,7 +2549,7 @@ static noinline void kvmppc_run_core(struct kvmppc_vcore *vc) } /* Let secondaries go back to the offline loop */ - for (i = 0; i < threads_per_subcore; ++i) { + for (i = 0; i < controlled_threads; ++i) { kvmppc_release_hwthread(pcpu + i); if (sip && sip->napped[i]) kvmppc_ipi_thread(pcpu + i); @@ -2545,9 +2601,6 @@ static void grow_halt_poll_ns(struct kvmppc_vcore *vc) vc->halt_poll_ns = 10000; else vc->halt_poll_ns *= halt_poll_ns_grow; - - if (vc->halt_poll_ns > halt_poll_max_ns) - vc->halt_poll_ns = halt_poll_max_ns; } static void shrink_halt_poll_ns(struct kvmppc_vcore *vc) @@ -2558,7 +2611,8 @@ static void shrink_halt_poll_ns(struct kvmppc_vcore *vc) vc->halt_poll_ns /= halt_poll_ns_shrink; } -/* Check to see if any of the runnable vcpus on the vcore have pending +/* + * Check to see if any of the runnable vcpus on the vcore have pending * exceptions or are no longer ceded */ static int kvmppc_vcore_check_block(struct kvmppc_vcore *vc) @@ -2657,16 +2711,18 @@ out: } /* Adjust poll time */ - if (halt_poll_max_ns) { + if (halt_poll_ns) { if (block_ns <= vc->halt_poll_ns) ; /* We slept and blocked for longer than the max halt time */ - else if (vc->halt_poll_ns && block_ns > halt_poll_max_ns) + else if (vc->halt_poll_ns && block_ns > halt_poll_ns) shrink_halt_poll_ns(vc); /* We slept and our poll time is too small */ - else if (vc->halt_poll_ns < halt_poll_max_ns && - block_ns < halt_poll_max_ns) + else if (vc->halt_poll_ns < halt_poll_ns && + block_ns < halt_poll_ns) grow_halt_poll_ns(vc); + if (vc->halt_poll_ns > halt_poll_ns) + vc->halt_poll_ns = halt_poll_ns; } else vc->halt_poll_ns = 0; @@ -2973,6 +3029,15 @@ static void kvmppc_core_commit_memory_region_hv(struct kvm *kvm, struct kvm_memslots *slots; struct kvm_memory_slot *memslot; + /* + * If we are making a new memslot, it might make + * some address that was previously cached as emulated + * MMIO be no longer emulated MMIO, so invalidate + * all the caches of emulated MMIO translations. + */ + if (npages) + atomic64_inc(&kvm->arch.mmio_update); + if (npages && old->npages) { /* * If modifying a memslot, reset all the rmap dirty bits. @@ -3017,6 +3082,22 @@ static void kvmppc_mmu_destroy_hv(struct kvm_vcpu *vcpu) return; } +static void kvmppc_setup_partition_table(struct kvm *kvm) +{ + unsigned long dw0, dw1; + + /* PS field - page size for VRMA */ + dw0 = ((kvm->arch.vrma_slb_v & SLB_VSID_L) >> 1) | + ((kvm->arch.vrma_slb_v & SLB_VSID_LP) << 1); + /* HTABSIZE and HTABORG fields */ + dw0 |= kvm->arch.sdr1; + + /* Second dword has GR=0; other fields are unused since UPRT=0 */ + dw1 = 0; + + mmu_partition_table_set_entry(kvm->arch.lpid, dw0, dw1); +} + static int kvmppc_hv_setup_htab_rma(struct kvm_vcpu *vcpu) { int err = 0; @@ -3068,17 +3149,20 @@ static int kvmppc_hv_setup_htab_rma(struct kvm_vcpu *vcpu) psize == 0x1000000)) goto out_srcu; - /* Update VRMASD field in the LPCR */ senc = slb_pgsize_encoding(psize); kvm->arch.vrma_slb_v = senc | SLB_VSID_B_1T | (VRMA_VSID << SLB_VSID_SHIFT_1T); - /* the -4 is to account for senc values starting at 0x10 */ - lpcr = senc << (LPCR_VRMASD_SH - 4); - /* Create HPTEs in the hash page table for the VRMA */ kvmppc_map_vrma(vcpu, memslot, porder); - kvmppc_update_lpcr(kvm, lpcr, LPCR_VRMASD); + /* Update VRMASD field in the LPCR */ + if (!cpu_has_feature(CPU_FTR_ARCH_300)) { + /* the -4 is to account for senc values starting at 0x10 */ + lpcr = senc << (LPCR_VRMASD_SH - 4); + kvmppc_update_lpcr(kvm, lpcr, LPCR_VRMASD); + } else { + kvmppc_setup_partition_table(kvm); + } /* Order updates to kvm->arch.lpcr etc. vs. hpte_setup_done */ smp_wmb(); @@ -3193,14 +3277,18 @@ static int kvmppc_core_init_vm_hv(struct kvm *kvm) * Since we don't flush the TLB when tearing down a VM, * and this lpid might have previously been used, * make sure we flush on each core before running the new VM. + * On POWER9, the tlbie in mmu_partition_table_set_entry() + * does this flush for us. */ - cpumask_setall(&kvm->arch.need_tlb_flush); + if (!cpu_has_feature(CPU_FTR_ARCH_300)) + cpumask_setall(&kvm->arch.need_tlb_flush); /* Start out with the default set of hcalls enabled */ memcpy(kvm->arch.enabled_hcalls, default_enabled_hcalls, sizeof(kvm->arch.enabled_hcalls)); - kvm->arch.host_sdr1 = mfspr(SPRN_SDR1); + if (!cpu_has_feature(CPU_FTR_ARCH_300)) + kvm->arch.host_sdr1 = mfspr(SPRN_SDR1); /* Init LPCR for virtual RMA mode */ kvm->arch.host_lpid = mfspr(SPRN_LPID); @@ -3213,9 +3301,29 @@ static int kvmppc_core_init_vm_hv(struct kvm *kvm) /* On POWER8 turn on online bit to enable PURR/SPURR */ if (cpu_has_feature(CPU_FTR_ARCH_207S)) lpcr |= LPCR_ONL; + /* + * On POWER9, VPM0 bit is reserved (VPM0=1 behaviour is assumed) + * Set HVICE bit to enable hypervisor virtualization interrupts. + */ + if (cpu_has_feature(CPU_FTR_ARCH_300)) { + lpcr &= ~LPCR_VPM0; + lpcr |= LPCR_HVICE; + } + kvm->arch.lpcr = lpcr; /* + * Work out how many sets the TLB has, for the use of + * the TLB invalidation loop in book3s_hv_rmhandlers.S. + */ + if (cpu_has_feature(CPU_FTR_ARCH_300)) + kvm->arch.tlb_sets = POWER9_TLB_SETS_HASH; /* 256 */ + else if (cpu_has_feature(CPU_FTR_ARCH_207S)) + kvm->arch.tlb_sets = POWER8_TLB_SETS; /* 512 */ + else + kvm->arch.tlb_sets = POWER7_TLB_SETS; /* 128 */ + + /* * Track that we now have a HV mode VM active. This blocks secondary * CPU threads from coming online. */ @@ -3279,9 +3387,9 @@ static int kvmppc_core_check_processor_compat_hv(void) !cpu_has_feature(CPU_FTR_ARCH_206)) return -EIO; /* - * Disable KVM for Power9, untill the required bits merged. + * Disable KVM for Power9 in radix mode. */ - if (cpu_has_feature(CPU_FTR_ARCH_300)) + if (cpu_has_feature(CPU_FTR_ARCH_300) && radix_enabled()) return -EIO; return 0; @@ -3635,6 +3743,23 @@ static int kvmppc_book3s_init_hv(void) if (r) return r; + /* + * We need a way of accessing the XICS interrupt controller, + * either directly, via paca[cpu].kvm_hstate.xics_phys, or + * indirectly, via OPAL. + */ +#ifdef CONFIG_SMP + if (!get_paca()->kvm_hstate.xics_phys) { + struct device_node *np; + + np = of_find_compatible_node(NULL, NULL, "ibm,opal-intc"); + if (!np) { + pr_err("KVM-HV: Cannot determine method for accessing XICS\n"); + return -ENODEV; + } + } +#endif + kvm_ops_hv.owner = THIS_MODULE; kvmppc_hv_ops = &kvm_ops_hv; @@ -3657,3 +3782,4 @@ module_exit(kvmppc_book3s_exit_hv); MODULE_LICENSE("GPL"); MODULE_ALIAS_MISCDEV(KVM_MINOR); MODULE_ALIAS("devname:kvm"); + diff --git a/arch/powerpc/kvm/book3s_hv_builtin.c b/arch/powerpc/kvm/book3s_hv_builtin.c index 0c84d6b..5bb24be 100644 --- a/arch/powerpc/kvm/book3s_hv_builtin.c +++ b/arch/powerpc/kvm/book3s_hv_builtin.c @@ -26,6 +26,8 @@ #include <asm/dbell.h> #include <asm/cputhreads.h> #include <asm/io.h> +#include <asm/opal.h> +#include <asm/smp.h> #define KVM_CMA_CHUNK_ORDER 18 @@ -205,12 +207,18 @@ static inline void rm_writeb(unsigned long paddr, u8 val) void kvmhv_rm_send_ipi(int cpu) { unsigned long xics_phys; + unsigned long msg = PPC_DBELL_TYPE(PPC_DBELL_SERVER); - /* On POWER8 for IPIs to threads in the same core, use msgsnd */ + /* On POWER9 we can use msgsnd for any destination cpu. */ + if (cpu_has_feature(CPU_FTR_ARCH_300)) { + msg |= get_hard_smp_processor_id(cpu); + __asm__ __volatile__ (PPC_MSGSND(%0) : : "r" (msg)); + return; + } + /* On POWER8 for IPIs to threads in the same core, use msgsnd. */ if (cpu_has_feature(CPU_FTR_ARCH_207S) && cpu_first_thread_sibling(cpu) == cpu_first_thread_sibling(raw_smp_processor_id())) { - unsigned long msg = PPC_DBELL_TYPE(PPC_DBELL_SERVER); msg |= cpu_thread_in_core(cpu); __asm__ __volatile__ (PPC_MSGSND(%0) : : "r" (msg)); return; @@ -218,7 +226,11 @@ void kvmhv_rm_send_ipi(int cpu) /* Else poke the target with an IPI */ xics_phys = paca[cpu].kvm_hstate.xics_phys; - rm_writeb(xics_phys + XICS_MFRR, IPI_PRIORITY); + if (xics_phys) + rm_writeb(xics_phys + XICS_MFRR, IPI_PRIORITY); + else + opal_rm_int_set_mfrr(get_hard_smp_processor_id(cpu), + IPI_PRIORITY); } /* @@ -329,7 +341,7 @@ static struct kvmppc_irq_map *get_irqmap(struct kvmppc_passthru_irqmap *pimap, * saved a copy of the XIRR in the PACA, it will be picked up by * the host ICP driver. */ -static int kvmppc_check_passthru(u32 xisr, __be32 xirr) +static int kvmppc_check_passthru(u32 xisr, __be32 xirr, bool *again) { struct kvmppc_passthru_irqmap *pimap; struct kvmppc_irq_map *irq_map; @@ -348,11 +360,11 @@ static int kvmppc_check_passthru(u32 xisr, __be32 xirr) /* We're handling this interrupt, generic code doesn't need to */ local_paca->kvm_hstate.saved_xirr = 0; - return kvmppc_deliver_irq_passthru(vcpu, xirr, irq_map, pimap); + return kvmppc_deliver_irq_passthru(vcpu, xirr, irq_map, pimap, again); } #else -static inline int kvmppc_check_passthru(u32 xisr, __be32 xirr) +static inline int kvmppc_check_passthru(u32 xisr, __be32 xirr, bool *again) { return 1; } @@ -367,14 +379,31 @@ static inline int kvmppc_check_passthru(u32 xisr, __be32 xirr) * -1 if there was a guest wakeup IPI (which has now been cleared) * -2 if there is PCI passthrough external interrupt that was handled */ +static long kvmppc_read_one_intr(bool *again); long kvmppc_read_intr(void) { + long ret = 0; + long rc; + bool again; + + do { + again = false; + rc = kvmppc_read_one_intr(&again); + if (rc && (ret == 0 || rc > ret)) + ret = rc; + } while (again); + return ret; +} + +static long kvmppc_read_one_intr(bool *again) +{ unsigned long xics_phys; u32 h_xirr; __be32 xirr; u32 xisr; u8 host_ipi; + int64_t rc; /* see if a host IPI is pending */ host_ipi = local_paca->kvm_hstate.host_ipi; @@ -383,8 +412,14 @@ long kvmppc_read_intr(void) /* Now read the interrupt from the ICP */ xics_phys = local_paca->kvm_hstate.xics_phys; - if (unlikely(!xics_phys)) - return 1; + if (!xics_phys) { + /* Use OPAL to read the XIRR */ + rc = opal_rm_int_get_xirr(&xirr, false); + if (rc < 0) + return 1; + } else { + xirr = _lwzcix(xics_phys + XICS_XIRR); + } /* * Save XIRR for later. Since we get control in reverse endian @@ -392,7 +427,6 @@ long kvmppc_read_intr(void) * host endian. Note that xirr is the value read from the * XIRR register, while h_xirr is the host endian version. */ - xirr = _lwzcix(xics_phys + XICS_XIRR); h_xirr = be32_to_cpu(xirr); local_paca->kvm_hstate.saved_xirr = h_xirr; xisr = h_xirr & 0xffffff; @@ -411,8 +445,16 @@ long kvmppc_read_intr(void) * If it is an IPI, clear the MFRR and EOI it. */ if (xisr == XICS_IPI) { - _stbcix(xics_phys + XICS_MFRR, 0xff); - _stwcix(xics_phys + XICS_XIRR, xirr); + if (xics_phys) { + _stbcix(xics_phys + XICS_MFRR, 0xff); + _stwcix(xics_phys + XICS_XIRR, xirr); + } else { + opal_rm_int_set_mfrr(hard_smp_processor_id(), 0xff); + rc = opal_rm_int_eoi(h_xirr); + /* If rc > 0, there is another interrupt pending */ + *again = rc > 0; + } + /* * Need to ensure side effects of above stores * complete before proceeding. @@ -429,7 +471,11 @@ long kvmppc_read_intr(void) /* We raced with the host, * we need to resend that IPI, bummer */ - _stbcix(xics_phys + XICS_MFRR, IPI_PRIORITY); + if (xics_phys) + _stbcix(xics_phys + XICS_MFRR, IPI_PRIORITY); + else + opal_rm_int_set_mfrr(hard_smp_processor_id(), + IPI_PRIORITY); /* Let side effects complete */ smp_mb(); return 1; @@ -440,5 +486,5 @@ long kvmppc_read_intr(void) return -1; } - return kvmppc_check_passthru(xisr, xirr); + return kvmppc_check_passthru(xisr, xirr, again); } diff --git a/arch/powerpc/kvm/book3s_hv_ras.c b/arch/powerpc/kvm/book3s_hv_ras.c index 0fa70a9..7ef0993 100644 --- a/arch/powerpc/kvm/book3s_hv_ras.c +++ b/arch/powerpc/kvm/book3s_hv_ras.c @@ -16,6 +16,7 @@ #include <asm/machdep.h> #include <asm/cputhreads.h> #include <asm/hmi.h> +#include <asm/kvm_ppc.h> /* SRR1 bits for machine check on POWER7 */ #define SRR1_MC_LDSTERR (1ul << (63-42)) diff --git a/arch/powerpc/kvm/book3s_hv_rm_mmu.c b/arch/powerpc/kvm/book3s_hv_rm_mmu.c index 99b4e9d..9ef3c4b 100644 --- a/arch/powerpc/kvm/book3s_hv_rm_mmu.c +++ b/arch/powerpc/kvm/book3s_hv_rm_mmu.c @@ -264,8 +264,10 @@ long kvmppc_do_h_enter(struct kvm *kvm, unsigned long flags, if (pa) pteh |= HPTE_V_VALID; - else + else { pteh |= HPTE_V_ABSENT; + ptel &= ~(HPTE_R_KEY_HI | HPTE_R_KEY_LO); + } /*If we had host pte mapping then Check WIMG */ if (ptep && !hpte_cache_flags_ok(ptel, is_ci)) { @@ -351,6 +353,7 @@ long kvmppc_do_h_enter(struct kvm *kvm, unsigned long flags, /* inval in progress, write a non-present HPTE */ pteh |= HPTE_V_ABSENT; pteh &= ~HPTE_V_VALID; + ptel &= ~(HPTE_R_KEY_HI | HPTE_R_KEY_LO); unlock_rmap(rmap); } else { kvmppc_add_revmap_chain(kvm, rev, rmap, pte_index, @@ -361,6 +364,11 @@ long kvmppc_do_h_enter(struct kvm *kvm, unsigned long flags, } } + /* Convert to new format on P9 */ + if (cpu_has_feature(CPU_FTR_ARCH_300)) { + ptel = hpte_old_to_new_r(pteh, ptel); + pteh = hpte_old_to_new_v(pteh); + } hpte[1] = cpu_to_be64(ptel); /* Write the first HPTE dword, unlocking the HPTE and making it valid */ @@ -386,6 +394,13 @@ long kvmppc_h_enter(struct kvm_vcpu *vcpu, unsigned long flags, #define LOCK_TOKEN (*(u32 *)(&get_paca()->paca_index)) #endif +static inline int is_mmio_hpte(unsigned long v, unsigned long r) +{ + return ((v & HPTE_V_ABSENT) && + (r & (HPTE_R_KEY_HI | HPTE_R_KEY_LO)) == + (HPTE_R_KEY_HI | HPTE_R_KEY_LO)); +} + static inline int try_lock_tlbie(unsigned int *lock) { unsigned int tmp, old; @@ -409,13 +424,18 @@ static void do_tlbies(struct kvm *kvm, unsigned long *rbvalues, { long i; + /* + * We use the POWER9 5-operand versions of tlbie and tlbiel here. + * Since we are using RIC=0 PRS=0 R=0, and P7/P8 tlbiel ignores + * the RS field, this is backwards-compatible with P7 and P8. + */ if (global) { while (!try_lock_tlbie(&kvm->arch.tlbie_lock)) cpu_relax(); if (need_sync) asm volatile("ptesync" : : : "memory"); for (i = 0; i < npages; ++i) - asm volatile(PPC_TLBIE(%1,%0) : : + asm volatile(PPC_TLBIE_5(%0,%1,0,0,0) : : "r" (rbvalues[i]), "r" (kvm->arch.lpid)); asm volatile("eieio; tlbsync; ptesync" : : : "memory"); kvm->arch.tlbie_lock = 0; @@ -423,7 +443,8 @@ static void do_tlbies(struct kvm *kvm, unsigned long *rbvalues, if (need_sync) asm volatile("ptesync" : : : "memory"); for (i = 0; i < npages; ++i) - asm volatile("tlbiel %0" : : "r" (rbvalues[i])); + asm volatile(PPC_TLBIEL(%0,%1,0,0,0) : : + "r" (rbvalues[i]), "r" (0)); asm volatile("ptesync" : : : "memory"); } } @@ -435,18 +456,23 @@ long kvmppc_do_h_remove(struct kvm *kvm, unsigned long flags, __be64 *hpte; unsigned long v, r, rb; struct revmap_entry *rev; - u64 pte; + u64 pte, orig_pte, pte_r; if (pte_index >= kvm->arch.hpt_npte) return H_PARAMETER; hpte = (__be64 *)(kvm->arch.hpt_virt + (pte_index << 4)); while (!try_lock_hpte(hpte, HPTE_V_HVLOCK)) cpu_relax(); - pte = be64_to_cpu(hpte[0]); + pte = orig_pte = be64_to_cpu(hpte[0]); + pte_r = be64_to_cpu(hpte[1]); + if (cpu_has_feature(CPU_FTR_ARCH_300)) { + pte = hpte_new_to_old_v(pte, pte_r); + pte_r = hpte_new_to_old_r(pte_r); + } if ((pte & (HPTE_V_ABSENT | HPTE_V_VALID)) == 0 || ((flags & H_AVPN) && (pte & ~0x7fUL) != avpn) || ((flags & H_ANDCOND) && (pte & avpn) != 0)) { - __unlock_hpte(hpte, pte); + __unlock_hpte(hpte, orig_pte); return H_NOT_FOUND; } @@ -454,7 +480,7 @@ long kvmppc_do_h_remove(struct kvm *kvm, unsigned long flags, v = pte & ~HPTE_V_HVLOCK; if (v & HPTE_V_VALID) { hpte[0] &= ~cpu_to_be64(HPTE_V_VALID); - rb = compute_tlbie_rb(v, be64_to_cpu(hpte[1]), pte_index); + rb = compute_tlbie_rb(v, pte_r, pte_index); do_tlbies(kvm, &rb, 1, global_invalidates(kvm, flags), true); /* * The reference (R) and change (C) bits in a HPT @@ -472,6 +498,9 @@ long kvmppc_do_h_remove(struct kvm *kvm, unsigned long flags, note_hpte_modification(kvm, rev); unlock_hpte(hpte, 0); + if (is_mmio_hpte(v, pte_r)) + atomic64_inc(&kvm->arch.mmio_update); + if (v & HPTE_V_ABSENT) v = (v & ~HPTE_V_ABSENT) | HPTE_V_VALID; hpret[0] = v; @@ -498,7 +527,7 @@ long kvmppc_h_bulk_remove(struct kvm_vcpu *vcpu) int global; long int ret = H_SUCCESS; struct revmap_entry *rev, *revs[4]; - u64 hp0; + u64 hp0, hp1; global = global_invalidates(kvm, 0); for (i = 0; i < 4 && ret == H_SUCCESS; ) { @@ -531,6 +560,11 @@ long kvmppc_h_bulk_remove(struct kvm_vcpu *vcpu) } found = 0; hp0 = be64_to_cpu(hp[0]); + hp1 = be64_to_cpu(hp[1]); + if (cpu_has_feature(CPU_FTR_ARCH_300)) { + hp0 = hpte_new_to_old_v(hp0, hp1); + hp1 = hpte_new_to_old_r(hp1); + } if (hp0 & (HPTE_V_ABSENT | HPTE_V_VALID)) { switch (flags & 3) { case 0: /* absolute */ @@ -561,13 +595,14 @@ long kvmppc_h_bulk_remove(struct kvm_vcpu *vcpu) rcbits = rev->guest_rpte & (HPTE_R_R|HPTE_R_C); args[j] |= rcbits << (56 - 5); hp[0] = 0; + if (is_mmio_hpte(hp0, hp1)) + atomic64_inc(&kvm->arch.mmio_update); continue; } /* leave it locked */ hp[0] &= ~cpu_to_be64(HPTE_V_VALID); - tlbrb[n] = compute_tlbie_rb(be64_to_cpu(hp[0]), - be64_to_cpu(hp[1]), pte_index); + tlbrb[n] = compute_tlbie_rb(hp0, hp1, pte_index); indexes[n] = j; hptes[n] = hp; revs[n] = rev; @@ -605,7 +640,7 @@ long kvmppc_h_protect(struct kvm_vcpu *vcpu, unsigned long flags, __be64 *hpte; struct revmap_entry *rev; unsigned long v, r, rb, mask, bits; - u64 pte; + u64 pte_v, pte_r; if (pte_index >= kvm->arch.hpt_npte) return H_PARAMETER; @@ -613,14 +648,16 @@ long kvmppc_h_protect(struct kvm_vcpu *vcpu, unsigned long flags, hpte = (__be64 *)(kvm->arch.hpt_virt + (pte_index << 4)); while (!try_lock_hpte(hpte, HPTE_V_HVLOCK)) cpu_relax(); - pte = be64_to_cpu(hpte[0]); - if ((pte & (HPTE_V_ABSENT | HPTE_V_VALID)) == 0 || - ((flags & H_AVPN) && (pte & ~0x7fUL) != avpn)) { - __unlock_hpte(hpte, pte); + v = pte_v = be64_to_cpu(hpte[0]); + if (cpu_has_feature(CPU_FTR_ARCH_300)) + v = hpte_new_to_old_v(v, be64_to_cpu(hpte[1])); + if ((v & (HPTE_V_ABSENT | HPTE_V_VALID)) == 0 || + ((flags & H_AVPN) && (v & ~0x7fUL) != avpn)) { + __unlock_hpte(hpte, pte_v); return H_NOT_FOUND; } - v = pte; + pte_r = be64_to_cpu(hpte[1]); bits = (flags << 55) & HPTE_R_PP0; bits |= (flags << 48) & HPTE_R_KEY_HI; bits |= flags & (HPTE_R_PP | HPTE_R_N | HPTE_R_KEY_LO); @@ -642,22 +679,26 @@ long kvmppc_h_protect(struct kvm_vcpu *vcpu, unsigned long flags, * readonly to writable. If it should be writable, we'll * take a trap and let the page fault code sort it out. */ - pte = be64_to_cpu(hpte[1]); - r = (pte & ~mask) | bits; - if (hpte_is_writable(r) && !hpte_is_writable(pte)) + r = (pte_r & ~mask) | bits; + if (hpte_is_writable(r) && !hpte_is_writable(pte_r)) r = hpte_make_readonly(r); /* If the PTE is changing, invalidate it first */ - if (r != pte) { + if (r != pte_r) { rb = compute_tlbie_rb(v, r, pte_index); - hpte[0] = cpu_to_be64((v & ~HPTE_V_VALID) | + hpte[0] = cpu_to_be64((pte_v & ~HPTE_V_VALID) | HPTE_V_ABSENT); do_tlbies(kvm, &rb, 1, global_invalidates(kvm, flags), true); + /* Don't lose R/C bit updates done by hardware */ + r |= be64_to_cpu(hpte[1]) & (HPTE_R_R | HPTE_R_C); hpte[1] = cpu_to_be64(r); } } - unlock_hpte(hpte, v & ~HPTE_V_HVLOCK); + unlock_hpte(hpte, pte_v & ~HPTE_V_HVLOCK); asm volatile("ptesync" : : : "memory"); + if (is_mmio_hpte(v, pte_r)) + atomic64_inc(&kvm->arch.mmio_update); + return H_SUCCESS; } @@ -681,6 +722,10 @@ long kvmppc_h_read(struct kvm_vcpu *vcpu, unsigned long flags, hpte = (__be64 *)(kvm->arch.hpt_virt + (pte_index << 4)); v = be64_to_cpu(hpte[0]) & ~HPTE_V_HVLOCK; r = be64_to_cpu(hpte[1]); + if (cpu_has_feature(CPU_FTR_ARCH_300)) { + v = hpte_new_to_old_v(v, r); + r = hpte_new_to_old_r(r); + } if (v & HPTE_V_ABSENT) { v &= ~HPTE_V_ABSENT; v |= HPTE_V_VALID; @@ -798,10 +843,16 @@ void kvmppc_invalidate_hpte(struct kvm *kvm, __be64 *hptep, unsigned long pte_index) { unsigned long rb; + u64 hp0, hp1; hptep[0] &= ~cpu_to_be64(HPTE_V_VALID); - rb = compute_tlbie_rb(be64_to_cpu(hptep[0]), be64_to_cpu(hptep[1]), - pte_index); + hp0 = be64_to_cpu(hptep[0]); + hp1 = be64_to_cpu(hptep[1]); + if (cpu_has_feature(CPU_FTR_ARCH_300)) { + hp0 = hpte_new_to_old_v(hp0, hp1); + hp1 = hpte_new_to_old_r(hp1); + } + rb = compute_tlbie_rb(hp0, hp1, pte_index); do_tlbies(kvm, &rb, 1, 1, true); } EXPORT_SYMBOL_GPL(kvmppc_invalidate_hpte); @@ -811,9 +862,15 @@ void kvmppc_clear_ref_hpte(struct kvm *kvm, __be64 *hptep, { unsigned long rb; unsigned char rbyte; + u64 hp0, hp1; - rb = compute_tlbie_rb(be64_to_cpu(hptep[0]), be64_to_cpu(hptep[1]), - pte_index); + hp0 = be64_to_cpu(hptep[0]); + hp1 = be64_to_cpu(hptep[1]); + if (cpu_has_feature(CPU_FTR_ARCH_300)) { + hp0 = hpte_new_to_old_v(hp0, hp1); + hp1 = hpte_new_to_old_r(hp1); + } + rb = compute_tlbie_rb(hp0, hp1, pte_index); rbyte = (be64_to_cpu(hptep[1]) & ~HPTE_R_R) >> 8; /* modify only the second-last byte, which contains the ref bit */ *((char *)hptep + 14) = rbyte; @@ -828,6 +885,37 @@ static int slb_base_page_shift[4] = { 20, /* 1M, unsupported */ }; +static struct mmio_hpte_cache_entry *mmio_cache_search(struct kvm_vcpu *vcpu, + unsigned long eaddr, unsigned long slb_v, long mmio_update) +{ + struct mmio_hpte_cache_entry *entry = NULL; + unsigned int pshift; + unsigned int i; + + for (i = 0; i < MMIO_HPTE_CACHE_SIZE; i++) { + entry = &vcpu->arch.mmio_cache.entry[i]; + if (entry->mmio_update == mmio_update) { + pshift = entry->slb_base_pshift; + if ((entry->eaddr >> pshift) == (eaddr >> pshift) && + entry->slb_v == slb_v) + return entry; + } + } + return NULL; +} + +static struct mmio_hpte_cache_entry * + next_mmio_cache_entry(struct kvm_vcpu *vcpu) +{ + unsigned int index = vcpu->arch.mmio_cache.index; + + vcpu->arch.mmio_cache.index++; + if (vcpu->arch.mmio_cache.index == MMIO_HPTE_CACHE_SIZE) + vcpu->arch.mmio_cache.index = 0; + + return &vcpu->arch.mmio_cache.entry[index]; +} + /* When called from virtmode, this func should be protected by * preempt_disable(), otherwise, the holding of HPTE_V_HVLOCK * can trigger deadlock issue. @@ -842,7 +930,7 @@ long kvmppc_hv_find_lock_hpte(struct kvm *kvm, gva_t eaddr, unsigned long slb_v, unsigned long avpn; __be64 *hpte; unsigned long mask, val; - unsigned long v, r; + unsigned long v, r, orig_v; /* Get page shift, work out hash and AVPN etc. */ mask = SLB_VSID_B | HPTE_V_AVPN | HPTE_V_SECONDARY; @@ -877,6 +965,8 @@ long kvmppc_hv_find_lock_hpte(struct kvm *kvm, gva_t eaddr, unsigned long slb_v, for (i = 0; i < 16; i += 2) { /* Read the PTE racily */ v = be64_to_cpu(hpte[i]) & ~HPTE_V_HVLOCK; + if (cpu_has_feature(CPU_FTR_ARCH_300)) + v = hpte_new_to_old_v(v, be64_to_cpu(hpte[i+1])); /* Check valid/absent, hash, segment size and AVPN */ if (!(v & valid) || (v & mask) != val) @@ -885,8 +975,12 @@ long kvmppc_hv_find_lock_hpte(struct kvm *kvm, gva_t eaddr, unsigned long slb_v, /* Lock the PTE and read it under the lock */ while (!try_lock_hpte(&hpte[i], HPTE_V_HVLOCK)) cpu_relax(); - v = be64_to_cpu(hpte[i]) & ~HPTE_V_HVLOCK; + v = orig_v = be64_to_cpu(hpte[i]) & ~HPTE_V_HVLOCK; r = be64_to_cpu(hpte[i+1]); + if (cpu_has_feature(CPU_FTR_ARCH_300)) { + v = hpte_new_to_old_v(v, r); + r = hpte_new_to_old_r(r); + } /* * Check the HPTE again, including base page size @@ -896,7 +990,7 @@ long kvmppc_hv_find_lock_hpte(struct kvm *kvm, gva_t eaddr, unsigned long slb_v, /* Return with the HPTE still locked */ return (hash << 3) + (i >> 1); - __unlock_hpte(&hpte[i], v); + __unlock_hpte(&hpte[i], orig_v); } if (val & HPTE_V_SECONDARY) @@ -924,30 +1018,45 @@ long kvmppc_hpte_hv_fault(struct kvm_vcpu *vcpu, unsigned long addr, { struct kvm *kvm = vcpu->kvm; long int index; - unsigned long v, r, gr; + unsigned long v, r, gr, orig_v; __be64 *hpte; unsigned long valid; struct revmap_entry *rev; unsigned long pp, key; + struct mmio_hpte_cache_entry *cache_entry = NULL; + long mmio_update = 0; /* For protection fault, expect to find a valid HPTE */ valid = HPTE_V_VALID; - if (status & DSISR_NOHPTE) + if (status & DSISR_NOHPTE) { valid |= HPTE_V_ABSENT; - - index = kvmppc_hv_find_lock_hpte(kvm, addr, slb_v, valid); - if (index < 0) { - if (status & DSISR_NOHPTE) - return status; /* there really was no HPTE */ - return 0; /* for prot fault, HPTE disappeared */ + mmio_update = atomic64_read(&kvm->arch.mmio_update); + cache_entry = mmio_cache_search(vcpu, addr, slb_v, mmio_update); } - hpte = (__be64 *)(kvm->arch.hpt_virt + (index << 4)); - v = be64_to_cpu(hpte[0]) & ~HPTE_V_HVLOCK; - r = be64_to_cpu(hpte[1]); - rev = real_vmalloc_addr(&kvm->arch.revmap[index]); - gr = rev->guest_rpte; + if (cache_entry) { + index = cache_entry->pte_index; + v = cache_entry->hpte_v; + r = cache_entry->hpte_r; + gr = cache_entry->rpte; + } else { + index = kvmppc_hv_find_lock_hpte(kvm, addr, slb_v, valid); + if (index < 0) { + if (status & DSISR_NOHPTE) + return status; /* there really was no HPTE */ + return 0; /* for prot fault, HPTE disappeared */ + } + hpte = (__be64 *)(kvm->arch.hpt_virt + (index << 4)); + v = orig_v = be64_to_cpu(hpte[0]) & ~HPTE_V_HVLOCK; + r = be64_to_cpu(hpte[1]); + if (cpu_has_feature(CPU_FTR_ARCH_300)) { + v = hpte_new_to_old_v(v, r); + r = hpte_new_to_old_r(r); + } + rev = real_vmalloc_addr(&kvm->arch.revmap[index]); + gr = rev->guest_rpte; - unlock_hpte(hpte, v); + unlock_hpte(hpte, orig_v); + } /* For not found, if the HPTE is valid by now, retry the instruction */ if ((status & DSISR_NOHPTE) && (v & HPTE_V_VALID)) @@ -985,12 +1094,32 @@ long kvmppc_hpte_hv_fault(struct kvm_vcpu *vcpu, unsigned long addr, vcpu->arch.pgfault_index = index; vcpu->arch.pgfault_hpte[0] = v; vcpu->arch.pgfault_hpte[1] = r; + vcpu->arch.pgfault_cache = cache_entry; /* Check the storage key to see if it is possibly emulated MMIO */ - if (data && (vcpu->arch.shregs.msr & MSR_IR) && - (r & (HPTE_R_KEY_HI | HPTE_R_KEY_LO)) == - (HPTE_R_KEY_HI | HPTE_R_KEY_LO)) - return -2; /* MMIO emulation - load instr word */ + if ((r & (HPTE_R_KEY_HI | HPTE_R_KEY_LO)) == + (HPTE_R_KEY_HI | HPTE_R_KEY_LO)) { + if (!cache_entry) { + unsigned int pshift = 12; + unsigned int pshift_index; + + if (slb_v & SLB_VSID_L) { + pshift_index = ((slb_v & SLB_VSID_LP) >> 4); + pshift = slb_base_page_shift[pshift_index]; + } + cache_entry = next_mmio_cache_entry(vcpu); + cache_entry->eaddr = addr; + cache_entry->slb_base_pshift = pshift; + cache_entry->pte_index = index; + cache_entry->hpte_v = v; + cache_entry->hpte_r = r; + cache_entry->rpte = gr; + cache_entry->slb_v = slb_v; + cache_entry->mmio_update = mmio_update; + } + if (data && (vcpu->arch.shregs.msr & MSR_IR)) + return -2; /* MMIO emulation - load instr word */ + } return -1; /* send fault up to host kernel mode */ } diff --git a/arch/powerpc/kvm/book3s_hv_rm_xics.c b/arch/powerpc/kvm/book3s_hv_rm_xics.c index a0ea63a..06edc43 100644 --- a/arch/powerpc/kvm/book3s_hv_rm_xics.c +++ b/arch/powerpc/kvm/book3s_hv_rm_xics.c @@ -70,7 +70,11 @@ static inline void icp_send_hcore_msg(int hcore, struct kvm_vcpu *vcpu) hcpu = hcore << threads_shift; kvmppc_host_rm_ops_hv->rm_core[hcore].rm_data = vcpu; smp_muxed_ipi_set_message(hcpu, PPC_MSG_RM_HOST_ACTION); - icp_native_cause_ipi_rm(hcpu); + if (paca[hcpu].kvm_hstate.xics_phys) + icp_native_cause_ipi_rm(hcpu); + else + opal_rm_int_set_mfrr(get_hard_smp_processor_id(hcpu), + IPI_PRIORITY); } #else static inline void icp_send_hcore_msg(int hcore, struct kvm_vcpu *vcpu) { } @@ -737,7 +741,7 @@ int kvmppc_rm_h_eoi(struct kvm_vcpu *vcpu, unsigned long xirr) unsigned long eoi_rc; -static void icp_eoi(struct irq_chip *c, u32 hwirq, u32 xirr) +static void icp_eoi(struct irq_chip *c, u32 hwirq, __be32 xirr, bool *again) { unsigned long xics_phys; int64_t rc; @@ -751,7 +755,12 @@ static void icp_eoi(struct irq_chip *c, u32 hwirq, u32 xirr) /* EOI it */ xics_phys = local_paca->kvm_hstate.xics_phys; - _stwcix(xics_phys + XICS_XIRR, xirr); + if (xics_phys) { + _stwcix(xics_phys + XICS_XIRR, xirr); + } else { + rc = opal_rm_int_eoi(be32_to_cpu(xirr)); + *again = rc > 0; + } } static int xics_opal_rm_set_server(unsigned int hw_irq, int server_cpu) @@ -809,9 +818,10 @@ static void kvmppc_rm_handle_irq_desc(struct irq_desc *desc) } long kvmppc_deliver_irq_passthru(struct kvm_vcpu *vcpu, - u32 xirr, + __be32 xirr, struct kvmppc_irq_map *irq_map, - struct kvmppc_passthru_irqmap *pimap) + struct kvmppc_passthru_irqmap *pimap, + bool *again) { struct kvmppc_xics *xics; struct kvmppc_icp *icp; @@ -825,7 +835,8 @@ long kvmppc_deliver_irq_passthru(struct kvm_vcpu *vcpu, icp_rm_deliver_irq(xics, icp, irq); /* EOI the interrupt */ - icp_eoi(irq_desc_get_chip(irq_map->desc), irq_map->r_hwirq, xirr); + icp_eoi(irq_desc_get_chip(irq_map->desc), irq_map->r_hwirq, xirr, + again); if (check_too_hard(xics, icp) == H_TOO_HARD) return 2; diff --git a/arch/powerpc/kvm/book3s_hv_rmhandlers.S b/arch/powerpc/kvm/book3s_hv_rmhandlers.S index c3c1d1b..9338a81 100644 --- a/arch/powerpc/kvm/book3s_hv_rmhandlers.S +++ b/arch/powerpc/kvm/book3s_hv_rmhandlers.S @@ -501,17 +501,9 @@ END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S) cmpwi r0, 0 beq 57f li r3, (LPCR_PECEDH | LPCR_PECE0) >> 4 - mfspr r4, SPRN_LPCR - rlwimi r4, r3, 4, (LPCR_PECEDP | LPCR_PECEDH | LPCR_PECE0 | LPCR_PECE1) - mtspr SPRN_LPCR, r4 - isync - std r0, HSTATE_SCRATCH0(r13) - ptesync - ld r0, HSTATE_SCRATCH0(r13) -1: cmpd r0, r0 - bne 1b - nap - b . + mfspr r5, SPRN_LPCR + rlwimi r5, r3, 4, (LPCR_PECEDP | LPCR_PECEDH | LPCR_PECE0 | LPCR_PECE1) + b kvm_nap_sequence 57: li r0, 0 stbx r0, r3, r4 @@ -523,6 +515,10 @@ END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S) * * *****************************************************************************/ +/* Stack frame offsets */ +#define STACK_SLOT_TID (112-16) +#define STACK_SLOT_PSSCR (112-24) + .global kvmppc_hv_entry kvmppc_hv_entry: @@ -581,12 +577,14 @@ kvmppc_hv_entry: ld r9,VCORE_KVM(r5) /* pointer to struct kvm */ cmpwi r6,0 bne 10f - ld r6,KVM_SDR1(r9) lwz r7,KVM_LPID(r9) +BEGIN_FTR_SECTION + ld r6,KVM_SDR1(r9) li r0,LPID_RSVD /* switch to reserved LPID */ mtspr SPRN_LPID,r0 ptesync mtspr SPRN_SDR1,r6 /* switch to partition page table */ +END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_300) mtspr SPRN_LPID,r7 isync @@ -607,12 +605,8 @@ kvmppc_hv_entry: stdcx. r7,0,r6 bne 23b /* Flush the TLB of any entries for this LPID */ - /* use arch 2.07S as a proxy for POWER8 */ -BEGIN_FTR_SECTION - li r6,512 /* POWER8 has 512 sets */ -FTR_SECTION_ELSE - li r6,128 /* POWER7 has 128 sets */ -ALT_FTR_SECTION_END_IFSET(CPU_FTR_ARCH_207S) + lwz r6,KVM_TLB_SETS(r9) + li r0,0 /* RS for P9 version of tlbiel */ mtctr r6 li r7,0x800 /* IS field = 0b10 */ ptesync @@ -698,6 +692,14 @@ kvmppc_got_guest: mtspr SPRN_PURR,r7 mtspr SPRN_SPURR,r8 + /* Save host values of some registers */ +BEGIN_FTR_SECTION + mfspr r5, SPRN_TIDR + mfspr r6, SPRN_PSSCR + std r5, STACK_SLOT_TID(r1) + std r6, STACK_SLOT_PSSCR(r1) +END_FTR_SECTION_IFSET(CPU_FTR_ARCH_300) + BEGIN_FTR_SECTION /* Set partition DABR */ /* Do this before re-enabling PMU to avoid P7 DABR corruption bug */ @@ -750,14 +752,16 @@ END_FTR_SECTION_IFSET(CPU_FTR_PMAO_BUG) BEGIN_FTR_SECTION ld r5, VCPU_MMCR + 24(r4) ld r6, VCPU_SIER(r4) + mtspr SPRN_MMCR2, r5 + mtspr SPRN_SIER, r6 +BEGIN_FTR_SECTION_NESTED(96) lwz r7, VCPU_PMC + 24(r4) lwz r8, VCPU_PMC + 28(r4) ld r9, VCPU_MMCR + 32(r4) - mtspr SPRN_MMCR2, r5 - mtspr SPRN_SIER, r6 mtspr SPRN_SPMC1, r7 mtspr SPRN_SPMC2, r8 mtspr SPRN_MMCRS, r9 +END_FTR_SECTION_NESTED(CPU_FTR_ARCH_300, 0, 96) END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S) mtspr SPRN_MMCR0, r3 isync @@ -813,20 +817,30 @@ END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_207S) mtspr SPRN_EBBHR, r8 ld r5, VCPU_EBBRR(r4) ld r6, VCPU_BESCR(r4) - ld r7, VCPU_CSIGR(r4) - ld r8, VCPU_TACR(r4) + lwz r7, VCPU_GUEST_PID(r4) + ld r8, VCPU_WORT(r4) mtspr SPRN_EBBRR, r5 mtspr SPRN_BESCR, r6 - mtspr SPRN_CSIGR, r7 - mtspr SPRN_TACR, r8 + mtspr SPRN_PID, r7 + mtspr SPRN_WORT, r8 +BEGIN_FTR_SECTION + /* POWER8-only registers */ ld r5, VCPU_TCSCR(r4) ld r6, VCPU_ACOP(r4) - lwz r7, VCPU_GUEST_PID(r4) - ld r8, VCPU_WORT(r4) + ld r7, VCPU_CSIGR(r4) + ld r8, VCPU_TACR(r4) mtspr SPRN_TCSCR, r5 mtspr SPRN_ACOP, r6 - mtspr SPRN_PID, r7 - mtspr SPRN_WORT, r8 + mtspr SPRN_CSIGR, r7 + mtspr SPRN_TACR, r8 +FTR_SECTION_ELSE + /* POWER9-only registers */ + ld r5, VCPU_TID(r4) + ld r6, VCPU_PSSCR(r4) + oris r6, r6, PSSCR_EC@h /* This makes stop trap to HV */ + mtspr SPRN_TIDR, r5 + mtspr SPRN_PSSCR, r6 +ALT_FTR_SECTION_END_IFCLR(CPU_FTR_ARCH_300) 8: /* @@ -1341,20 +1355,29 @@ END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_207S) std r8, VCPU_EBBHR(r9) mfspr r5, SPRN_EBBRR mfspr r6, SPRN_BESCR - mfspr r7, SPRN_CSIGR - mfspr r8, SPRN_TACR + mfspr r7, SPRN_PID + mfspr r8, SPRN_WORT std r5, VCPU_EBBRR(r9) std r6, VCPU_BESCR(r9) - std r7, VCPU_CSIGR(r9) - std r8, VCPU_TACR(r9) + stw r7, VCPU_GUEST_PID(r9) + std r8, VCPU_WORT(r9) +BEGIN_FTR_SECTION mfspr r5, SPRN_TCSCR mfspr r6, SPRN_ACOP - mfspr r7, SPRN_PID - mfspr r8, SPRN_WORT + mfspr r7, SPRN_CSIGR + mfspr r8, SPRN_TACR std r5, VCPU_TCSCR(r9) std r6, VCPU_ACOP(r9) - stw r7, VCPU_GUEST_PID(r9) - std r8, VCPU_WORT(r9) + std r7, VCPU_CSIGR(r9) + std r8, VCPU_TACR(r9) +FTR_SECTION_ELSE + mfspr r5, SPRN_TIDR + mfspr r6, SPRN_PSSCR + std r5, VCPU_TID(r9) + rldicl r6, r6, 4, 50 /* r6 &= PSSCR_GUEST_VIS */ + rotldi r6, r6, 60 + std r6, VCPU_PSSCR(r9) +ALT_FTR_SECTION_END_IFCLR(CPU_FTR_ARCH_300) /* * Restore various registers to 0, where non-zero values * set by the guest could disrupt the host. @@ -1363,12 +1386,14 @@ END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_207S) mtspr SPRN_IAMR, r0 mtspr SPRN_CIABR, r0 mtspr SPRN_DAWRX, r0 - mtspr SPRN_TCSCR, r0 mtspr SPRN_WORT, r0 +BEGIN_FTR_SECTION + mtspr SPRN_TCSCR, r0 /* Set MMCRS to 1<<31 to freeze and disable the SPMC counters */ li r0, 1 sldi r0, r0, 31 mtspr SPRN_MMCRS, r0 +END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_300) 8: /* Save and reset AMR and UAMOR before turning on the MMU */ @@ -1502,15 +1527,17 @@ END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S) stw r8, VCPU_PMC + 20(r9) BEGIN_FTR_SECTION mfspr r5, SPRN_SIER + std r5, VCPU_SIER(r9) +BEGIN_FTR_SECTION_NESTED(96) mfspr r6, SPRN_SPMC1 mfspr r7, SPRN_SPMC2 mfspr r8, SPRN_MMCRS - std r5, VCPU_SIER(r9) stw r6, VCPU_PMC + 24(r9) stw r7, VCPU_PMC + 28(r9) std r8, VCPU_MMCR + 32(r9) lis r4, 0x8000 mtspr SPRN_MMCRS, r4 +END_FTR_SECTION_NESTED(CPU_FTR_ARCH_300, 0, 96) END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S) 22: /* Clear out SLB */ @@ -1519,6 +1546,14 @@ END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S) slbia ptesync + /* Restore host values of some registers */ +BEGIN_FTR_SECTION + ld r5, STACK_SLOT_TID(r1) + ld r6, STACK_SLOT_PSSCR(r1) + mtspr SPRN_TIDR, r5 + mtspr SPRN_PSSCR, r6 +END_FTR_SECTION_IFSET(CPU_FTR_ARCH_300) + /* * POWER7/POWER8 guest -> host partition switch code. * We don't have to lock against tlbies but we do @@ -1552,12 +1587,14 @@ kvmhv_switch_to_host: beq 19f /* Primary thread switches back to host partition */ - ld r6,KVM_HOST_SDR1(r4) lwz r7,KVM_HOST_LPID(r4) +BEGIN_FTR_SECTION + ld r6,KVM_HOST_SDR1(r4) li r8,LPID_RSVD /* switch to reserved LPID */ mtspr SPRN_LPID,r8 ptesync - mtspr SPRN_SDR1,r6 /* switch to partition page table */ + mtspr SPRN_SDR1,r6 /* switch to host page table */ +END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_300) mtspr SPRN_LPID,r7 isync @@ -2211,6 +2248,21 @@ BEGIN_FTR_SECTION ori r5, r5, LPCR_PECEDH rlwimi r5, r3, 0, LPCR_PECEDP END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S) + +kvm_nap_sequence: /* desired LPCR value in r5 */ +BEGIN_FTR_SECTION + /* + * PSSCR bits: exit criterion = 1 (wakeup based on LPCR at sreset) + * enable state loss = 1 (allow SMT mode switch) + * requested level = 0 (just stop dispatching) + */ + lis r3, (PSSCR_EC | PSSCR_ESL)@h + mtspr SPRN_PSSCR, r3 + /* Set LPCR_PECE_HVEE bit to enable wakeup by HV interrupts */ + li r4, LPCR_PECE_HVEE@higher + sldi r4, r4, 32 + or r5, r5, r4 +END_FTR_SECTION_IFSET(CPU_FTR_ARCH_300) mtspr SPRN_LPCR,r5 isync li r0, 0 @@ -2219,7 +2271,11 @@ END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S) ld r0, HSTATE_SCRATCH0(r13) 1: cmpd r0, r0 bne 1b +BEGIN_FTR_SECTION nap +FTR_SECTION_ELSE + PPC_STOP +ALT_FTR_SECTION_END_IFCLR(CPU_FTR_ARCH_300) b . 33: mr r4, r3 @@ -2600,11 +2656,13 @@ kvmppc_save_tm: mfctr r7 mfspr r8, SPRN_AMR mfspr r10, SPRN_TAR + mfxer r11 std r5, VCPU_LR_TM(r9) stw r6, VCPU_CR_TM(r9) std r7, VCPU_CTR_TM(r9) std r8, VCPU_AMR_TM(r9) std r10, VCPU_TAR_TM(r9) + std r11, VCPU_XER_TM(r9) /* Restore r12 as trap number. */ lwz r12, VCPU_TRAP(r9) @@ -2697,11 +2755,13 @@ kvmppc_restore_tm: ld r7, VCPU_CTR_TM(r4) ld r8, VCPU_AMR_TM(r4) ld r9, VCPU_TAR_TM(r4) + ld r10, VCPU_XER_TM(r4) mtlr r5 mtcr r6 mtctr r7 mtspr SPRN_AMR, r8 mtspr SPRN_TAR, r9 + mtxer r10 /* * Load up PPR and DSCR values but don't put them in the actual SPRs diff --git a/arch/powerpc/kvm/powerpc.c b/arch/powerpc/kvm/powerpc.c index 70963c8..efd1183 100644 --- a/arch/powerpc/kvm/powerpc.c +++ b/arch/powerpc/kvm/powerpc.c @@ -536,7 +536,6 @@ int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext) #ifdef CONFIG_PPC_BOOK3S_64 case KVM_CAP_SPAPR_TCE: case KVM_CAP_SPAPR_TCE_64: - case KVM_CAP_PPC_ALLOC_HTAB: case KVM_CAP_PPC_RTAS: case KVM_CAP_PPC_FIXUP_HCALL: case KVM_CAP_PPC_ENABLE_HCALL: @@ -545,13 +544,20 @@ int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext) #endif r = 1; break; + + case KVM_CAP_PPC_ALLOC_HTAB: + r = hv_enabled; + break; #endif /* CONFIG_PPC_BOOK3S_64 */ #ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE case KVM_CAP_PPC_SMT: - if (hv_enabled) - r = threads_per_subcore; - else - r = 0; + r = 0; + if (hv_enabled) { + if (cpu_has_feature(CPU_FTR_ARCH_300)) + r = 1; + else + r = threads_per_subcore; + } break; case KVM_CAP_PPC_RMA: r = 0; diff --git a/arch/powerpc/kvm/trace_hv.h b/arch/powerpc/kvm/trace_hv.h index fb21990..ebc6dd4 100644 --- a/arch/powerpc/kvm/trace_hv.h +++ b/arch/powerpc/kvm/trace_hv.h @@ -449,7 +449,7 @@ TRACE_EVENT(kvmppc_vcore_wakeup, __entry->tgid = current->tgid; ), - TP_printk("%s time %lld ns, tgid=%d", + TP_printk("%s time %llu ns, tgid=%d", __entry->waited ? "wait" : "poll", __entry->ns, __entry->tgid) ); diff --git a/arch/powerpc/mm/hash_native_64.c b/arch/powerpc/mm/hash_native_64.c index 83ddc0e..ad9fd52 100644 --- a/arch/powerpc/mm/hash_native_64.c +++ b/arch/powerpc/mm/hash_native_64.c @@ -221,13 +221,18 @@ static long native_hpte_insert(unsigned long hpte_group, unsigned long vpn, return -1; hpte_v = hpte_encode_v(vpn, psize, apsize, ssize) | vflags | HPTE_V_VALID; - hpte_r = hpte_encode_r(pa, psize, apsize, ssize) | rflags; + hpte_r = hpte_encode_r(pa, psize, apsize) | rflags; if (!(vflags & HPTE_V_BOLTED)) { DBG_LOW(" i=%x hpte_v=%016lx, hpte_r=%016lx\n", i, hpte_v, hpte_r); } + if (cpu_has_feature(CPU_FTR_ARCH_300)) { + hpte_r = hpte_old_to_new_r(hpte_v, hpte_r); + hpte_v = hpte_old_to_new_v(hpte_v); + } + hptep->r = cpu_to_be64(hpte_r); /* Guarantee the second dword is visible before the valid bit */ eieio(); @@ -295,6 +300,8 @@ static long native_hpte_updatepp(unsigned long slot, unsigned long newpp, vpn, want_v & HPTE_V_AVPN, slot, newpp); hpte_v = be64_to_cpu(hptep->v); + if (cpu_has_feature(CPU_FTR_ARCH_300)) + hpte_v = hpte_new_to_old_v(hpte_v, be64_to_cpu(hptep->r)); /* * We need to invalidate the TLB always because hpte_remove doesn't do * a tlb invalidate. If a hash bucket gets full, we "evict" a more/less @@ -309,6 +316,8 @@ static long native_hpte_updatepp(unsigned long slot, unsigned long newpp, native_lock_hpte(hptep); /* recheck with locks held */ hpte_v = be64_to_cpu(hptep->v); + if (cpu_has_feature(CPU_FTR_ARCH_300)) + hpte_v = hpte_new_to_old_v(hpte_v, be64_to_cpu(hptep->r)); if (unlikely(!HPTE_V_COMPARE(hpte_v, want_v) || !(hpte_v & HPTE_V_VALID))) { ret = -1; @@ -350,6 +359,8 @@ static long native_hpte_find(unsigned long vpn, int psize, int ssize) for (i = 0; i < HPTES_PER_GROUP; i++) { hptep = htab_address + slot; hpte_v = be64_to_cpu(hptep->v); + if (cpu_has_feature(CPU_FTR_ARCH_300)) + hpte_v = hpte_new_to_old_v(hpte_v, be64_to_cpu(hptep->r)); if (HPTE_V_COMPARE(hpte_v, want_v) && (hpte_v & HPTE_V_VALID)) /* HPTE matches */ @@ -409,6 +420,8 @@ static void native_hpte_invalidate(unsigned long slot, unsigned long vpn, want_v = hpte_encode_avpn(vpn, bpsize, ssize); native_lock_hpte(hptep); hpte_v = be64_to_cpu(hptep->v); + if (cpu_has_feature(CPU_FTR_ARCH_300)) + hpte_v = hpte_new_to_old_v(hpte_v, be64_to_cpu(hptep->r)); /* * We need to invalidate the TLB always because hpte_remove doesn't do @@ -467,6 +480,8 @@ static void native_hugepage_invalidate(unsigned long vsid, want_v = hpte_encode_avpn(vpn, psize, ssize); native_lock_hpte(hptep); hpte_v = be64_to_cpu(hptep->v); + if (cpu_has_feature(CPU_FTR_ARCH_300)) + hpte_v = hpte_new_to_old_v(hpte_v, be64_to_cpu(hptep->r)); /* Even if we miss, we need to invalidate the TLB */ if (!HPTE_V_COMPARE(hpte_v, want_v) || !(hpte_v & HPTE_V_VALID)) @@ -504,6 +519,10 @@ static void hpte_decode(struct hash_pte *hpte, unsigned long slot, /* Look at the 8 bit LP value */ unsigned int lp = (hpte_r >> LP_SHIFT) & ((1 << LP_BITS) - 1); + if (cpu_has_feature(CPU_FTR_ARCH_300)) { + hpte_v = hpte_new_to_old_v(hpte_v, hpte_r); + hpte_r = hpte_new_to_old_r(hpte_r); + } if (!(hpte_v & HPTE_V_LARGE)) { size = MMU_PAGE_4K; a_size = MMU_PAGE_4K; @@ -512,11 +531,7 @@ static void hpte_decode(struct hash_pte *hpte, unsigned long slot, a_size = hpte_page_sizes[lp] >> 4; } /* This works for all page sizes, and for 256M and 1T segments */ - if (cpu_has_feature(CPU_FTR_ARCH_300)) - *ssize = hpte_r >> HPTE_R_3_0_SSIZE_SHIFT; - else - *ssize = hpte_v >> HPTE_V_SSIZE_SHIFT; - + *ssize = hpte_v >> HPTE_V_SSIZE_SHIFT; shift = mmu_psize_defs[size].shift; avpn = (HPTE_V_AVPN_VAL(hpte_v) & ~mmu_psize_defs[size].avpnm); @@ -639,6 +654,9 @@ static void native_flush_hash_range(unsigned long number, int local) want_v = hpte_encode_avpn(vpn, psize, ssize); native_lock_hpte(hptep); hpte_v = be64_to_cpu(hptep->v); + if (cpu_has_feature(CPU_FTR_ARCH_300)) + hpte_v = hpte_new_to_old_v(hpte_v, + be64_to_cpu(hptep->r)); if (!HPTE_V_COMPARE(hpte_v, want_v) || !(hpte_v & HPTE_V_VALID)) native_unlock_hpte(hptep); diff --git a/arch/powerpc/mm/hash_utils_64.c b/arch/powerpc/mm/hash_utils_64.c index 78dabf06..8410b4b 100644 --- a/arch/powerpc/mm/hash_utils_64.c +++ b/arch/powerpc/mm/hash_utils_64.c @@ -796,37 +796,17 @@ static void update_hid_for_hash(void) static void __init hash_init_partition_table(phys_addr_t hash_table, unsigned long htab_size) { - unsigned long ps_field; - unsigned long patb_size = 1UL << PATB_SIZE_SHIFT; + mmu_partition_table_init(); /* - * slb llp encoding for the page size used in VPM real mode. - * We can ignore that for lpid 0 + * PS field (VRMA page size) is not used for LPID 0, hence set to 0. + * For now, UPRT is 0 and we have no segment table. */ - ps_field = 0; htab_size = __ilog2(htab_size) - 18; - - BUILD_BUG_ON_MSG((PATB_SIZE_SHIFT > 24), "Partition table size too large."); - partition_tb = __va(memblock_alloc_base(patb_size, patb_size, - MEMBLOCK_ALLOC_ANYWHERE)); - - /* Initialize the Partition Table with no entries */ - memset((void *)partition_tb, 0, patb_size); - partition_tb->patb0 = cpu_to_be64(ps_field | hash_table | htab_size); - /* - * FIXME!! This should be done via update_partition table - * For now UPRT is 0 for us. - */ - partition_tb->patb1 = 0; + mmu_partition_table_set_entry(0, hash_table | htab_size, 0); pr_info("Partition table %p\n", partition_tb); if (cpu_has_feature(CPU_FTR_POWER9_DD1)) update_hid_for_hash(); - /* - * update partition table control register, - * 64 K size. - */ - mtspr(SPRN_PTCR, __pa(partition_tb) | (PATB_SIZE_SHIFT - 12)); - } static void __init htab_initialize(void) diff --git a/arch/powerpc/mm/pgtable-radix.c b/arch/powerpc/mm/pgtable-radix.c index 688b545..8d941c6 100644 --- a/arch/powerpc/mm/pgtable-radix.c +++ b/arch/powerpc/mm/pgtable-radix.c @@ -177,23 +177,15 @@ redo: static void __init radix_init_partition_table(void) { - unsigned long rts_field; + unsigned long rts_field, dw0; + mmu_partition_table_init(); rts_field = radix__get_tree_size(); + dw0 = rts_field | __pa(init_mm.pgd) | RADIX_PGD_INDEX_SIZE | PATB_HR; + mmu_partition_table_set_entry(0, dw0, 0); - BUILD_BUG_ON_MSG((PATB_SIZE_SHIFT > 24), "Partition table size too large."); - partition_tb = early_alloc_pgtable(1UL << PATB_SIZE_SHIFT); - partition_tb->patb0 = cpu_to_be64(rts_field | __pa(init_mm.pgd) | - RADIX_PGD_INDEX_SIZE | PATB_HR); pr_info("Initializing Radix MMU\n"); pr_info("Partition table %p\n", partition_tb); - - memblock_set_current_limit(MEMBLOCK_ALLOC_ANYWHERE); - /* - * update partition table control register, - * 64 K size. - */ - mtspr(SPRN_PTCR, __pa(partition_tb) | (PATB_SIZE_SHIFT - 12)); } void __init radix_init_native(void) @@ -378,6 +370,8 @@ void __init radix__early_init_mmu(void) radix_init_partition_table(); } + memblock_set_current_limit(MEMBLOCK_ALLOC_ANYWHERE); + radix_init_pgtable(); } diff --git a/arch/powerpc/mm/pgtable_64.c b/arch/powerpc/mm/pgtable_64.c index f5e8d4e..8bca7f5 100644 --- a/arch/powerpc/mm/pgtable_64.c +++ b/arch/powerpc/mm/pgtable_64.c @@ -431,3 +431,37 @@ void pgtable_free_tlb(struct mmu_gather *tlb, void *table, int shift) } } #endif + +#ifdef CONFIG_PPC_BOOK3S_64 +void __init mmu_partition_table_init(void) +{ + unsigned long patb_size = 1UL << PATB_SIZE_SHIFT; + + BUILD_BUG_ON_MSG((PATB_SIZE_SHIFT > 36), "Partition table size too large."); + partition_tb = __va(memblock_alloc_base(patb_size, patb_size, + MEMBLOCK_ALLOC_ANYWHERE)); + + /* Initialize the Partition Table with no entries */ + memset((void *)partition_tb, 0, patb_size); + + /* + * update partition table control register, + * 64 K size. + */ + mtspr(SPRN_PTCR, __pa(partition_tb) | (PATB_SIZE_SHIFT - 12)); +} + +void mmu_partition_table_set_entry(unsigned int lpid, unsigned long dw0, + unsigned long dw1) +{ + partition_tb[lpid].patb0 = cpu_to_be64(dw0); + partition_tb[lpid].patb1 = cpu_to_be64(dw1); + + /* Global flush of TLBs and partition table caches for this lpid */ + asm volatile("ptesync" : : : "memory"); + asm volatile(PPC_TLBIE_5(%0,%1,2,0,0) : : + "r" (TLBIEL_INVAL_SET_LPID), "r" (lpid)); + asm volatile("eieio; tlbsync; ptesync" : : : "memory"); +} +EXPORT_SYMBOL_GPL(mmu_partition_table_set_entry); +#endif /* CONFIG_PPC_BOOK3S_64 */ diff --git a/arch/powerpc/platforms/powernv/opal-wrappers.S b/arch/powerpc/platforms/powernv/opal-wrappers.S index 44d2d84..3aa40f1 100644 --- a/arch/powerpc/platforms/powernv/opal-wrappers.S +++ b/arch/powerpc/platforms/powernv/opal-wrappers.S @@ -304,8 +304,11 @@ OPAL_CALL(opal_pci_get_presence_state, OPAL_PCI_GET_PRESENCE_STATE); OPAL_CALL(opal_pci_get_power_state, OPAL_PCI_GET_POWER_STATE); OPAL_CALL(opal_pci_set_power_state, OPAL_PCI_SET_POWER_STATE); OPAL_CALL(opal_int_get_xirr, OPAL_INT_GET_XIRR); +OPAL_CALL_REAL(opal_rm_int_get_xirr, OPAL_INT_GET_XIRR); OPAL_CALL(opal_int_set_cppr, OPAL_INT_SET_CPPR); OPAL_CALL(opal_int_eoi, OPAL_INT_EOI); +OPAL_CALL_REAL(opal_rm_int_eoi, OPAL_INT_EOI); OPAL_CALL(opal_int_set_mfrr, OPAL_INT_SET_MFRR); +OPAL_CALL_REAL(opal_rm_int_set_mfrr, OPAL_INT_SET_MFRR); OPAL_CALL(opal_pci_tce_kill, OPAL_PCI_TCE_KILL); OPAL_CALL_REAL(opal_rm_pci_tce_kill, OPAL_PCI_TCE_KILL); diff --git a/arch/powerpc/platforms/powernv/opal.c b/arch/powerpc/platforms/powernv/opal.c index 6c9a65b..b3b8930 100644 --- a/arch/powerpc/platforms/powernv/opal.c +++ b/arch/powerpc/platforms/powernv/opal.c @@ -896,3 +896,5 @@ EXPORT_SYMBOL_GPL(opal_leds_get_ind); EXPORT_SYMBOL_GPL(opal_leds_set_ind); /* Export this symbol for PowerNV Operator Panel class driver */ EXPORT_SYMBOL_GPL(opal_write_oppanel_async); +/* Export this for KVM */ +EXPORT_SYMBOL_GPL(opal_int_set_mfrr); diff --git a/arch/powerpc/platforms/ps3/htab.c b/arch/powerpc/platforms/ps3/htab.c index cb3c503..cc2b281 100644 --- a/arch/powerpc/platforms/ps3/htab.c +++ b/arch/powerpc/platforms/ps3/htab.c @@ -63,7 +63,7 @@ static long ps3_hpte_insert(unsigned long hpte_group, unsigned long vpn, vflags &= ~HPTE_V_SECONDARY; hpte_v = hpte_encode_v(vpn, psize, apsize, ssize) | vflags | HPTE_V_VALID; - hpte_r = hpte_encode_r(ps3_mm_phys_to_lpar(pa), psize, apsize, ssize) | rflags; + hpte_r = hpte_encode_r(ps3_mm_phys_to_lpar(pa), psize, apsize) | rflags; spin_lock_irqsave(&ps3_htab_lock, flags); diff --git a/arch/powerpc/platforms/pseries/lpar.c b/arch/powerpc/platforms/pseries/lpar.c index aa35245..f2c98f6 100644 --- a/arch/powerpc/platforms/pseries/lpar.c +++ b/arch/powerpc/platforms/pseries/lpar.c @@ -145,7 +145,7 @@ static long pSeries_lpar_hpte_insert(unsigned long hpte_group, hpte_group, vpn, pa, rflags, vflags, psize); hpte_v = hpte_encode_v(vpn, psize, apsize, ssize) | vflags | HPTE_V_VALID; - hpte_r = hpte_encode_r(pa, psize, apsize, ssize) | rflags; + hpte_r = hpte_encode_r(pa, psize, apsize) | rflags; if (!(vflags & HPTE_V_BOLTED)) pr_devel(" hpte_v=%016lx, hpte_r=%016lx\n", hpte_v, hpte_r); diff --git a/arch/s390/Kconfig b/arch/s390/Kconfig index 028f97b..c6722112 100644 --- a/arch/s390/Kconfig +++ b/arch/s390/Kconfig @@ -136,6 +136,7 @@ config S390 select HAVE_CMPXCHG_LOCAL select HAVE_DEBUG_KMEMLEAK select HAVE_DMA_API_DEBUG + select HAVE_DMA_CONTIGUOUS select HAVE_DYNAMIC_FTRACE select HAVE_DYNAMIC_FTRACE_WITH_REGS select HAVE_EFFICIENT_UNALIGNED_ACCESS @@ -169,6 +170,7 @@ config S390 select OLD_SIGSUSPEND3 select SPARSE_IRQ select SYSCTL_EXCEPTION_TRACE + select THREAD_INFO_IN_TASK select TTY select VIRT_CPU_ACCOUNTING select ARCH_HAS_SCALED_CPUTIME diff --git a/arch/s390/boot/compressed/head.S b/arch/s390/boot/compressed/head.S index 28c4f96..11f6254 100644 --- a/arch/s390/boot/compressed/head.S +++ b/arch/s390/boot/compressed/head.S @@ -46,7 +46,7 @@ mover_end: .align 8 .Lstack: - .quad 0x8000 + (1<<(PAGE_SHIFT+THREAD_ORDER)) + .quad 0x8000 + (1<<(PAGE_SHIFT+THREAD_SIZE_ORDER)) .Loffset: .quad 0x11000 .Lmvsize: diff --git a/arch/s390/configs/default_defconfig b/arch/s390/configs/default_defconfig index 4596868..e659daf 100644 --- a/arch/s390/configs/default_defconfig +++ b/arch/s390/configs/default_defconfig @@ -66,6 +66,8 @@ CONFIG_TRANSPARENT_HUGEPAGE=y CONFIG_CLEANCACHE=y CONFIG_FRONTSWAP=y CONFIG_CMA=y +CONFIG_CMA_DEBUG=y +CONFIG_CMA_DEBUGFS=y CONFIG_MEM_SOFT_DIRTY=y CONFIG_ZPOOL=m CONFIG_ZBUD=m @@ -366,6 +368,8 @@ CONFIG_BPF_JIT=y CONFIG_NET_PKTGEN=m CONFIG_NET_TCPPROBE=m CONFIG_DEVTMPFS=y +CONFIG_DMA_CMA=y +CONFIG_CMA_SIZE_MBYTES=0 CONFIG_CONNECTOR=y CONFIG_BLK_DEV_LOOP=m CONFIG_BLK_DEV_CRYPTOLOOP=m @@ -438,7 +442,6 @@ CONFIG_TUN=m CONFIG_VETH=m CONFIG_VIRTIO_NET=m CONFIG_NLMON=m -CONFIG_VHOST_NET=m # CONFIG_NET_VENDOR_ARC is not set # CONFIG_NET_VENDOR_CHELSIO is not set # CONFIG_NET_VENDOR_INTEL is not set @@ -693,3 +696,4 @@ CONFIG_CMM=m CONFIG_APPLDATA_BASE=y CONFIG_KVM=m CONFIG_KVM_S390_UCONTROL=y +CONFIG_VHOST_NET=m diff --git a/arch/s390/configs/gcov_defconfig b/arch/s390/configs/gcov_defconfig index 1dd05e3..95ceac5 100644 --- a/arch/s390/configs/gcov_defconfig +++ b/arch/s390/configs/gcov_defconfig @@ -362,6 +362,8 @@ CONFIG_BPF_JIT=y CONFIG_NET_PKTGEN=m CONFIG_NET_TCPPROBE=m CONFIG_DEVTMPFS=y +CONFIG_DMA_CMA=y +CONFIG_CMA_SIZE_MBYTES=0 CONFIG_CONNECTOR=y CONFIG_BLK_DEV_LOOP=m CONFIG_BLK_DEV_CRYPTOLOOP=m @@ -434,7 +436,6 @@ CONFIG_TUN=m CONFIG_VETH=m CONFIG_VIRTIO_NET=m CONFIG_NLMON=m -CONFIG_VHOST_NET=m # CONFIG_NET_VENDOR_ARC is not set # CONFIG_NET_VENDOR_CHELSIO is not set # CONFIG_NET_VENDOR_INTEL is not set @@ -633,3 +634,4 @@ CONFIG_CMM=m CONFIG_APPLDATA_BASE=y CONFIG_KVM=m CONFIG_KVM_S390_UCONTROL=y +CONFIG_VHOST_NET=m diff --git a/arch/s390/configs/performance_defconfig b/arch/s390/configs/performance_defconfig index 29d1178..bc7b176 100644 --- a/arch/s390/configs/performance_defconfig +++ b/arch/s390/configs/performance_defconfig @@ -362,6 +362,8 @@ CONFIG_BPF_JIT=y CONFIG_NET_PKTGEN=m CONFIG_NET_TCPPROBE=m CONFIG_DEVTMPFS=y +CONFIG_DMA_CMA=y +CONFIG_CMA_SIZE_MBYTES=0 CONFIG_CONNECTOR=y CONFIG_BLK_DEV_LOOP=m CONFIG_BLK_DEV_CRYPTOLOOP=m @@ -434,7 +436,6 @@ CONFIG_TUN=m CONFIG_VETH=m CONFIG_VIRTIO_NET=m CONFIG_NLMON=m -CONFIG_VHOST_NET=m # CONFIG_NET_VENDOR_ARC is not set # CONFIG_NET_VENDOR_CHELSIO is not set # CONFIG_NET_VENDOR_INTEL is not set @@ -632,3 +633,4 @@ CONFIG_CMM=m CONFIG_APPLDATA_BASE=y CONFIG_KVM=m CONFIG_KVM_S390_UCONTROL=y +CONFIG_VHOST_NET=m diff --git a/arch/s390/crypto/prng.c b/arch/s390/crypto/prng.c index 9cc050f..1113389 100644 --- a/arch/s390/crypto/prng.c +++ b/arch/s390/crypto/prng.c @@ -507,8 +507,10 @@ static ssize_t prng_tdes_read(struct file *file, char __user *ubuf, prng_data->prngws.byte_counter += n; prng_data->prngws.reseed_counter += n; - if (copy_to_user(ubuf, prng_data->buf, chunk)) - return -EFAULT; + if (copy_to_user(ubuf, prng_data->buf, chunk)) { + ret = -EFAULT; + break; + } nbytes -= chunk; ret += chunk; diff --git a/arch/s390/hypfs/inode.c b/arch/s390/hypfs/inode.c index 09bccb2..cf8a2d9 100644 --- a/arch/s390/hypfs/inode.c +++ b/arch/s390/hypfs/inode.c @@ -3,6 +3,7 @@ * * Copyright IBM Corp. 2006, 2008 * Author(s): Michael Holzheu <holzheu@de.ibm.com> + * License: GPL */ #define KMSG_COMPONENT "hypfs" @@ -18,7 +19,8 @@ #include <linux/time.h> #include <linux/parser.h> #include <linux/sysfs.h> -#include <linux/module.h> +#include <linux/init.h> +#include <linux/kobject.h> #include <linux/seq_file.h> #include <linux/mount.h> #include <linux/uio.h> @@ -443,7 +445,6 @@ static struct file_system_type hypfs_type = { .mount = hypfs_mount, .kill_sb = hypfs_kill_super }; -MODULE_ALIAS_FS("s390_hypfs"); static const struct super_operations hypfs_s_ops = { .statfs = simple_statfs, @@ -497,21 +498,4 @@ fail_dbfs_exit: pr_err("Initialization of hypfs failed with rc=%i\n", rc); return rc; } - -static void __exit hypfs_exit(void) -{ - unregister_filesystem(&hypfs_type); - sysfs_remove_mount_point(hypervisor_kobj, "s390"); - hypfs_diag0c_exit(); - hypfs_sprp_exit(); - hypfs_vm_exit(); - hypfs_diag_exit(); - hypfs_dbfs_exit(); -} - -module_init(hypfs_init) -module_exit(hypfs_exit) - -MODULE_LICENSE("GPL"); -MODULE_AUTHOR("Michael Holzheu <holzheu@de.ibm.com>"); -MODULE_DESCRIPTION("s390 Hypervisor Filesystem"); +device_initcall(hypfs_init) diff --git a/arch/s390/include/asm/Kbuild b/arch/s390/include/asm/Kbuild index 20f196b..8aea32f 100644 --- a/arch/s390/include/asm/Kbuild +++ b/arch/s390/include/asm/Kbuild @@ -1,6 +1,6 @@ - - +generic-y += asm-offsets.h generic-y += clkdev.h +generic-y += dma-contiguous.h generic-y += export.h generic-y += irq_work.h generic-y += mcs_spinlock.h diff --git a/arch/s390/include/asm/asm-offsets.h b/arch/s390/include/asm/asm-offsets.h deleted file mode 100644 index d370ee3..0000000 --- a/arch/s390/include/asm/asm-offsets.h +++ /dev/null @@ -1 +0,0 @@ -#include <generated/asm-offsets.h> diff --git a/arch/s390/include/asm/atomic.h b/arch/s390/include/asm/atomic.h index d28cc2f..f7f69df 100644 --- a/arch/s390/include/asm/atomic.h +++ b/arch/s390/include/asm/atomic.h @@ -1,13 +1,8 @@ /* - * Copyright IBM Corp. 1999, 2009 + * Copyright IBM Corp. 1999, 2016 * Author(s): Martin Schwidefsky <schwidefsky@de.ibm.com>, * Denis Joseph Barrow, - * Arnd Bergmann <arndb@de.ibm.com>, - * - * Atomic operations that C can't guarantee us. - * Useful for resource counting etc. - * s390 uses 'Compare And Swap' for atomicity in SMP environment. - * + * Arnd Bergmann, */ #ifndef __ARCH_S390_ATOMIC__ @@ -15,62 +10,12 @@ #include <linux/compiler.h> #include <linux/types.h> +#include <asm/atomic_ops.h> #include <asm/barrier.h> #include <asm/cmpxchg.h> #define ATOMIC_INIT(i) { (i) } -#define __ATOMIC_NO_BARRIER "\n" - -#ifdef CONFIG_HAVE_MARCH_Z196_FEATURES - -#define __ATOMIC_OR "lao" -#define __ATOMIC_AND "lan" -#define __ATOMIC_ADD "laa" -#define __ATOMIC_XOR "lax" -#define __ATOMIC_BARRIER "bcr 14,0\n" - -#define __ATOMIC_LOOP(ptr, op_val, op_string, __barrier) \ -({ \ - int old_val; \ - \ - typecheck(atomic_t *, ptr); \ - asm volatile( \ - op_string " %0,%2,%1\n" \ - __barrier \ - : "=d" (old_val), "+Q" ((ptr)->counter) \ - : "d" (op_val) \ - : "cc", "memory"); \ - old_val; \ -}) - -#else /* CONFIG_HAVE_MARCH_Z196_FEATURES */ - -#define __ATOMIC_OR "or" -#define __ATOMIC_AND "nr" -#define __ATOMIC_ADD "ar" -#define __ATOMIC_XOR "xr" -#define __ATOMIC_BARRIER "\n" - -#define __ATOMIC_LOOP(ptr, op_val, op_string, __barrier) \ -({ \ - int old_val, new_val; \ - \ - typecheck(atomic_t *, ptr); \ - asm volatile( \ - " l %0,%2\n" \ - "0: lr %1,%0\n" \ - op_string " %1,%3\n" \ - " cs %0,%1,%2\n" \ - " jl 0b" \ - : "=&d" (old_val), "=&d" (new_val), "+Q" ((ptr)->counter)\ - : "d" (op_val) \ - : "cc", "memory"); \ - old_val; \ -}) - -#endif /* CONFIG_HAVE_MARCH_Z196_FEATURES */ - static inline int atomic_read(const atomic_t *v) { int c; @@ -90,27 +35,23 @@ static inline void atomic_set(atomic_t *v, int i) static inline int atomic_add_return(int i, atomic_t *v) { - return __ATOMIC_LOOP(v, i, __ATOMIC_ADD, __ATOMIC_BARRIER) + i; + return __atomic_add_barrier(i, &v->counter) + i; } static inline int atomic_fetch_add(int i, atomic_t *v) { - return __ATOMIC_LOOP(v, i, __ATOMIC_ADD, __ATOMIC_BARRIER); + return __atomic_add_barrier(i, &v->counter); } static inline void atomic_add(int i, atomic_t *v) { #ifdef CONFIG_HAVE_MARCH_Z196_FEATURES if (__builtin_constant_p(i) && (i > -129) && (i < 128)) { - asm volatile( - "asi %0,%1\n" - : "+Q" (v->counter) - : "i" (i) - : "cc", "memory"); + __atomic_add_const(i, &v->counter); return; } #endif - __ATOMIC_LOOP(v, i, __ATOMIC_ADD, __ATOMIC_NO_BARRIER); + __atomic_add(i, &v->counter); } #define atomic_add_negative(_i, _v) (atomic_add_return(_i, _v) < 0) @@ -125,19 +66,19 @@ static inline void atomic_add(int i, atomic_t *v) #define atomic_dec_return(_v) atomic_sub_return(1, _v) #define atomic_dec_and_test(_v) (atomic_sub_return(1, _v) == 0) -#define ATOMIC_OPS(op, OP) \ +#define ATOMIC_OPS(op) \ static inline void atomic_##op(int i, atomic_t *v) \ { \ - __ATOMIC_LOOP(v, i, __ATOMIC_##OP, __ATOMIC_NO_BARRIER); \ + __atomic_##op(i, &v->counter); \ } \ static inline int atomic_fetch_##op(int i, atomic_t *v) \ { \ - return __ATOMIC_LOOP(v, i, __ATOMIC_##OP, __ATOMIC_BARRIER); \ + return __atomic_##op##_barrier(i, &v->counter); \ } -ATOMIC_OPS(and, AND) -ATOMIC_OPS(or, OR) -ATOMIC_OPS(xor, XOR) +ATOMIC_OPS(and) +ATOMIC_OPS(or) +ATOMIC_OPS(xor) #undef ATOMIC_OPS @@ -145,12 +86,7 @@ ATOMIC_OPS(xor, XOR) static inline int atomic_cmpxchg(atomic_t *v, int old, int new) { - asm volatile( - " cs %0,%2,%1" - : "+d" (old), "+Q" (v->counter) - : "d" (new) - : "cc", "memory"); - return old; + return __atomic_cmpxchg(&v->counter, old, new); } static inline int __atomic_add_unless(atomic_t *v, int a, int u) @@ -168,65 +104,11 @@ static inline int __atomic_add_unless(atomic_t *v, int a, int u) return c; } - -#undef __ATOMIC_LOOP - #define ATOMIC64_INIT(i) { (i) } -#define __ATOMIC64_NO_BARRIER "\n" - -#ifdef CONFIG_HAVE_MARCH_Z196_FEATURES - -#define __ATOMIC64_OR "laog" -#define __ATOMIC64_AND "lang" -#define __ATOMIC64_ADD "laag" -#define __ATOMIC64_XOR "laxg" -#define __ATOMIC64_BARRIER "bcr 14,0\n" - -#define __ATOMIC64_LOOP(ptr, op_val, op_string, __barrier) \ -({ \ - long long old_val; \ - \ - typecheck(atomic64_t *, ptr); \ - asm volatile( \ - op_string " %0,%2,%1\n" \ - __barrier \ - : "=d" (old_val), "+Q" ((ptr)->counter) \ - : "d" (op_val) \ - : "cc", "memory"); \ - old_val; \ -}) - -#else /* CONFIG_HAVE_MARCH_Z196_FEATURES */ - -#define __ATOMIC64_OR "ogr" -#define __ATOMIC64_AND "ngr" -#define __ATOMIC64_ADD "agr" -#define __ATOMIC64_XOR "xgr" -#define __ATOMIC64_BARRIER "\n" - -#define __ATOMIC64_LOOP(ptr, op_val, op_string, __barrier) \ -({ \ - long long old_val, new_val; \ - \ - typecheck(atomic64_t *, ptr); \ - asm volatile( \ - " lg %0,%2\n" \ - "0: lgr %1,%0\n" \ - op_string " %1,%3\n" \ - " csg %0,%1,%2\n" \ - " jl 0b" \ - : "=&d" (old_val), "=&d" (new_val), "+Q" ((ptr)->counter)\ - : "d" (op_val) \ - : "cc", "memory"); \ - old_val; \ -}) - -#endif /* CONFIG_HAVE_MARCH_Z196_FEATURES */ - -static inline long long atomic64_read(const atomic64_t *v) +static inline long atomic64_read(const atomic64_t *v) { - long long c; + long c; asm volatile( " lg %0,%1\n" @@ -234,71 +116,60 @@ static inline long long atomic64_read(const atomic64_t *v) return c; } -static inline void atomic64_set(atomic64_t *v, long long i) +static inline void atomic64_set(atomic64_t *v, long i) { asm volatile( " stg %1,%0\n" : "=Q" (v->counter) : "d" (i)); } -static inline long long atomic64_add_return(long long i, atomic64_t *v) +static inline long atomic64_add_return(long i, atomic64_t *v) { - return __ATOMIC64_LOOP(v, i, __ATOMIC64_ADD, __ATOMIC64_BARRIER) + i; + return __atomic64_add_barrier(i, &v->counter) + i; } -static inline long long atomic64_fetch_add(long long i, atomic64_t *v) +static inline long atomic64_fetch_add(long i, atomic64_t *v) { - return __ATOMIC64_LOOP(v, i, __ATOMIC64_ADD, __ATOMIC64_BARRIER); + return __atomic64_add_barrier(i, &v->counter); } -static inline void atomic64_add(long long i, atomic64_t *v) +static inline void atomic64_add(long i, atomic64_t *v) { #ifdef CONFIG_HAVE_MARCH_Z196_FEATURES if (__builtin_constant_p(i) && (i > -129) && (i < 128)) { - asm volatile( - "agsi %0,%1\n" - : "+Q" (v->counter) - : "i" (i) - : "cc", "memory"); + __atomic64_add_const(i, &v->counter); return; } #endif - __ATOMIC64_LOOP(v, i, __ATOMIC64_ADD, __ATOMIC64_NO_BARRIER); + __atomic64_add(i, &v->counter); } #define atomic64_xchg(v, new) (xchg(&((v)->counter), new)) -static inline long long atomic64_cmpxchg(atomic64_t *v, - long long old, long long new) +static inline long atomic64_cmpxchg(atomic64_t *v, long old, long new) { - asm volatile( - " csg %0,%2,%1" - : "+d" (old), "+Q" (v->counter) - : "d" (new) - : "cc", "memory"); - return old; + return __atomic64_cmpxchg(&v->counter, old, new); } -#define ATOMIC64_OPS(op, OP) \ +#define ATOMIC64_OPS(op) \ static inline void atomic64_##op(long i, atomic64_t *v) \ { \ - __ATOMIC64_LOOP(v, i, __ATOMIC64_##OP, __ATOMIC64_NO_BARRIER); \ + __atomic64_##op(i, &v->counter); \ } \ static inline long atomic64_fetch_##op(long i, atomic64_t *v) \ { \ - return __ATOMIC64_LOOP(v, i, __ATOMIC64_##OP, __ATOMIC64_BARRIER); \ + return __atomic64_##op##_barrier(i, &v->counter); \ } -ATOMIC64_OPS(and, AND) -ATOMIC64_OPS(or, OR) -ATOMIC64_OPS(xor, XOR) +ATOMIC64_OPS(and) +ATOMIC64_OPS(or) +ATOMIC64_OPS(xor) #undef ATOMIC64_OPS -#undef __ATOMIC64_LOOP -static inline int atomic64_add_unless(atomic64_t *v, long long i, long long u) +static inline int atomic64_add_unless(atomic64_t *v, long i, long u) { - long long c, old; + long c, old; c = atomic64_read(v); for (;;) { @@ -312,9 +183,9 @@ static inline int atomic64_add_unless(atomic64_t *v, long long i, long long u) return c != u; } -static inline long long atomic64_dec_if_positive(atomic64_t *v) +static inline long atomic64_dec_if_positive(atomic64_t *v) { - long long c, old, dec; + long c, old, dec; c = atomic64_read(v); for (;;) { @@ -333,9 +204,9 @@ static inline long long atomic64_dec_if_positive(atomic64_t *v) #define atomic64_inc(_v) atomic64_add(1, _v) #define atomic64_inc_return(_v) atomic64_add_return(1, _v) #define atomic64_inc_and_test(_v) (atomic64_add_return(1, _v) == 0) -#define atomic64_sub_return(_i, _v) atomic64_add_return(-(long long)(_i), _v) -#define atomic64_fetch_sub(_i, _v) atomic64_fetch_add(-(long long)(_i), _v) -#define atomic64_sub(_i, _v) atomic64_add(-(long long)(_i), _v) +#define atomic64_sub_return(_i, _v) atomic64_add_return(-(long)(_i), _v) +#define atomic64_fetch_sub(_i, _v) atomic64_fetch_add(-(long)(_i), _v) +#define atomic64_sub(_i, _v) atomic64_add(-(long)(_i), _v) #define atomic64_sub_and_test(_i, _v) (atomic64_sub_return(_i, _v) == 0) #define atomic64_dec(_v) atomic64_sub(1, _v) #define atomic64_dec_return(_v) atomic64_sub_return(1, _v) diff --git a/arch/s390/include/asm/atomic_ops.h b/arch/s390/include/asm/atomic_ops.h new file mode 100644 index 0000000..ac9e2b9 --- /dev/null +++ b/arch/s390/include/asm/atomic_ops.h @@ -0,0 +1,130 @@ +/* + * Low level function for atomic operations + * + * Copyright IBM Corp. 1999, 2016 + */ + +#ifndef __ARCH_S390_ATOMIC_OPS__ +#define __ARCH_S390_ATOMIC_OPS__ + +#ifdef CONFIG_HAVE_MARCH_Z196_FEATURES + +#define __ATOMIC_OP(op_name, op_type, op_string, op_barrier) \ +static inline op_type op_name(op_type val, op_type *ptr) \ +{ \ + op_type old; \ + \ + asm volatile( \ + op_string " %[old],%[val],%[ptr]\n" \ + op_barrier \ + : [old] "=d" (old), [ptr] "+Q" (*ptr) \ + : [val] "d" (val) : "cc", "memory"); \ + return old; \ +} \ + +#define __ATOMIC_OPS(op_name, op_type, op_string) \ + __ATOMIC_OP(op_name, op_type, op_string, "\n") \ + __ATOMIC_OP(op_name##_barrier, op_type, op_string, "bcr 14,0\n") + +__ATOMIC_OPS(__atomic_add, int, "laa") +__ATOMIC_OPS(__atomic_and, int, "lan") +__ATOMIC_OPS(__atomic_or, int, "lao") +__ATOMIC_OPS(__atomic_xor, int, "lax") + +__ATOMIC_OPS(__atomic64_add, long, "laag") +__ATOMIC_OPS(__atomic64_and, long, "lang") +__ATOMIC_OPS(__atomic64_or, long, "laog") +__ATOMIC_OPS(__atomic64_xor, long, "laxg") + +#undef __ATOMIC_OPS +#undef __ATOMIC_OP + +static inline void __atomic_add_const(int val, int *ptr) +{ + asm volatile( + " asi %[ptr],%[val]\n" + : [ptr] "+Q" (*ptr) : [val] "i" (val) : "cc"); +} + +static inline void __atomic64_add_const(long val, long *ptr) +{ + asm volatile( + " agsi %[ptr],%[val]\n" + : [ptr] "+Q" (*ptr) : [val] "i" (val) : "cc"); +} + +#else /* CONFIG_HAVE_MARCH_Z196_FEATURES */ + +#define __ATOMIC_OP(op_name, op_string) \ +static inline int op_name(int val, int *ptr) \ +{ \ + int old, new; \ + \ + asm volatile( \ + "0: lr %[new],%[old]\n" \ + op_string " %[new],%[val]\n" \ + " cs %[old],%[new],%[ptr]\n" \ + " jl 0b" \ + : [old] "=d" (old), [new] "=&d" (new), [ptr] "+Q" (*ptr)\ + : [val] "d" (val), "0" (*ptr) : "cc", "memory"); \ + return old; \ +} + +#define __ATOMIC_OPS(op_name, op_string) \ + __ATOMIC_OP(op_name, op_string) \ + __ATOMIC_OP(op_name##_barrier, op_string) + +__ATOMIC_OPS(__atomic_add, "ar") +__ATOMIC_OPS(__atomic_and, "nr") +__ATOMIC_OPS(__atomic_or, "or") +__ATOMIC_OPS(__atomic_xor, "xr") + +#undef __ATOMIC_OPS + +#define __ATOMIC64_OP(op_name, op_string) \ +static inline long op_name(long val, long *ptr) \ +{ \ + long old, new; \ + \ + asm volatile( \ + "0: lgr %[new],%[old]\n" \ + op_string " %[new],%[val]\n" \ + " csg %[old],%[new],%[ptr]\n" \ + " jl 0b" \ + : [old] "=d" (old), [new] "=&d" (new), [ptr] "+Q" (*ptr)\ + : [val] "d" (val), "0" (*ptr) : "cc", "memory"); \ + return old; \ +} + +#define __ATOMIC64_OPS(op_name, op_string) \ + __ATOMIC64_OP(op_name, op_string) \ + __ATOMIC64_OP(op_name##_barrier, op_string) + +__ATOMIC64_OPS(__atomic64_add, "agr") +__ATOMIC64_OPS(__atomic64_and, "ngr") +__ATOMIC64_OPS(__atomic64_or, "ogr") +__ATOMIC64_OPS(__atomic64_xor, "xgr") + +#undef __ATOMIC64_OPS + +#endif /* CONFIG_HAVE_MARCH_Z196_FEATURES */ + +static inline int __atomic_cmpxchg(int *ptr, int old, int new) +{ + asm volatile( + " cs %[old],%[new],%[ptr]" + : [old] "+d" (old), [ptr] "+Q" (*ptr) + : [new] "d" (new) : "cc", "memory"); + return old; +} + +static inline long __atomic64_cmpxchg(long *ptr, long old, long new) +{ + asm volatile( + " csg %[old],%[new],%[ptr]" + : [old] "+d" (old), [ptr] "+Q" (*ptr) + : [new] "d" (new) : "cc", "memory"); + return old; +} + +#endif /* __ARCH_S390_ATOMIC_OPS__ */ diff --git a/arch/s390/include/asm/bitops.h b/arch/s390/include/asm/bitops.h index 8043f10..d92047d 100644 --- a/arch/s390/include/asm/bitops.h +++ b/arch/s390/include/asm/bitops.h @@ -42,57 +42,9 @@ #include <linux/typecheck.h> #include <linux/compiler.h> +#include <asm/atomic_ops.h> #include <asm/barrier.h> -#define __BITOPS_NO_BARRIER "\n" - -#ifdef CONFIG_HAVE_MARCH_Z196_FEATURES - -#define __BITOPS_OR "laog" -#define __BITOPS_AND "lang" -#define __BITOPS_XOR "laxg" -#define __BITOPS_BARRIER "bcr 14,0\n" - -#define __BITOPS_LOOP(__addr, __val, __op_string, __barrier) \ -({ \ - unsigned long __old; \ - \ - typecheck(unsigned long *, (__addr)); \ - asm volatile( \ - __op_string " %0,%2,%1\n" \ - __barrier \ - : "=d" (__old), "+Q" (*(__addr)) \ - : "d" (__val) \ - : "cc", "memory"); \ - __old; \ -}) - -#else /* CONFIG_HAVE_MARCH_Z196_FEATURES */ - -#define __BITOPS_OR "ogr" -#define __BITOPS_AND "ngr" -#define __BITOPS_XOR "xgr" -#define __BITOPS_BARRIER "\n" - -#define __BITOPS_LOOP(__addr, __val, __op_string, __barrier) \ -({ \ - unsigned long __old, __new; \ - \ - typecheck(unsigned long *, (__addr)); \ - asm volatile( \ - " lg %0,%2\n" \ - "0: lgr %1,%0\n" \ - __op_string " %1,%3\n" \ - " csg %0,%1,%2\n" \ - " jl 0b" \ - : "=&d" (__old), "=&d" (__new), "+Q" (*(__addr))\ - : "d" (__val) \ - : "cc", "memory"); \ - __old; \ -}) - -#endif /* CONFIG_HAVE_MARCH_Z196_FEATURES */ - #define __BITOPS_WORDS(bits) (((bits) + BITS_PER_LONG - 1) / BITS_PER_LONG) static inline unsigned long * @@ -128,7 +80,7 @@ static inline void set_bit(unsigned long nr, volatile unsigned long *ptr) } #endif mask = 1UL << (nr & (BITS_PER_LONG - 1)); - __BITOPS_LOOP(addr, mask, __BITOPS_OR, __BITOPS_NO_BARRIER); + __atomic64_or(mask, addr); } static inline void clear_bit(unsigned long nr, volatile unsigned long *ptr) @@ -149,7 +101,7 @@ static inline void clear_bit(unsigned long nr, volatile unsigned long *ptr) } #endif mask = ~(1UL << (nr & (BITS_PER_LONG - 1))); - __BITOPS_LOOP(addr, mask, __BITOPS_AND, __BITOPS_NO_BARRIER); + __atomic64_and(mask, addr); } static inline void change_bit(unsigned long nr, volatile unsigned long *ptr) @@ -170,7 +122,7 @@ static inline void change_bit(unsigned long nr, volatile unsigned long *ptr) } #endif mask = 1UL << (nr & (BITS_PER_LONG - 1)); - __BITOPS_LOOP(addr, mask, __BITOPS_XOR, __BITOPS_NO_BARRIER); + __atomic64_xor(mask, addr); } static inline int @@ -180,7 +132,7 @@ test_and_set_bit(unsigned long nr, volatile unsigned long *ptr) unsigned long old, mask; mask = 1UL << (nr & (BITS_PER_LONG - 1)); - old = __BITOPS_LOOP(addr, mask, __BITOPS_OR, __BITOPS_BARRIER); + old = __atomic64_or_barrier(mask, addr); return (old & mask) != 0; } @@ -191,7 +143,7 @@ test_and_clear_bit(unsigned long nr, volatile unsigned long *ptr) unsigned long old, mask; mask = ~(1UL << (nr & (BITS_PER_LONG - 1))); - old = __BITOPS_LOOP(addr, mask, __BITOPS_AND, __BITOPS_BARRIER); + old = __atomic64_and_barrier(mask, addr); return (old & ~mask) != 0; } @@ -202,7 +154,7 @@ test_and_change_bit(unsigned long nr, volatile unsigned long *ptr) unsigned long old, mask; mask = 1UL << (nr & (BITS_PER_LONG - 1)); - old = __BITOPS_LOOP(addr, mask, __BITOPS_XOR, __BITOPS_BARRIER); + old = __atomic64_xor_barrier(mask, addr); return (old & mask) != 0; } diff --git a/arch/s390/include/asm/cpu_mf.h b/arch/s390/include/asm/cpu_mf.h index 0351647..b69d8bc 100644 --- a/arch/s390/include/asm/cpu_mf.h +++ b/arch/s390/include/asm/cpu_mf.h @@ -104,7 +104,8 @@ struct hws_basic_entry { unsigned int P:1; /* 28 PSW Problem state */ unsigned int AS:2; /* 29-30 PSW address-space control */ unsigned int I:1; /* 31 entry valid or invalid */ - unsigned int:16; + unsigned int CL:2; /* 32-33 Configuration Level */ + unsigned int:14; unsigned int prim_asn:16; /* primary ASN */ unsigned long long ia; /* Instruction Address */ unsigned long long gpp; /* Guest Program Parameter */ diff --git a/arch/s390/include/asm/elf.h b/arch/s390/include/asm/elf.h index 1736c7d..f4381e1 100644 --- a/arch/s390/include/asm/elf.h +++ b/arch/s390/include/asm/elf.h @@ -193,7 +193,7 @@ extern char elf_platform[]; do { \ set_personality(PER_LINUX | \ (current->personality & (~PER_MASK))); \ - current_thread_info()->sys_call_table = \ + current->thread.sys_call_table = \ (unsigned long) &sys_call_table; \ } while (0) #else /* CONFIG_COMPAT */ @@ -204,11 +204,11 @@ do { \ (current->personality & ~PER_MASK)); \ if ((ex).e_ident[EI_CLASS] == ELFCLASS32) { \ set_thread_flag(TIF_31BIT); \ - current_thread_info()->sys_call_table = \ + current->thread.sys_call_table = \ (unsigned long) &sys_call_table_emu; \ } else { \ clear_thread_flag(TIF_31BIT); \ - current_thread_info()->sys_call_table = \ + current->thread.sys_call_table = \ (unsigned long) &sys_call_table; \ } \ } while (0) diff --git a/arch/s390/include/asm/facilities_src.h b/arch/s390/include/asm/facilities_src.h deleted file mode 100644 index 3b758f6..0000000 --- a/arch/s390/include/asm/facilities_src.h +++ /dev/null @@ -1,82 +0,0 @@ -/* - * Copyright IBM Corp. 2015 - */ - -#ifndef S390_GEN_FACILITIES_C -#error "This file can only be included by gen_facilities.c" -#endif - -#include <linux/kconfig.h> - -struct facility_def { - char *name; - int *bits; -}; - -static struct facility_def facility_defs[] = { - { - /* - * FACILITIES_ALS contains the list of facilities that are - * required to run a kernel that is compiled e.g. with - * -march=<machine>. - */ - .name = "FACILITIES_ALS", - .bits = (int[]){ -#ifdef CONFIG_HAVE_MARCH_Z900_FEATURES - 0, /* N3 instructions */ - 1, /* z/Arch mode installed */ -#endif -#ifdef CONFIG_HAVE_MARCH_Z990_FEATURES - 18, /* long displacement facility */ -#endif -#ifdef CONFIG_HAVE_MARCH_Z9_109_FEATURES - 7, /* stfle */ - 17, /* message security assist */ - 21, /* extended-immediate facility */ - 25, /* store clock fast */ -#endif -#ifdef CONFIG_HAVE_MARCH_Z10_FEATURES - 27, /* mvcos */ - 32, /* compare and swap and store */ - 33, /* compare and swap and store 2 */ - 34, /* general extension facility */ - 35, /* execute extensions */ -#endif -#ifdef CONFIG_HAVE_MARCH_Z196_FEATURES - 45, /* fast-BCR, etc. */ -#endif -#ifdef CONFIG_HAVE_MARCH_ZEC12_FEATURES - 49, /* misc-instruction-extensions */ - 52, /* interlocked facility 2 */ -#endif -#ifdef CONFIG_HAVE_MARCH_Z13_FEATURES - 53, /* load-and-zero-rightmost-byte, etc. */ -#endif - -1 /* END */ - } - }, - { - .name = "FACILITIES_KVM", - .bits = (int[]){ - 0, /* N3 instructions */ - 1, /* z/Arch mode installed */ - 2, /* z/Arch mode active */ - 3, /* DAT-enhancement */ - 4, /* idte segment table */ - 5, /* idte region table */ - 6, /* ASN-and-LX reuse */ - 7, /* stfle */ - 8, /* enhanced-DAT 1 */ - 9, /* sense-running-status */ - 10, /* conditional sske */ - 13, /* ipte-range */ - 14, /* nonquiescing key-setting */ - 73, /* transactional execution */ - 75, /* access-exception-fetch/store indication */ - 76, /* msa extension 3 */ - 77, /* msa extension 4 */ - 78, /* enhanced-DAT 2 */ - -1 /* END */ - } - }, -}; diff --git a/arch/s390/include/asm/ipl.h b/arch/s390/include/asm/ipl.h index 4da22b2..edb5161 100644 --- a/arch/s390/include/asm/ipl.h +++ b/arch/s390/include/asm/ipl.h @@ -97,7 +97,7 @@ void __init save_area_add_vxrs(struct save_area *, __vector128 *vxrs); extern void do_reipl(void); extern void do_halt(void); extern void do_poff(void); -extern void ipl_save_parameters(void); +extern void ipl_verify_parameters(void); extern void ipl_update_parameters(void); extern size_t append_ipl_vmparm(char *, size_t); extern size_t append_ipl_scpdata(char *, size_t); diff --git a/arch/s390/include/asm/lowcore.h b/arch/s390/include/asm/lowcore.h index 7b93b78..9bfad2a 100644 --- a/arch/s390/include/asm/lowcore.h +++ b/arch/s390/include/asm/lowcore.h @@ -95,7 +95,7 @@ struct lowcore { /* Current process. */ __u64 current_task; /* 0x0310 */ - __u64 thread_info; /* 0x0318 */ + __u8 pad_0x318[0x320-0x318]; /* 0x0318 */ __u64 kernel_stack; /* 0x0320 */ /* Interrupt, panic and restart stack. */ @@ -126,7 +126,8 @@ struct lowcore { __u64 percpu_offset; /* 0x0378 */ __u64 vdso_per_cpu_data; /* 0x0380 */ __u64 machine_flags; /* 0x0388 */ - __u8 pad_0x0390[0x0398-0x0390]; /* 0x0390 */ + __u32 preempt_count; /* 0x0390 */ + __u8 pad_0x0394[0x0398-0x0394]; /* 0x0394 */ __u64 gmap; /* 0x0398 */ __u32 spinlock_lockval; /* 0x03a0 */ __u32 fpu_flags; /* 0x03a4 */ diff --git a/arch/s390/include/asm/pci_clp.h b/arch/s390/include/asm/pci_clp.h index e75c64cb..c232ef9 100644 --- a/arch/s390/include/asm/pci_clp.h +++ b/arch/s390/include/asm/pci_clp.h @@ -46,6 +46,8 @@ struct clp_fh_list_entry { #define CLP_UTIL_STR_LEN 64 #define CLP_PFIP_NR_SEGMENTS 4 +extern bool zpci_unique_uid; + /* List PCI functions request */ struct clp_req_list_pci { struct clp_req_hdr hdr; @@ -59,7 +61,8 @@ struct clp_rsp_list_pci { u64 resume_token; u32 reserved2; u16 max_fn; - u8 reserved3; + u8 : 7; + u8 uid_checking : 1; u8 entry_size; struct clp_fh_list_entry fh_list[CLP_FH_LIST_NR_ENTRIES]; } __packed; diff --git a/arch/s390/include/asm/pgalloc.h b/arch/s390/include/asm/pgalloc.h index f4eb984..166f703 100644 --- a/arch/s390/include/asm/pgalloc.h +++ b/arch/s390/include/asm/pgalloc.h @@ -27,17 +27,17 @@ extern int page_table_allocate_pgste; static inline void clear_table(unsigned long *s, unsigned long val, size_t n) { - typedef struct { char _[n]; } addrtype; - - *s = val; - n = (n / 256) - 1; - asm volatile( - " mvc 8(248,%0),0(%0)\n" - "0: mvc 256(256,%0),0(%0)\n" - " la %0,256(%0)\n" - " brct %1,0b\n" - : "+a" (s), "+d" (n), "=m" (*(addrtype *) s) - : "m" (*(addrtype *) s)); + struct addrtype { char _[256]; }; + int i; + + for (i = 0; i < n; i += 256) { + *s = val; + asm volatile( + "mvc 8(248,%[s]),0(%[s])\n" + : "+m" (*(struct addrtype *) s) + : [s] "a" (s)); + s += 256 / sizeof(long); + } } static inline void crst_table_init(unsigned long *crst, unsigned long entry) diff --git a/arch/s390/include/asm/preempt.h b/arch/s390/include/asm/preempt.h new file mode 100644 index 0000000..b0776b2 --- /dev/null +++ b/arch/s390/include/asm/preempt.h @@ -0,0 +1,137 @@ +#ifndef __ASM_PREEMPT_H +#define __ASM_PREEMPT_H + +#include <asm/current.h> +#include <linux/thread_info.h> +#include <asm/atomic_ops.h> + +#ifdef CONFIG_HAVE_MARCH_Z196_FEATURES + +#define PREEMPT_ENABLED (0 + PREEMPT_NEED_RESCHED) + +static inline int preempt_count(void) +{ + return READ_ONCE(S390_lowcore.preempt_count) & ~PREEMPT_NEED_RESCHED; +} + +static inline void preempt_count_set(int pc) +{ + int old, new; + + do { + old = READ_ONCE(S390_lowcore.preempt_count); + new = (old & PREEMPT_NEED_RESCHED) | + (pc & ~PREEMPT_NEED_RESCHED); + } while (__atomic_cmpxchg(&S390_lowcore.preempt_count, + old, new) != old); +} + +#define init_task_preempt_count(p) do { } while (0) + +#define init_idle_preempt_count(p, cpu) do { \ + S390_lowcore.preempt_count = PREEMPT_ENABLED; \ +} while (0) + +static inline void set_preempt_need_resched(void) +{ + __atomic_and(~PREEMPT_NEED_RESCHED, &S390_lowcore.preempt_count); +} + +static inline void clear_preempt_need_resched(void) +{ + __atomic_or(PREEMPT_NEED_RESCHED, &S390_lowcore.preempt_count); +} + +static inline bool test_preempt_need_resched(void) +{ + return !(READ_ONCE(S390_lowcore.preempt_count) & PREEMPT_NEED_RESCHED); +} + +static inline void __preempt_count_add(int val) +{ + if (__builtin_constant_p(val) && (val >= -128) && (val <= 127)) + __atomic_add_const(val, &S390_lowcore.preempt_count); + else + __atomic_add(val, &S390_lowcore.preempt_count); +} + +static inline void __preempt_count_sub(int val) +{ + __preempt_count_add(-val); +} + +static inline bool __preempt_count_dec_and_test(void) +{ + return __atomic_add(-1, &S390_lowcore.preempt_count) == 1; +} + +static inline bool should_resched(int preempt_offset) +{ + return unlikely(READ_ONCE(S390_lowcore.preempt_count) == + preempt_offset); +} + +#else /* CONFIG_HAVE_MARCH_Z196_FEATURES */ + +#define PREEMPT_ENABLED (0) + +static inline int preempt_count(void) +{ + return READ_ONCE(S390_lowcore.preempt_count); +} + +static inline void preempt_count_set(int pc) +{ + S390_lowcore.preempt_count = pc; +} + +#define init_task_preempt_count(p) do { } while (0) + +#define init_idle_preempt_count(p, cpu) do { \ + S390_lowcore.preempt_count = PREEMPT_ENABLED; \ +} while (0) + +static inline void set_preempt_need_resched(void) +{ +} + +static inline void clear_preempt_need_resched(void) +{ +} + +static inline bool test_preempt_need_resched(void) +{ + return false; +} + +static inline void __preempt_count_add(int val) +{ + S390_lowcore.preempt_count += val; +} + +static inline void __preempt_count_sub(int val) +{ + S390_lowcore.preempt_count -= val; +} + +static inline bool __preempt_count_dec_and_test(void) +{ + return !--S390_lowcore.preempt_count && tif_need_resched(); +} + +static inline bool should_resched(int preempt_offset) +{ + return unlikely(preempt_count() == preempt_offset && + tif_need_resched()); +} + +#endif /* CONFIG_HAVE_MARCH_Z196_FEATURES */ + +#ifdef CONFIG_PREEMPT +extern asmlinkage void preempt_schedule(void); +#define __preempt_schedule() preempt_schedule() +extern asmlinkage void preempt_schedule_notrace(void); +#define __preempt_schedule_notrace() preempt_schedule_notrace() +#endif /* CONFIG_PREEMPT */ + +#endif /* __ASM_PREEMPT_H */ diff --git a/arch/s390/include/asm/processor.h b/arch/s390/include/asm/processor.h index 9d3a21a..6bca916 100644 --- a/arch/s390/include/asm/processor.h +++ b/arch/s390/include/asm/processor.h @@ -110,14 +110,20 @@ typedef struct { struct thread_struct { unsigned int acrs[NUM_ACRS]; unsigned long ksp; /* kernel stack pointer */ + unsigned long user_timer; /* task cputime in user space */ + unsigned long system_timer; /* task cputime in kernel space */ + unsigned long sys_call_table; /* system call table address */ mm_segment_t mm_segment; unsigned long gmap_addr; /* address of last gmap fault. */ unsigned int gmap_write_flag; /* gmap fault write indication */ unsigned int gmap_int_code; /* int code of last gmap fault */ unsigned int gmap_pfault; /* signal of a pending guest pfault */ + /* Per-thread information related to debugging */ struct per_regs per_user; /* User specified PER registers */ struct per_event per_event; /* Cause of the last PER trap */ unsigned long per_flags; /* Flags to control debug behavior */ + unsigned int system_call; /* system call number in signal */ + unsigned long last_break; /* last breaking-event-address. */ /* pfault_wait is used to block the process on a pfault event */ unsigned long pfault_wait; struct list_head list; diff --git a/arch/s390/include/asm/sclp.h b/arch/s390/include/asm/sclp.h index 2ad9c20..8db92a5 100644 --- a/arch/s390/include/asm/sclp.h +++ b/arch/s390/include/asm/sclp.h @@ -101,7 +101,8 @@ struct zpci_report_error_header { u8 data[0]; /* Subsequent Data passed verbatim to SCLP ET 24 */ } __packed; -int sclp_get_core_info(struct sclp_core_info *info); +int _sclp_get_core_info_early(struct sclp_core_info *info); +int _sclp_get_core_info(struct sclp_core_info *info); int sclp_core_configure(u8 core); int sclp_core_deconfigure(u8 core); int sclp_sdias_blk_count(void); @@ -119,4 +120,11 @@ void sclp_early_detect(void); void _sclp_print_early(const char *); void sclp_ocf_cpc_name_copy(char *dst); +static inline int sclp_get_core_info(struct sclp_core_info *info, int early) +{ + if (early) + return _sclp_get_core_info_early(info); + return _sclp_get_core_info(info); +} + #endif /* _ASM_S390_SCLP_H */ diff --git a/arch/s390/include/asm/scsw.h b/arch/s390/include/asm/scsw.h index 4af99cd..17a7904f0 100644 --- a/arch/s390/include/asm/scsw.h +++ b/arch/s390/include/asm/scsw.h @@ -96,7 +96,8 @@ struct tm_scsw { u32 dstat:8; u32 cstat:8; u32 fcxs:8; - u32 schxs:8; + u32 ifob:1; + u32 sesq:7; } __attribute__ ((packed)); /** @@ -177,6 +178,9 @@ union scsw { #define SCHN_STAT_INTF_CTRL_CHK 0x02 #define SCHN_STAT_CHAIN_CHECK 0x01 +#define SCSW_SESQ_DEV_NOFCX 3 +#define SCSW_SESQ_PATH_NOFCX 4 + /* * architectured values for first sense byte */ diff --git a/arch/s390/include/asm/smp.h b/arch/s390/include/asm/smp.h index 0cc383b..3deb134 100644 --- a/arch/s390/include/asm/smp.h +++ b/arch/s390/include/asm/smp.h @@ -36,6 +36,7 @@ extern void smp_yield_cpu(int cpu); extern void smp_cpu_set_polarization(int cpu, int val); extern int smp_cpu_get_polarization(int cpu); extern void smp_fill_possible_mask(void); +extern void smp_detect_cpus(void); #else /* CONFIG_SMP */ @@ -56,6 +57,7 @@ static inline int smp_store_status(int cpu) { return 0; } static inline int smp_vcpu_scheduled(int cpu) { return 1; } static inline void smp_yield_cpu(int cpu) { } static inline void smp_fill_possible_mask(void) { } +static inline void smp_detect_cpus(void) { } #endif /* CONFIG_SMP */ @@ -69,6 +71,12 @@ static inline void smp_stop_cpu(void) } } +/* Return thread 0 CPU number as base CPU */ +static inline int smp_get_base_cpu(int cpu) +{ + return cpu - (cpu % (smp_cpu_mtid + 1)); +} + #ifdef CONFIG_HOTPLUG_CPU extern int smp_rescan_cpus(void); extern void __noreturn cpu_die(void); diff --git a/arch/s390/include/asm/string.h b/arch/s390/include/asm/string.h index 8662f5c..15a3c00 100644 --- a/arch/s390/include/asm/string.h +++ b/arch/s390/include/asm/string.h @@ -14,6 +14,7 @@ #define __HAVE_ARCH_MEMCHR /* inline & arch function */ #define __HAVE_ARCH_MEMCMP /* arch function */ #define __HAVE_ARCH_MEMCPY /* gcc builtin & arch function */ +#define __HAVE_ARCH_MEMMOVE /* gcc builtin & arch function */ #define __HAVE_ARCH_MEMSCAN /* inline & arch function */ #define __HAVE_ARCH_MEMSET /* gcc builtin & arch function */ #define __HAVE_ARCH_STRCAT /* inline & arch function */ @@ -32,6 +33,7 @@ extern int memcmp(const void *, const void *, size_t); extern void *memcpy(void *, const void *, size_t); extern void *memset(void *, int, size_t); +extern void *memmove(void *, const void *, size_t); extern int strcmp(const char *,const char *); extern size_t strlcat(char *, const char *, size_t); extern size_t strlcpy(char *, const char *, size_t); @@ -40,7 +42,6 @@ extern char *strncpy(char *, const char *, size_t); extern char *strrchr(const char *, int); extern char *strstr(const char *, const char *); -#undef __HAVE_ARCH_MEMMOVE #undef __HAVE_ARCH_STRCHR #undef __HAVE_ARCH_STRNCHR #undef __HAVE_ARCH_STRNCMP diff --git a/arch/s390/include/asm/sysinfo.h b/arch/s390/include/asm/sysinfo.h index 2728114..229326c 100644 --- a/arch/s390/include/asm/sysinfo.h +++ b/arch/s390/include/asm/sysinfo.h @@ -107,6 +107,11 @@ struct sysinfo_2_2_2 { char reserved_3[5]; unsigned short cpus_dedicated; unsigned short cpus_shared; + char reserved_4[3]; + unsigned char vsne; + uuid_be uuid; + char reserved_5[160]; + char ext_name[256]; }; #define LPAR_CHAR_DEDICATED (1 << 7) @@ -127,7 +132,7 @@ struct sysinfo_3_2_2 { unsigned int caf; char cpi[16]; char reserved_1[3]; - char ext_name_encoding; + unsigned char evmne; unsigned int reserved_2; uuid_be uuid; } vm[8]; diff --git a/arch/s390/include/asm/thread_info.h b/arch/s390/include/asm/thread_info.h index f15c039..a5b54a4 100644 --- a/arch/s390/include/asm/thread_info.h +++ b/arch/s390/include/asm/thread_info.h @@ -12,10 +12,10 @@ /* * Size of kernel stack for each process */ -#define THREAD_ORDER 2 +#define THREAD_SIZE_ORDER 2 #define ASYNC_ORDER 2 -#define THREAD_SIZE (PAGE_SIZE << THREAD_ORDER) +#define THREAD_SIZE (PAGE_SIZE << THREAD_SIZE_ORDER) #define ASYNC_SIZE (PAGE_SIZE << ASYNC_ORDER) #ifndef __ASSEMBLY__ @@ -30,15 +30,7 @@ * - if the contents of this structure are changed, the assembly constants must also be changed */ struct thread_info { - struct task_struct *task; /* main task structure */ unsigned long flags; /* low level flags */ - unsigned long sys_call_table; /* System call table address */ - unsigned int cpu; /* current CPU */ - int preempt_count; /* 0 => preemptable, <0 => BUG */ - unsigned int system_call; - __u64 user_timer; - __u64 system_timer; - unsigned long last_break; /* last breaking-event-address. */ }; /* @@ -46,26 +38,14 @@ struct thread_info { */ #define INIT_THREAD_INFO(tsk) \ { \ - .task = &tsk, \ .flags = 0, \ - .cpu = 0, \ - .preempt_count = INIT_PREEMPT_COUNT, \ } -#define init_thread_info (init_thread_union.thread_info) #define init_stack (init_thread_union.stack) -/* how to get the thread information struct from C */ -static inline struct thread_info *current_thread_info(void) -{ - return (struct thread_info *) S390_lowcore.thread_info; -} - void arch_release_task_struct(struct task_struct *tsk); int arch_dup_task_struct(struct task_struct *dst, struct task_struct *src); -#define THREAD_SIZE_ORDER THREAD_ORDER - #endif /* diff --git a/arch/s390/include/asm/timex.h b/arch/s390/include/asm/timex.h index 0bb08f3..de82988 100644 --- a/arch/s390/include/asm/timex.h +++ b/arch/s390/include/asm/timex.h @@ -52,11 +52,9 @@ static inline void store_clock_comparator(__u64 *time) void clock_comparator_work(void); -void __init ptff_init(void); +void __init time_early_init(void); extern unsigned char ptff_function_mask[16]; -extern unsigned long lpar_offset; -extern unsigned long initial_leap_seconds; /* Function codes for the ptff instruction. */ #define PTFF_QAF 0x00 /* query available functions */ @@ -100,21 +98,28 @@ struct ptff_qui { unsigned int pad_0x5c[41]; } __packed; -static inline int ptff(void *ptff_block, size_t len, unsigned int func) -{ - typedef struct { char _[len]; } addrtype; - register unsigned int reg0 asm("0") = func; - register unsigned long reg1 asm("1") = (unsigned long) ptff_block; - int rc; - - asm volatile( - " .word 0x0104\n" - " ipm %0\n" - " srl %0,28\n" - : "=d" (rc), "+m" (*(addrtype *) ptff_block) - : "d" (reg0), "d" (reg1) : "cc"); - return rc; -} +/* + * ptff - Perform timing facility function + * @ptff_block: Pointer to ptff parameter block + * @len: Length of parameter block + * @func: Function code + * Returns: Condition code (0 on success) + */ +#define ptff(ptff_block, len, func) \ +({ \ + struct addrtype { char _[len]; }; \ + register unsigned int reg0 asm("0") = func; \ + register unsigned long reg1 asm("1") = (unsigned long) (ptff_block);\ + int rc; \ + \ + asm volatile( \ + " .word 0x0104\n" \ + " ipm %0\n" \ + " srl %0,28\n" \ + : "=d" (rc), "+m" (*(struct addrtype *) reg1) \ + : "d" (reg0), "d" (reg1) : "cc"); \ + rc; \ +}) static inline unsigned long long local_tick_disable(void) { diff --git a/arch/s390/include/asm/topology.h b/arch/s390/include/asm/topology.h index f15f557..fa1bfce 100644 --- a/arch/s390/include/asm/topology.h +++ b/arch/s390/include/asm/topology.h @@ -22,21 +22,22 @@ struct cpu_topology_s390 { cpumask_t drawer_mask; }; -DECLARE_PER_CPU(struct cpu_topology_s390, cpu_topology); - -#define topology_physical_package_id(cpu) (per_cpu(cpu_topology, cpu).socket_id) -#define topology_thread_id(cpu) (per_cpu(cpu_topology, cpu).thread_id) -#define topology_sibling_cpumask(cpu) \ - (&per_cpu(cpu_topology, cpu).thread_mask) -#define topology_core_id(cpu) (per_cpu(cpu_topology, cpu).core_id) -#define topology_core_cpumask(cpu) (&per_cpu(cpu_topology, cpu).core_mask) -#define topology_book_id(cpu) (per_cpu(cpu_topology, cpu).book_id) -#define topology_book_cpumask(cpu) (&per_cpu(cpu_topology, cpu).book_mask) -#define topology_drawer_id(cpu) (per_cpu(cpu_topology, cpu).drawer_id) -#define topology_drawer_cpumask(cpu) (&per_cpu(cpu_topology, cpu).drawer_mask) +extern struct cpu_topology_s390 cpu_topology[NR_CPUS]; +extern cpumask_t cpus_with_topology; + +#define topology_physical_package_id(cpu) (cpu_topology[cpu].socket_id) +#define topology_thread_id(cpu) (cpu_topology[cpu].thread_id) +#define topology_sibling_cpumask(cpu) (&cpu_topology[cpu].thread_mask) +#define topology_core_id(cpu) (cpu_topology[cpu].core_id) +#define topology_core_cpumask(cpu) (&cpu_topology[cpu].core_mask) +#define topology_book_id(cpu) (cpu_topology[cpu].book_id) +#define topology_book_cpumask(cpu) (&cpu_topology[cpu].book_mask) +#define topology_drawer_id(cpu) (cpu_topology[cpu].drawer_id) +#define topology_drawer_cpumask(cpu) (&cpu_topology[cpu].drawer_mask) #define mc_capable() 1 +void topology_init_early(void); int topology_cpu_init(struct cpu *); int topology_set_cpu_management(int fc); void topology_schedule_update(void); @@ -46,6 +47,7 @@ const struct cpumask *cpu_coregroup_mask(int cpu); #else /* CONFIG_SCHED_TOPOLOGY */ +static inline void topology_init_early(void) { } static inline void topology_schedule_update(void) { } static inline int topology_cpu_init(struct cpu *cpu) { return 0; } static inline void topology_expect_change(void) { } @@ -65,7 +67,7 @@ static inline void topology_expect_change(void) { } #define cpu_to_node cpu_to_node static inline int cpu_to_node(int cpu) { - return per_cpu(cpu_topology, cpu).node_id; + return cpu_topology[cpu].node_id; } /* Returns a pointer to the cpumask of CPUs on node 'node'. */ diff --git a/arch/s390/include/asm/uaccess.h b/arch/s390/include/asm/uaccess.h index 52d7c87..f82b04e 100644 --- a/arch/s390/include/asm/uaccess.h +++ b/arch/s390/include/asm/uaccess.h @@ -37,14 +37,14 @@ #define get_ds() (KERNEL_DS) #define get_fs() (current->thread.mm_segment) -#define set_fs(x) \ -({ \ +#define set_fs(x) \ +{ \ unsigned long __pto; \ current->thread.mm_segment = (x); \ __pto = current->thread.mm_segment.ar4 ? \ S390_lowcore.user_asce : S390_lowcore.kernel_asce; \ __ctl_load(__pto, 7, 7); \ -}) +} #define segment_eq(a,b) ((a).ar4 == (b).ar4) diff --git a/arch/s390/include/asm/vdso.h b/arch/s390/include/asm/vdso.h index d0a2dbf..88bdc47 100644 --- a/arch/s390/include/asm/vdso.h +++ b/arch/s390/include/asm/vdso.h @@ -33,6 +33,8 @@ struct vdso_data { __u32 ectg_available; /* ECTG instruction present 0x58 */ __u32 tk_mult; /* Mult. used for xtime_nsec 0x5c */ __u32 tk_shift; /* Shift used for xtime_nsec 0x60 */ + __u32 ts_dir; /* TOD steering direction 0x64 */ + __u64 ts_end; /* TOD steering end 0x68 */ }; struct vdso_per_cpu_data { diff --git a/arch/s390/include/uapi/asm/Kbuild b/arch/s390/include/uapi/asm/Kbuild index cc44b09..bf736e7 100644 --- a/arch/s390/include/uapi/asm/Kbuild +++ b/arch/s390/include/uapi/asm/Kbuild @@ -12,6 +12,7 @@ header-y += dasd.h header-y += debug.h header-y += errno.h header-y += fcntl.h +header-y += hypfs.h header-y += ioctl.h header-y += ioctls.h header-y += ipcbuf.h @@ -29,16 +30,16 @@ header-y += ptrace.h header-y += qeth.h header-y += resource.h header-y += schid.h +header-y += sclp_ctl.h header-y += sembuf.h header-y += setup.h header-y += shmbuf.h +header-y += sie.h header-y += sigcontext.h header-y += siginfo.h header-y += signal.h header-y += socket.h header-y += sockios.h -header-y += sclp_ctl.h -header-y += sie.h header-y += stat.h header-y += statfs.h header-y += swab.h diff --git a/arch/s390/kernel/Makefile b/arch/s390/kernel/Makefile index 1f0fe98..36b5101 100644 --- a/arch/s390/kernel/Makefile +++ b/arch/s390/kernel/Makefile @@ -2,20 +2,47 @@ # Makefile for the linux kernel. # -KCOV_INSTRUMENT_early.o := n -KCOV_INSTRUMENT_sclp.o := n -KCOV_INSTRUMENT_als.o := n - ifdef CONFIG_FUNCTION_TRACER -# Don't trace early setup code and tracing code -CFLAGS_REMOVE_early.o = $(CC_FLAGS_FTRACE) -CFLAGS_REMOVE_ftrace.o = $(CC_FLAGS_FTRACE) + +# Do not trace tracer code +CFLAGS_REMOVE_ftrace.o = $(CC_FLAGS_FTRACE) + +# Do not trace early setup code +CFLAGS_REMOVE_als.o = $(CC_FLAGS_FTRACE) +CFLAGS_REMOVE_early.o = $(CC_FLAGS_FTRACE) +CFLAGS_REMOVE_sclp.o = $(CC_FLAGS_FTRACE) + +endif + +GCOV_PROFILE_als.o := n +GCOV_PROFILE_early.o := n +GCOV_PROFILE_sclp.o := n + +KCOV_INSTRUMENT_als.o := n +KCOV_INSTRUMENT_early.o := n +KCOV_INSTRUMENT_sclp.o := n + +UBSAN_SANITIZE_als.o := n +UBSAN_SANITIZE_early.o := n +UBSAN_SANITIZE_sclp.o := n + +# +# Use -march=z900 for sclp.c and als.c to be able to print an error +# message if the kernel is started on a machine which is too old +# +ifneq ($(CC_FLAGS_MARCH),-march=z900) +CFLAGS_REMOVE_als.o += $(CC_FLAGS_MARCH) +CFLAGS_als.o += -march=z900 +CFLAGS_REMOVE_sclp.o += $(CC_FLAGS_MARCH) +CFLAGS_sclp.o += -march=z900 +AFLAGS_REMOVE_head.o += $(CC_FLAGS_MARCH) +AFLAGS_head.o += -march=z900 endif # # Passing null pointers is ok for smp code, since we access the lowcore here. # -CFLAGS_smp.o := -Wno-nonnull +CFLAGS_smp.o := -Wno-nonnull # # Disable tailcall optimizations for stack / callchain walking functions @@ -30,27 +57,7 @@ CFLAGS_dumpstack.o += -fno-optimize-sibling-calls # CFLAGS_ptrace.o += -DUTS_MACHINE='"$(UTS_MACHINE)"' -CFLAGS_sysinfo.o += -w - -# -# Use -march=z900 for sclp.c and als.c to be able to print an error -# message if the kernel is started on a machine which is too old -# -CFLAGS_REMOVE_sclp.o = $(CC_FLAGS_FTRACE) -CFLAGS_REMOVE_als.o = $(CC_FLAGS_FTRACE) -ifneq ($(CC_FLAGS_MARCH),-march=z900) -CFLAGS_REMOVE_sclp.o += $(CC_FLAGS_MARCH) -CFLAGS_sclp.o += -march=z900 -CFLAGS_REMOVE_als.o += $(CC_FLAGS_MARCH) -CFLAGS_als.o += -march=z900 -AFLAGS_REMOVE_head.o += $(CC_FLAGS_MARCH) -AFLAGS_head.o += -march=z900 -endif -GCOV_PROFILE_sclp.o := n -GCOV_PROFILE_als.o := n -UBSAN_SANITIZE_als.o := n -UBSAN_SANITIZE_early.o := n -UBSAN_SANITIZE_sclp.o := n +CFLAGS_sysinfo.o += -w obj-y := traps.o time.o process.o base.o early.o setup.o idle.o vtime.o obj-y += processor.o sys_s390.o ptrace.o signal.o cpcmd.o ebcdic.o nmi.o diff --git a/arch/s390/kernel/asm-offsets.c b/arch/s390/kernel/asm-offsets.c index f3df9e0..c4b3570 100644 --- a/arch/s390/kernel/asm-offsets.c +++ b/arch/s390/kernel/asm-offsets.c @@ -25,12 +25,14 @@ int main(void) { /* task struct offsets */ - OFFSET(__TASK_thread_info, task_struct, stack); + OFFSET(__TASK_stack, task_struct, stack); OFFSET(__TASK_thread, task_struct, thread); OFFSET(__TASK_pid, task_struct, pid); BLANK(); /* thread struct offsets */ OFFSET(__THREAD_ksp, thread_struct, ksp); + OFFSET(__THREAD_sysc_table, thread_struct, sys_call_table); + OFFSET(__THREAD_last_break, thread_struct, last_break); OFFSET(__THREAD_FPU_fpc, thread_struct, fpu.fpc); OFFSET(__THREAD_FPU_regs, thread_struct, fpu.regs); OFFSET(__THREAD_per_cause, thread_struct, per_event.cause); @@ -39,14 +41,7 @@ int main(void) OFFSET(__THREAD_trap_tdb, thread_struct, trap_tdb); BLANK(); /* thread info offsets */ - OFFSET(__TI_task, thread_info, task); - OFFSET(__TI_flags, thread_info, flags); - OFFSET(__TI_sysc_table, thread_info, sys_call_table); - OFFSET(__TI_cpu, thread_info, cpu); - OFFSET(__TI_precount, thread_info, preempt_count); - OFFSET(__TI_user_timer, thread_info, user_timer); - OFFSET(__TI_system_timer, thread_info, system_timer); - OFFSET(__TI_last_break, thread_info, last_break); + OFFSET(__TI_flags, task_struct, thread_info.flags); BLANK(); /* pt_regs offsets */ OFFSET(__PT_ARGS, pt_regs, args); @@ -79,6 +74,8 @@ int main(void) OFFSET(__VDSO_ECTG_OK, vdso_data, ectg_available); OFFSET(__VDSO_TK_MULT, vdso_data, tk_mult); OFFSET(__VDSO_TK_SHIFT, vdso_data, tk_shift); + OFFSET(__VDSO_TS_DIR, vdso_data, ts_dir); + OFFSET(__VDSO_TS_END, vdso_data, ts_end); OFFSET(__VDSO_ECTG_BASE, vdso_per_cpu_data, ectg_timer_base); OFFSET(__VDSO_ECTG_USER, vdso_per_cpu_data, ectg_user_time); OFFSET(__VDSO_CPU_NR, vdso_per_cpu_data, cpu_nr); @@ -159,7 +156,6 @@ int main(void) OFFSET(__LC_INT_CLOCK, lowcore, int_clock); OFFSET(__LC_MCCK_CLOCK, lowcore, mcck_clock); OFFSET(__LC_CURRENT, lowcore, current_task); - OFFSET(__LC_THREAD_INFO, lowcore, thread_info); OFFSET(__LC_KERNEL_STACK, lowcore, kernel_stack); OFFSET(__LC_ASYNC_STACK, lowcore, async_stack); OFFSET(__LC_PANIC_STACK, lowcore, panic_stack); @@ -173,6 +169,7 @@ int main(void) OFFSET(__LC_PERCPU_OFFSET, lowcore, percpu_offset); OFFSET(__LC_VDSO_PER_CPU, lowcore, vdso_per_cpu_data); OFFSET(__LC_MACHINE_FLAGS, lowcore, machine_flags); + OFFSET(__LC_PREEMPT_COUNT, lowcore, preempt_count); OFFSET(__LC_GMAP, lowcore, gmap); OFFSET(__LC_PASTE, lowcore, paste); /* software defined ABI-relevant lowcore locations 0xe00 - 0xe20 */ diff --git a/arch/s390/kernel/compat_signal.c b/arch/s390/kernel/compat_signal.c index 4af6037..6f2a6ab 100644 --- a/arch/s390/kernel/compat_signal.c +++ b/arch/s390/kernel/compat_signal.c @@ -446,7 +446,7 @@ static int setup_frame32(struct ksignal *ksig, sigset_t *set, /* set extra registers only for synchronous signals */ regs->gprs[4] = regs->int_code & 127; regs->gprs[5] = regs->int_parm_long; - regs->gprs[6] = task_thread_info(current)->last_break; + regs->gprs[6] = current->thread.last_break; } return 0; @@ -523,7 +523,7 @@ static int setup_rt_frame32(struct ksignal *ksig, sigset_t *set, regs->gprs[2] = ksig->sig; regs->gprs[3] = (__force __u64) &frame->info; regs->gprs[4] = (__force __u64) &frame->uc; - regs->gprs[5] = task_thread_info(current)->last_break; + regs->gprs[5] = current->thread.last_break; return 0; } diff --git a/arch/s390/kernel/early.c b/arch/s390/kernel/early.c index 2374c5b..d038c8c 100644 --- a/arch/s390/kernel/early.c +++ b/arch/s390/kernel/early.c @@ -293,6 +293,7 @@ static noinline __init void setup_lowcore_early(void) psw.addr = (unsigned long) s390_base_pgm_handler; S390_lowcore.program_new_psw = psw; s390_base_pgm_handler_fn = early_pgm_check_handler; + S390_lowcore.preempt_count = INIT_PREEMPT_COUNT; } static noinline __init void setup_facility_list(void) @@ -391,7 +392,49 @@ static int __init cad_init(void) } early_initcall(cad_init); -static __init void rescue_initrd(void) +static __init void memmove_early(void *dst, const void *src, size_t n) +{ + unsigned long addr; + long incr; + psw_t old; + + if (!n) + return; + incr = 1; + if (dst > src) { + incr = -incr; + dst += n - 1; + src += n - 1; + } + old = S390_lowcore.program_new_psw; + S390_lowcore.program_new_psw.mask = __extract_psw(); + asm volatile( + " larl %[addr],1f\n" + " stg %[addr],%[psw_pgm_addr]\n" + "0: mvc 0(1,%[dst]),0(%[src])\n" + " agr %[dst],%[incr]\n" + " agr %[src],%[incr]\n" + " brctg %[n],0b\n" + "1:\n" + : [addr] "=&d" (addr), + [psw_pgm_addr] "=&Q" (S390_lowcore.program_new_psw.addr), + [dst] "+&a" (dst), [src] "+&a" (src), [n] "+d" (n) + : [incr] "d" (incr) + : "cc", "memory"); + S390_lowcore.program_new_psw = old; +} + +static __init noinline void ipl_save_parameters(void) +{ + void *src, *dst; + + src = (void *)(unsigned long) S390_lowcore.ipl_parmblock_ptr; + dst = (void *) IPL_PARMBLOCK_ORIGIN; + memmove_early(dst, src, PAGE_SIZE); + S390_lowcore.ipl_parmblock_ptr = IPL_PARMBLOCK_ORIGIN; +} + +static __init noinline void rescue_initrd(void) { #ifdef CONFIG_BLK_DEV_INITRD unsigned long min_initrd_addr = (unsigned long) _end + (4UL << 20); @@ -405,7 +448,7 @@ static __init void rescue_initrd(void) return; if (INITRD_START >= min_initrd_addr) return; - memmove((void *) min_initrd_addr, (void *) INITRD_START, INITRD_SIZE); + memmove_early((void *) min_initrd_addr, (void *) INITRD_START, INITRD_SIZE); INITRD_START = min_initrd_addr; #endif } @@ -467,7 +510,8 @@ void __init startup_init(void) ipl_save_parameters(); rescue_initrd(); clear_bss_section(); - ptff_init(); + ipl_verify_parameters(); + time_early_init(); init_kernel_storage_key(); lockdep_off(); setup_lowcore_early(); diff --git a/arch/s390/kernel/entry.S b/arch/s390/kernel/entry.S index 49a3073..97298c5 100644 --- a/arch/s390/kernel/entry.S +++ b/arch/s390/kernel/entry.S @@ -42,7 +42,7 @@ __PT_R13 = __PT_GPRS + 104 __PT_R14 = __PT_GPRS + 112 __PT_R15 = __PT_GPRS + 120 -STACK_SHIFT = PAGE_SHIFT + THREAD_ORDER +STACK_SHIFT = PAGE_SHIFT + THREAD_SIZE_ORDER STACK_SIZE = 1 << STACK_SHIFT STACK_INIT = STACK_SIZE - STACK_FRAME_OVERHEAD - __PT_SIZE @@ -123,8 +123,14 @@ _PIF_WORK = (_PIF_PER_TRAP) .macro LAST_BREAK scratch srag \scratch,%r10,23 +#ifdef CONFIG_HAVE_MARCH_Z990_FEATURES jz .+10 - stg %r10,__TI_last_break(%r12) + stg %r10,__TASK_thread+__THREAD_last_break(%r12) +#else + jz .+14 + lghi \scratch,__TASK_thread + stg %r10,__THREAD_last_break(\scratch,%r12) +#endif .endm .macro REENABLE_IRQS @@ -186,14 +192,13 @@ ENTRY(__switch_to) stmg %r6,%r15,__SF_GPRS(%r15) # store gprs of prev task lgr %r1,%r2 aghi %r1,__TASK_thread # thread_struct of prev task - lg %r5,__TASK_thread_info(%r3) # get thread_info of next + lg %r5,__TASK_stack(%r3) # start of kernel stack of next stg %r15,__THREAD_ksp(%r1) # store kernel stack of prev lgr %r1,%r3 aghi %r1,__TASK_thread # thread_struct of next task lgr %r15,%r5 aghi %r15,STACK_INIT # end of kernel stack of next stg %r3,__LC_CURRENT # store task struct of next - stg %r5,__LC_THREAD_INFO # store thread info of next stg %r15,__LC_KERNEL_STACK # store end of kernel stack lg %r15,__THREAD_ksp(%r1) # load kernel stack of next /* c4 is used in guest detection: arch/s390/kernel/perf_cpum_sf.c */ @@ -274,7 +279,7 @@ ENTRY(system_call) .Lsysc_stmg: stmg %r8,%r15,__LC_SAVE_AREA_SYNC lg %r10,__LC_LAST_BREAK - lg %r12,__LC_THREAD_INFO + lg %r12,__LC_CURRENT lghi %r14,_PIF_SYSCALL .Lsysc_per: lg %r15,__LC_KERNEL_STACK @@ -288,7 +293,13 @@ ENTRY(system_call) mvc __PT_INT_CODE(4,%r11),__LC_SVC_ILC stg %r14,__PT_FLAGS(%r11) .Lsysc_do_svc: - lg %r10,__TI_sysc_table(%r12) # address of system call table + # load address of system call table +#ifdef CONFIG_HAVE_MARCH_Z990_FEATURES + lg %r10,__TASK_thread+__THREAD_sysc_table(%r12) +#else + lghi %r13,__TASK_thread + lg %r10,__THREAD_sysc_table(%r13,%r12) +#endif llgh %r8,__PT_INT_CODE+2(%r11) slag %r8,%r8,2 # shift and test for svc 0 jnz .Lsysc_nr_ok @@ -389,7 +400,6 @@ ENTRY(system_call) TSTMSK __PT_FLAGS(%r11),_PIF_SYSCALL jno .Lsysc_return lmg %r2,%r7,__PT_R2(%r11) # load svc arguments - lg %r10,__TI_sysc_table(%r12) # address of system call table lghi %r8,0 # svc 0 returns -ENOSYS llgh %r1,__PT_INT_CODE+2(%r11) # load new svc number cghi %r1,NR_syscalls @@ -457,7 +467,7 @@ ENTRY(system_call) # ENTRY(ret_from_fork) la %r11,STACK_FRAME_OVERHEAD(%r15) - lg %r12,__LC_THREAD_INFO + lg %r12,__LC_CURRENT brasl %r14,schedule_tail TRACE_IRQS_ON ssm __LC_SVC_NEW_PSW # reenable interrupts @@ -478,7 +488,7 @@ ENTRY(pgm_check_handler) stpt __LC_SYNC_ENTER_TIMER stmg %r8,%r15,__LC_SAVE_AREA_SYNC lg %r10,__LC_LAST_BREAK - lg %r12,__LC_THREAD_INFO + lg %r12,__LC_CURRENT larl %r13,cleanup_critical lmg %r8,%r9,__LC_PGM_OLD_PSW tmhh %r8,0x0001 # test problem state bit @@ -501,7 +511,7 @@ ENTRY(pgm_check_handler) 2: LAST_BREAK %r14 UPDATE_VTIME %r14,%r15,__LC_SYNC_ENTER_TIMER lg %r15,__LC_KERNEL_STACK - lg %r14,__TI_task(%r12) + lgr %r14,%r12 aghi %r14,__TASK_thread # pointer to thread_struct lghi %r13,__LC_PGM_TDB tm __LC_PGM_ILC+2,0x02 # check for transaction abort @@ -567,7 +577,7 @@ ENTRY(io_int_handler) stpt __LC_ASYNC_ENTER_TIMER stmg %r8,%r15,__LC_SAVE_AREA_ASYNC lg %r10,__LC_LAST_BREAK - lg %r12,__LC_THREAD_INFO + lg %r12,__LC_CURRENT larl %r13,cleanup_critical lmg %r8,%r9,__LC_IO_OLD_PSW SWITCH_ASYNC __LC_SAVE_AREA_ASYNC,__LC_ASYNC_ENTER_TIMER @@ -626,7 +636,7 @@ ENTRY(io_int_handler) jo .Lio_work_user # yes -> do resched & signal #ifdef CONFIG_PREEMPT # check for preemptive scheduling - icm %r0,15,__TI_precount(%r12) + icm %r0,15,__LC_PREEMPT_COUNT jnz .Lio_restore # preemption is disabled TSTMSK __TI_flags(%r12),_TIF_NEED_RESCHED jno .Lio_restore @@ -741,7 +751,7 @@ ENTRY(ext_int_handler) stpt __LC_ASYNC_ENTER_TIMER stmg %r8,%r15,__LC_SAVE_AREA_ASYNC lg %r10,__LC_LAST_BREAK - lg %r12,__LC_THREAD_INFO + lg %r12,__LC_CURRENT larl %r13,cleanup_critical lmg %r8,%r9,__LC_EXT_OLD_PSW SWITCH_ASYNC __LC_SAVE_AREA_ASYNC,__LC_ASYNC_ENTER_TIMER @@ -798,13 +808,10 @@ ENTRY(save_fpu_regs) TSTMSK __LC_CPU_FLAGS,_CIF_FPU bor %r14 stfpc __THREAD_FPU_fpc(%r2) -.Lsave_fpu_regs_fpc_end: lg %r3,__THREAD_FPU_regs(%r2) TSTMSK __LC_MACHINE_FLAGS,MACHINE_FLAG_VX jz .Lsave_fpu_regs_fp # no -> store FP regs -.Lsave_fpu_regs_vx_low: VSTM %v0,%v15,0,%r3 # vstm 0,15,0(3) -.Lsave_fpu_regs_vx_high: VSTM %v16,%v31,256,%r3 # vstm 16,31,256(3) j .Lsave_fpu_regs_done # -> set CIF_FPU flag .Lsave_fpu_regs_fp: @@ -851,9 +858,7 @@ load_fpu_regs: TSTMSK __LC_MACHINE_FLAGS,MACHINE_FLAG_VX lg %r4,__THREAD_FPU_regs(%r4) # %r4 <- reg save area jz .Lload_fpu_regs_fp # -> no VX, load FP regs -.Lload_fpu_regs_vx: VLM %v0,%v15,0,%r4 -.Lload_fpu_regs_vx_high: VLM %v16,%v31,256,%r4 j .Lload_fpu_regs_done .Lload_fpu_regs_fp: @@ -889,7 +894,7 @@ ENTRY(mcck_int_handler) spt __LC_CPU_TIMER_SAVE_AREA-4095(%r1) # revalidate cpu timer lmg %r0,%r15,__LC_GPREGS_SAVE_AREA-4095(%r1)# revalidate gprs lg %r10,__LC_LAST_BREAK - lg %r12,__LC_THREAD_INFO + lg %r12,__LC_CURRENT larl %r13,cleanup_critical lmg %r8,%r9,__LC_MCK_OLD_PSW TSTMSK __LC_MCCK_CODE,MCCK_CODE_SYSTEM_DAMAGE @@ -948,7 +953,7 @@ ENTRY(mcck_int_handler) .Lmcck_panic: lg %r15,__LC_PANIC_STACK - aghi %r15,-(STACK_FRAME_OVERHEAD + __PT_SIZE) + la %r11,STACK_FRAME_OVERHEAD(%r15) j .Lmcck_skip # @@ -1085,7 +1090,7 @@ cleanup_critical: jhe 0f # set up saved registers r10 and r12 stg %r10,16(%r11) # r10 last break - stg %r12,32(%r11) # r12 thread-info pointer + stg %r12,32(%r11) # r12 task struct pointer 0: # check if the user time update has been done clg %r9,BASED(.Lcleanup_system_call_insn+24) jh 0f @@ -1106,7 +1111,9 @@ cleanup_critical: lg %r9,16(%r11) srag %r9,%r9,23 jz 0f - mvc __TI_last_break(8,%r12),16(%r11) + lgr %r9,%r12 + aghi %r9,__TASK_thread + mvc __THREAD_last_break(8,%r9),16(%r11) 0: # set up saved register r11 lg %r15,__LC_KERNEL_STACK la %r9,STACK_FRAME_OVERHEAD(%r15) diff --git a/arch/s390/kernel/head.S b/arch/s390/kernel/head.S index 4431905..0b5ebf8 100644 --- a/arch/s390/kernel/head.S +++ b/arch/s390/kernel/head.S @@ -315,7 +315,7 @@ ENTRY(startup_kdump) jg startup_continue .Lstack: - .long 0x8000 + (1<<(PAGE_SHIFT+THREAD_ORDER)) + .long 0x8000 + (1<<(PAGE_SHIFT+THREAD_SIZE_ORDER)) .align 8 6: .long 0x7fffffff,0xffffffff diff --git a/arch/s390/kernel/head64.S b/arch/s390/kernel/head64.S index 03c2b46..482d352 100644 --- a/arch/s390/kernel/head64.S +++ b/arch/s390/kernel/head64.S @@ -32,11 +32,10 @@ ENTRY(startup_continue) # # Setup stack # - larl %r15,init_thread_union - stg %r15,__LC_THREAD_INFO # cache thread info in lowcore - lg %r14,__TI_task(%r15) # cache current in lowcore + larl %r14,init_task stg %r14,__LC_CURRENT - aghi %r15,1<<(PAGE_SHIFT+THREAD_ORDER) # init_task_union + THREAD_SIZE + larl %r15,init_thread_union + aghi %r15,1<<(PAGE_SHIFT+THREAD_SIZE_ORDER) # init_task_union + THREAD_SIZE stg %r15,__LC_KERNEL_STACK # set end of kernel stack aghi %r15,-160 # diff --git a/arch/s390/kernel/ipl.c b/arch/s390/kernel/ipl.c index 295bfb7..ff3364a 100644 --- a/arch/s390/kernel/ipl.c +++ b/arch/s390/kernel/ipl.c @@ -1991,10 +1991,9 @@ void __init ipl_update_parameters(void) diag308_set_works = 1; } -void __init ipl_save_parameters(void) +void __init ipl_verify_parameters(void) { struct cio_iplinfo iplinfo; - void *src, *dst; if (cio_get_iplinfo(&iplinfo)) return; @@ -2005,10 +2004,6 @@ void __init ipl_save_parameters(void) if (!iplinfo.is_qdio) return; ipl_flags |= IPL_PARMBLOCK_VALID; - src = (void *)(unsigned long)S390_lowcore.ipl_parmblock_ptr; - dst = (void *)IPL_PARMBLOCK_ORIGIN; - memmove(dst, src, PAGE_SIZE); - S390_lowcore.ipl_parmblock_ptr = IPL_PARMBLOCK_ORIGIN; } static LIST_HEAD(rcall); diff --git a/arch/s390/kernel/irq.c b/arch/s390/kernel/irq.c index 285d656..ef60f41 100644 --- a/arch/s390/kernel/irq.c +++ b/arch/s390/kernel/irq.c @@ -168,7 +168,7 @@ void do_softirq_own_stack(void) old = current_stack_pointer(); /* Check against async. stack address range. */ new = S390_lowcore.async_stack; - if (((new - old) >> (PAGE_SHIFT + THREAD_ORDER)) != 0) { + if (((new - old) >> (PAGE_SHIFT + THREAD_SIZE_ORDER)) != 0) { /* Need to switch to the async. stack. */ new -= STACK_FRAME_OVERHEAD; ((struct stack_frame *) new)->back_chain = old; diff --git a/arch/s390/kernel/lgr.c b/arch/s390/kernel/lgr.c index 6ea6d69..ae7dff1 100644 --- a/arch/s390/kernel/lgr.c +++ b/arch/s390/kernel/lgr.c @@ -5,7 +5,8 @@ * Author(s): Michael Holzheu <holzheu@linux.vnet.ibm.com> */ -#include <linux/module.h> +#include <linux/init.h> +#include <linux/export.h> #include <linux/timer.h> #include <linux/slab.h> #include <asm/facility.h> @@ -183,4 +184,4 @@ static int __init lgr_init(void) lgr_timer_set(); return 0; } -module_init(lgr_init); +device_initcall(lgr_init); diff --git a/arch/s390/kernel/perf_cpum_sf.c b/arch/s390/kernel/perf_cpum_sf.c index fcc634c..763dec1 100644 --- a/arch/s390/kernel/perf_cpum_sf.c +++ b/arch/s390/kernel/perf_cpum_sf.c @@ -995,39 +995,36 @@ static int perf_push_sample(struct perf_event *event, struct sf_raw_sample *sfr) regs.int_parm = CPU_MF_INT_SF_PRA; sde_regs = (struct perf_sf_sde_regs *) ®s.int_parm_long; - regs.psw.addr = sfr->basic.ia; - if (sfr->basic.T) - regs.psw.mask |= PSW_MASK_DAT; - if (sfr->basic.W) - regs.psw.mask |= PSW_MASK_WAIT; - if (sfr->basic.P) - regs.psw.mask |= PSW_MASK_PSTATE; - switch (sfr->basic.AS) { - case 0x0: - regs.psw.mask |= PSW_ASC_PRIMARY; - break; - case 0x1: - regs.psw.mask |= PSW_ASC_ACCREG; - break; - case 0x2: - regs.psw.mask |= PSW_ASC_SECONDARY; - break; - case 0x3: - regs.psw.mask |= PSW_ASC_HOME; - break; - } + psw_bits(regs.psw).ia = sfr->basic.ia; + psw_bits(regs.psw).t = sfr->basic.T; + psw_bits(regs.psw).w = sfr->basic.W; + psw_bits(regs.psw).p = sfr->basic.P; + psw_bits(regs.psw).as = sfr->basic.AS; /* - * A non-zero guest program parameter indicates a guest - * sample. - * Note that some early samples or samples from guests without + * Use the hardware provided configuration level to decide if the + * sample belongs to a guest or host. If that is not available, + * fall back to the following heuristics: + * A non-zero guest program parameter always indicates a guest + * sample. Some early samples or samples from guests without * lpp usage would be misaccounted to the host. We use the asn - * value as a heuristic to detect most of these guest samples. - * If the value differs from the host hpp value, we assume - * it to be a KVM guest. + * value as an addon heuristic to detect most of these guest samples. + * If the value differs from the host hpp value, we assume to be a + * KVM guest. */ - if (sfr->basic.gpp || sfr->basic.prim_asn != (u16) sfr->basic.hpp) + switch (sfr->basic.CL) { + case 1: /* logical partition */ + sde_regs->in_guest = 0; + break; + case 2: /* virtual machine */ sde_regs->in_guest = 1; + break; + default: /* old machine, use heuristics */ + if (sfr->basic.gpp || + sfr->basic.prim_asn != (u16)sfr->basic.hpp) + sde_regs->in_guest = 1; + break; + } overflow = 0; if (perf_exclude_event(event, ®s, sde_regs)) diff --git a/arch/s390/kernel/process.c b/arch/s390/kernel/process.c index bba4fa7..400d14f 100644 --- a/arch/s390/kernel/process.c +++ b/arch/s390/kernel/process.c @@ -103,7 +103,6 @@ int arch_dup_task_struct(struct task_struct *dst, struct task_struct *src) int copy_thread(unsigned long clone_flags, unsigned long new_stackp, unsigned long arg, struct task_struct *p) { - struct thread_info *ti; struct fake_frame { struct stack_frame sf; @@ -121,9 +120,8 @@ int copy_thread(unsigned long clone_flags, unsigned long new_stackp, memset(&p->thread.per_event, 0, sizeof(p->thread.per_event)); clear_tsk_thread_flag(p, TIF_SINGLE_STEP); /* Initialize per thread user and system timer values */ - ti = task_thread_info(p); - ti->user_timer = 0; - ti->system_timer = 0; + p->thread.user_timer = 0; + p->thread.system_timer = 0; frame->sf.back_chain = 0; /* new return point is ret_from_fork */ diff --git a/arch/s390/kernel/ptrace.c b/arch/s390/kernel/ptrace.c index 9336e824..b81ab88 100644 --- a/arch/s390/kernel/ptrace.c +++ b/arch/s390/kernel/ptrace.c @@ -461,7 +461,7 @@ long arch_ptrace(struct task_struct *child, long request, } return 0; case PTRACE_GET_LAST_BREAK: - put_user(task_thread_info(child)->last_break, + put_user(child->thread.last_break, (unsigned long __user *) data); return 0; case PTRACE_ENABLE_TE: @@ -811,7 +811,7 @@ long compat_arch_ptrace(struct task_struct *child, compat_long_t request, } return 0; case PTRACE_GET_LAST_BREAK: - put_user(task_thread_info(child)->last_break, + put_user(child->thread.last_break, (unsigned int __user *) data); return 0; } @@ -997,10 +997,10 @@ static int s390_last_break_get(struct task_struct *target, if (count > 0) { if (kbuf) { unsigned long *k = kbuf; - *k = task_thread_info(target)->last_break; + *k = target->thread.last_break; } else { unsigned long __user *u = ubuf; - if (__put_user(task_thread_info(target)->last_break, u)) + if (__put_user(target->thread.last_break, u)) return -EFAULT; } } @@ -1113,7 +1113,7 @@ static int s390_system_call_get(struct task_struct *target, unsigned int pos, unsigned int count, void *kbuf, void __user *ubuf) { - unsigned int *data = &task_thread_info(target)->system_call; + unsigned int *data = &target->thread.system_call; return user_regset_copyout(&pos, &count, &kbuf, &ubuf, data, 0, sizeof(unsigned int)); } @@ -1123,7 +1123,7 @@ static int s390_system_call_set(struct task_struct *target, unsigned int pos, unsigned int count, const void *kbuf, const void __user *ubuf) { - unsigned int *data = &task_thread_info(target)->system_call; + unsigned int *data = &target->thread.system_call; return user_regset_copyin(&pos, &count, &kbuf, &ubuf, data, 0, sizeof(unsigned int)); } @@ -1327,7 +1327,7 @@ static int s390_compat_last_break_get(struct task_struct *target, compat_ulong_t last_break; if (count > 0) { - last_break = task_thread_info(target)->last_break; + last_break = target->thread.last_break; if (kbuf) { unsigned long *k = kbuf; *k = last_break; diff --git a/arch/s390/kernel/setup.c b/arch/s390/kernel/setup.c index 7f7ba5f2..adfac9f 100644 --- a/arch/s390/kernel/setup.c +++ b/arch/s390/kernel/setup.c @@ -35,6 +35,7 @@ #include <linux/root_dev.h> #include <linux/console.h> #include <linux/kernel_stat.h> +#include <linux/dma-contiguous.h> #include <linux/device.h> #include <linux/notifier.h> #include <linux/pfn.h> @@ -303,7 +304,7 @@ static void __init setup_lowcore(void) * Setup lowcore for boot cpu */ BUILD_BUG_ON(sizeof(struct lowcore) != LC_PAGES * 4096); - lc = __alloc_bootmem_low(LC_PAGES * PAGE_SIZE, LC_PAGES * PAGE_SIZE, 0); + lc = memblock_virt_alloc_low(sizeof(*lc), sizeof(*lc)); lc->restart_psw.mask = PSW_KERNEL_BITS; lc->restart_psw.addr = (unsigned long) restart_int_handler; lc->external_new_psw.mask = PSW_KERNEL_BITS | @@ -324,15 +325,15 @@ static void __init setup_lowcore(void) lc->kernel_stack = ((unsigned long) &init_thread_union) + THREAD_SIZE - STACK_FRAME_OVERHEAD - sizeof(struct pt_regs); lc->async_stack = (unsigned long) - __alloc_bootmem(ASYNC_SIZE, ASYNC_SIZE, 0) + memblock_virt_alloc(ASYNC_SIZE, ASYNC_SIZE) + ASYNC_SIZE - STACK_FRAME_OVERHEAD - sizeof(struct pt_regs); lc->panic_stack = (unsigned long) - __alloc_bootmem(PAGE_SIZE, PAGE_SIZE, 0) + memblock_virt_alloc(PAGE_SIZE, PAGE_SIZE) + PAGE_SIZE - STACK_FRAME_OVERHEAD - sizeof(struct pt_regs); - lc->current_task = (unsigned long) init_thread_union.thread_info.task; - lc->thread_info = (unsigned long) &init_thread_union; + lc->current_task = (unsigned long)&init_task; lc->lpp = LPP_MAGIC; lc->machine_flags = S390_lowcore.machine_flags; + lc->preempt_count = S390_lowcore.preempt_count; lc->stfl_fac_list = S390_lowcore.stfl_fac_list; memcpy(lc->stfle_fac_list, S390_lowcore.stfle_fac_list, MAX_FACILITY_BIT/8); @@ -349,7 +350,7 @@ static void __init setup_lowcore(void) lc->last_update_timer = S390_lowcore.last_update_timer; lc->last_update_clock = S390_lowcore.last_update_clock; - restart_stack = __alloc_bootmem(ASYNC_SIZE, ASYNC_SIZE, 0); + restart_stack = memblock_virt_alloc(ASYNC_SIZE, ASYNC_SIZE); restart_stack += ASYNC_SIZE; /* @@ -412,7 +413,7 @@ static void __init setup_resources(void) bss_resource.end = (unsigned long) &__bss_stop - 1; for_each_memblock(memory, reg) { - res = alloc_bootmem_low(sizeof(*res)); + res = memblock_virt_alloc(sizeof(*res), 8); res->flags = IORESOURCE_BUSY | IORESOURCE_SYSTEM_RAM; res->name = "System RAM"; @@ -426,7 +427,7 @@ static void __init setup_resources(void) std_res->start > res->end) continue; if (std_res->end > res->end) { - sub_res = alloc_bootmem_low(sizeof(*sub_res)); + sub_res = memblock_virt_alloc(sizeof(*sub_res), 8); *sub_res = *std_res; sub_res->end = res->end; std_res->start = res->end + 1; @@ -445,7 +446,7 @@ static void __init setup_resources(void) * part of the System RAM resource. */ if (crashk_res.end) { - memblock_add(crashk_res.start, resource_size(&crashk_res)); + memblock_add_node(crashk_res.start, resource_size(&crashk_res), 0); memblock_reserve(crashk_res.start, resource_size(&crashk_res)); insert_resource(&iomem_resource, &crashk_res); } @@ -903,6 +904,7 @@ void __init setup_arch(char **cmdline_p) setup_memory_end(); setup_memory(); + dma_contiguous_reserve(memory_end); check_initrd(); reserve_crashkernel(); @@ -921,6 +923,8 @@ void __init setup_arch(char **cmdline_p) cpu_detect_mhz_feature(); cpu_init(); numa_setup(); + smp_detect_cpus(); + topology_init_early(); /* * Create kernel page tables and switch to virtual addressing. diff --git a/arch/s390/kernel/signal.c b/arch/s390/kernel/signal.c index d82562c..9f241d1e 100644 --- a/arch/s390/kernel/signal.c +++ b/arch/s390/kernel/signal.c @@ -359,7 +359,7 @@ static int setup_frame(int sig, struct k_sigaction *ka, /* set extra registers only for synchronous signals */ regs->gprs[4] = regs->int_code & 127; regs->gprs[5] = regs->int_parm_long; - regs->gprs[6] = task_thread_info(current)->last_break; + regs->gprs[6] = current->thread.last_break; } return 0; } @@ -430,7 +430,7 @@ static int setup_rt_frame(struct ksignal *ksig, sigset_t *set, regs->gprs[2] = ksig->sig; regs->gprs[3] = (unsigned long) &frame->info; regs->gprs[4] = (unsigned long) &frame->uc; - regs->gprs[5] = task_thread_info(current)->last_break; + regs->gprs[5] = current->thread.last_break; return 0; } @@ -467,13 +467,13 @@ void do_signal(struct pt_regs *regs) * the debugger may change all our registers, including the system * call information. */ - current_thread_info()->system_call = + current->thread.system_call = test_pt_regs_flag(regs, PIF_SYSCALL) ? regs->int_code : 0; if (get_signal(&ksig)) { /* Whee! Actually deliver the signal. */ - if (current_thread_info()->system_call) { - regs->int_code = current_thread_info()->system_call; + if (current->thread.system_call) { + regs->int_code = current->thread.system_call; /* Check for system call restarting. */ switch (regs->gprs[2]) { case -ERESTART_RESTARTBLOCK: @@ -506,8 +506,8 @@ void do_signal(struct pt_regs *regs) /* No handlers present - check for system call restart */ clear_pt_regs_flag(regs, PIF_SYSCALL); - if (current_thread_info()->system_call) { - regs->int_code = current_thread_info()->system_call; + if (current->thread.system_call) { + regs->int_code = current->thread.system_call; switch (regs->gprs[2]) { case -ERESTART_RESTARTBLOCK: /* Restart with sys_restart_syscall */ diff --git a/arch/s390/kernel/smp.c b/arch/s390/kernel/smp.c index df4a508ff..e49f61a 100644 --- a/arch/s390/kernel/smp.c +++ b/arch/s390/kernel/smp.c @@ -19,6 +19,7 @@ #define pr_fmt(fmt) KMSG_COMPONENT ": " fmt #include <linux/workqueue.h> +#include <linux/bootmem.h> #include <linux/module.h> #include <linux/init.h> #include <linux/mm.h> @@ -259,16 +260,14 @@ static void pcpu_prepare_secondary(struct pcpu *pcpu, int cpu) static void pcpu_attach_task(struct pcpu *pcpu, struct task_struct *tsk) { struct lowcore *lc = pcpu->lowcore; - struct thread_info *ti = task_thread_info(tsk); lc->kernel_stack = (unsigned long) task_stack_page(tsk) + THREAD_SIZE - STACK_FRAME_OVERHEAD - sizeof(struct pt_regs); - lc->thread_info = (unsigned long) task_thread_info(tsk); lc->current_task = (unsigned long) tsk; lc->lpp = LPP_MAGIC; lc->current_pid = tsk->pid; - lc->user_timer = ti->user_timer; - lc->system_timer = ti->system_timer; + lc->user_timer = tsk->thread.user_timer; + lc->system_timer = tsk->thread.system_timer; lc->steal_timer = 0; } @@ -662,14 +661,12 @@ int smp_cpu_get_polarization(int cpu) return pcpu_devices[cpu].polarization; } -static struct sclp_core_info *smp_get_core_info(void) +static void __ref smp_get_core_info(struct sclp_core_info *info, int early) { static int use_sigp_detection; - struct sclp_core_info *info; int address; - info = kzalloc(sizeof(*info), GFP_KERNEL); - if (info && (use_sigp_detection || sclp_get_core_info(info))) { + if (use_sigp_detection || sclp_get_core_info(info, early)) { use_sigp_detection = 1; for (address = 0; address < (SCLP_MAX_CORES << smp_cpu_mt_shift); @@ -683,7 +680,6 @@ static struct sclp_core_info *smp_get_core_info(void) } info->combined = info->configured; } - return info; } static int smp_add_present_cpu(int cpu); @@ -724,17 +720,15 @@ static int __smp_rescan_cpus(struct sclp_core_info *info, int sysfs_add) return nr; } -static void __init smp_detect_cpus(void) +void __init smp_detect_cpus(void) { unsigned int cpu, mtid, c_cpus, s_cpus; struct sclp_core_info *info; u16 address; /* Get CPU information */ - info = smp_get_core_info(); - if (!info) - panic("smp_detect_cpus failed to allocate memory\n"); - + info = memblock_virt_alloc(sizeof(*info), 8); + smp_get_core_info(info, 1); /* Find boot CPU type */ if (sclp.has_core_type) { address = stap(); @@ -770,7 +764,7 @@ static void __init smp_detect_cpus(void) get_online_cpus(); __smp_rescan_cpus(info, 0); put_online_cpus(); - kfree(info); + memblock_free_early((unsigned long)info, sizeof(*info)); } /* @@ -807,7 +801,7 @@ int __cpu_up(unsigned int cpu, struct task_struct *tidle) pcpu = pcpu_devices + cpu; if (pcpu->state != CPU_STATE_CONFIGURED) return -EIO; - base = cpu - (cpu % (smp_cpu_mtid + 1)); + base = smp_get_base_cpu(cpu); for (i = 0; i <= smp_cpu_mtid; i++) { if (base + i < nr_cpu_ids) if (cpu_online(base + i)) @@ -907,7 +901,6 @@ void __init smp_prepare_cpus(unsigned int max_cpus) /* request the 0x1202 external call external interrupt */ if (register_external_irq(EXT_IRQ_EXTERNAL_CALL, do_ext_call_interrupt)) panic("Couldn't request external interrupt 0x1202"); - smp_detect_cpus(); } void __init smp_prepare_boot_cpu(void) @@ -973,7 +966,7 @@ static ssize_t cpu_configure_store(struct device *dev, rc = -EBUSY; /* disallow configuration changes of online cpus and cpu 0 */ cpu = dev->id; - cpu -= cpu % (smp_cpu_mtid + 1); + cpu = smp_get_base_cpu(cpu); if (cpu == 0) goto out; for (i = 0; i <= smp_cpu_mtid; i++) @@ -1106,9 +1099,10 @@ int __ref smp_rescan_cpus(void) struct sclp_core_info *info; int nr; - info = smp_get_core_info(); + info = kzalloc(sizeof(*info), GFP_KERNEL); if (!info) return -ENOMEM; + smp_get_core_info(info, 0); get_online_cpus(); mutex_lock(&smp_cpu_state_mutex); nr = __smp_rescan_cpus(info, 1); diff --git a/arch/s390/kernel/swsusp.S b/arch/s390/kernel/swsusp.S index 2d6b6e8..1ff21f0 100644 --- a/arch/s390/kernel/swsusp.S +++ b/arch/s390/kernel/swsusp.S @@ -194,7 +194,7 @@ pgm_check_entry: /* Suspend CPU not available -> panic */ larl %r15,init_thread_union - ahi %r15,1<<(PAGE_SHIFT+THREAD_ORDER) + ahi %r15,1<<(PAGE_SHIFT+THREAD_SIZE_ORDER) larl %r2,.Lpanic_string larl %r3,_sclp_print_early lghi %r1,0 diff --git a/arch/s390/kernel/sysinfo.c b/arch/s390/kernel/sysinfo.c index bfda6aa..24021c1 100644 --- a/arch/s390/kernel/sysinfo.c +++ b/arch/s390/kernel/sysinfo.c @@ -56,6 +56,20 @@ int stsi(void *sysinfo, int fc, int sel1, int sel2) } EXPORT_SYMBOL(stsi); +static bool convert_ext_name(unsigned char encoding, char *name, size_t len) +{ + switch (encoding) { + case 1: /* EBCDIC */ + EBCASC(name, len); + break; + case 2: /* UTF-8 */ + break; + default: + return false; + } + return true; +} + static void stsi_1_1_1(struct seq_file *m, struct sysinfo_1_1_1 *info) { int i; @@ -207,24 +221,19 @@ static void stsi_2_2_2(struct seq_file *m, struct sysinfo_2_2_2 *info) seq_printf(m, "LPAR CPUs S-MTID: %d\n", info->mt_stid); seq_printf(m, "LPAR CPUs PS-MTID: %d\n", info->mt_psmtid); } + if (convert_ext_name(info->vsne, info->ext_name, sizeof(info->ext_name))) { + seq_printf(m, "LPAR Extended Name: %-.256s\n", info->ext_name); + seq_printf(m, "LPAR UUID: %pUb\n", &info->uuid); + } } static void print_ext_name(struct seq_file *m, int lvl, struct sysinfo_3_2_2 *info) { - if (info->vm[lvl].ext_name_encoding == 0) - return; - if (info->ext_names[lvl][0] == 0) - return; - switch (info->vm[lvl].ext_name_encoding) { - case 1: /* EBCDIC */ - EBCASC(info->ext_names[lvl], sizeof(info->ext_names[lvl])); - break; - case 2: /* UTF-8 */ - break; - default: + size_t len = sizeof(info->ext_names[lvl]); + + if (!convert_ext_name(info->vm[lvl].evmne, info->ext_names[lvl], len)) return; - } seq_printf(m, "VM%02d Extended Name: %-.256s\n", lvl, info->ext_names[lvl]); } diff --git a/arch/s390/kernel/time.c b/arch/s390/kernel/time.c index 0bfcc49..867d0a0 100644 --- a/arch/s390/kernel/time.c +++ b/arch/s390/kernel/time.c @@ -59,19 +59,27 @@ ATOMIC_NOTIFIER_HEAD(s390_epoch_delta_notifier); EXPORT_SYMBOL(s390_epoch_delta_notifier); unsigned char ptff_function_mask[16]; -unsigned long lpar_offset; -unsigned long initial_leap_seconds; + +static unsigned long long lpar_offset; +static unsigned long long initial_leap_seconds; +static unsigned long long tod_steering_end; +static long long tod_steering_delta; /* * Get time offsets with PTFF */ -void __init ptff_init(void) +void __init time_early_init(void) { struct ptff_qto qto; struct ptff_qui qui; + /* Initialize TOD steering parameters */ + tod_steering_end = sched_clock_base_cc; + vdso_data->ts_end = tod_steering_end; + if (!test_facility(28)) return; + ptff(&ptff_function_mask, sizeof(ptff_function_mask), PTFF_QAF); /* get LPAR offset */ @@ -80,7 +88,7 @@ void __init ptff_init(void) /* get initial leap seconds */ if (ptff_query(PTFF_QUI) && ptff(&qui, sizeof(qui), PTFF_QUI) == 0) - initial_leap_seconds = (unsigned long) + initial_leap_seconds = (unsigned long long) ((long) qui.old_leap * 4096000000L); } @@ -123,18 +131,6 @@ void clock_comparator_work(void) cd->event_handler(cd); } -/* - * Fixup the clock comparator. - */ -static void fixup_clock_comparator(unsigned long long delta) -{ - /* If nobody is waiting there's nothing to fix. */ - if (S390_lowcore.clock_comparator == -1ULL) - return; - S390_lowcore.clock_comparator += delta; - set_clock_comparator(S390_lowcore.clock_comparator); -} - static int s390_next_event(unsigned long delta, struct clock_event_device *evt) { @@ -215,7 +211,21 @@ void read_boot_clock64(struct timespec64 *ts) static cycle_t read_tod_clock(struct clocksource *cs) { - return get_tod_clock(); + unsigned long long now, adj; + + preempt_disable(); /* protect from changes to steering parameters */ + now = get_tod_clock(); + adj = tod_steering_end - now; + if (unlikely((s64) adj >= 0)) + /* + * manually steer by 1 cycle every 2^16 cycles. This + * corresponds to shifting the tod delta by 15. 1s is + * therefore steered in ~9h. The adjust will decrease + * over time, until it finally reaches 0. + */ + now += (tod_steering_delta < 0) ? (adj >> 15) : -(adj >> 15); + preempt_enable(); + return now; } static struct clocksource clocksource_tod = { @@ -384,6 +394,55 @@ static inline int check_sync_clock(void) return rc; } +/* + * Apply clock delta to the global data structures. + * This is called once on the CPU that performed the clock sync. + */ +static void clock_sync_global(unsigned long long delta) +{ + unsigned long now, adj; + struct ptff_qto qto; + + /* Fixup the monotonic sched clock. */ + sched_clock_base_cc += delta; + /* Adjust TOD steering parameters. */ + vdso_data->tb_update_count++; + now = get_tod_clock(); + adj = tod_steering_end - now; + if (unlikely((s64) adj >= 0)) + /* Calculate how much of the old adjustment is left. */ + tod_steering_delta = (tod_steering_delta < 0) ? + -(adj >> 15) : (adj >> 15); + tod_steering_delta += delta; + if ((abs(tod_steering_delta) >> 48) != 0) + panic("TOD clock sync offset %lli is too large to drift\n", + tod_steering_delta); + tod_steering_end = now + (abs(tod_steering_delta) << 15); + vdso_data->ts_dir = (tod_steering_delta < 0) ? 0 : 1; + vdso_data->ts_end = tod_steering_end; + vdso_data->tb_update_count++; + /* Update LPAR offset. */ + if (ptff_query(PTFF_QTO) && ptff(&qto, sizeof(qto), PTFF_QTO) == 0) + lpar_offset = qto.tod_epoch_difference; + /* Call the TOD clock change notifier. */ + atomic_notifier_call_chain(&s390_epoch_delta_notifier, 0, &delta); +} + +/* + * Apply clock delta to the per-CPU data structures of this CPU. + * This is called for each online CPU after the call to clock_sync_global. + */ +static void clock_sync_local(unsigned long long delta) +{ + /* Add the delta to the clock comparator. */ + if (S390_lowcore.clock_comparator != -1ULL) { + S390_lowcore.clock_comparator += delta; + set_clock_comparator(S390_lowcore.clock_comparator); + } + /* Adjust the last_update_clock time-stamp. */ + S390_lowcore.last_update_clock += delta; +} + /* Single threaded workqueue used for stp sync events */ static struct workqueue_struct *time_sync_wq; @@ -397,31 +456,9 @@ static void __init time_init_wq(void) struct clock_sync_data { atomic_t cpus; int in_sync; - unsigned long long fixup_cc; + unsigned long long clock_delta; }; -static void clock_sync_cpu(struct clock_sync_data *sync) -{ - atomic_dec(&sync->cpus); - enable_sync_clock(); - while (sync->in_sync == 0) { - __udelay(1); - /* - * A different cpu changes *in_sync. Therefore use - * barrier() to force memory access. - */ - barrier(); - } - if (sync->in_sync != 1) - /* Didn't work. Clear per-cpu in sync bit again. */ - disable_sync_clock(NULL); - /* - * This round of TOD syncing is done. Set the clock comparator - * to the next tick and let the processor continue. - */ - fixup_clock_comparator(sync->fixup_cc); -} - /* * Server Time Protocol (STP) code. */ @@ -523,54 +560,46 @@ void stp_queue_work(void) static int stp_sync_clock(void *data) { - static int first; + struct clock_sync_data *sync = data; unsigned long long clock_delta; - struct clock_sync_data *stp_sync; - struct ptff_qto qto; + static int first; int rc; - stp_sync = data; - - if (xchg(&first, 1) == 1) { - /* Slave */ - clock_sync_cpu(stp_sync); - return 0; - } - - /* Wait until all other cpus entered the sync function. */ - while (atomic_read(&stp_sync->cpus) != 0) - cpu_relax(); - enable_sync_clock(); - - rc = 0; - if (stp_info.todoff[0] || stp_info.todoff[1] || - stp_info.todoff[2] || stp_info.todoff[3] || - stp_info.tmd != 2) { - rc = chsc_sstpc(stp_page, STP_OP_SYNC, 0, &clock_delta); - if (rc == 0) { - /* fixup the monotonic sched clock */ - sched_clock_base_cc += clock_delta; - if (ptff_query(PTFF_QTO) && - ptff(&qto, sizeof(qto), PTFF_QTO) == 0) - /* Update LPAR offset */ - lpar_offset = qto.tod_epoch_difference; - atomic_notifier_call_chain(&s390_epoch_delta_notifier, - 0, &clock_delta); - stp_sync->fixup_cc = clock_delta; - fixup_clock_comparator(clock_delta); - rc = chsc_sstpi(stp_page, &stp_info, - sizeof(struct stp_sstpi)); - if (rc == 0 && stp_info.tmd != 2) - rc = -EAGAIN; + if (xchg(&first, 1) == 0) { + /* Wait until all other cpus entered the sync function. */ + while (atomic_read(&sync->cpus) != 0) + cpu_relax(); + rc = 0; + if (stp_info.todoff[0] || stp_info.todoff[1] || + stp_info.todoff[2] || stp_info.todoff[3] || + stp_info.tmd != 2) { + rc = chsc_sstpc(stp_page, STP_OP_SYNC, 0, + &clock_delta); + if (rc == 0) { + sync->clock_delta = clock_delta; + clock_sync_global(clock_delta); + rc = chsc_sstpi(stp_page, &stp_info, + sizeof(struct stp_sstpi)); + if (rc == 0 && stp_info.tmd != 2) + rc = -EAGAIN; + } } + sync->in_sync = rc ? -EAGAIN : 1; + xchg(&first, 0); + } else { + /* Slave */ + atomic_dec(&sync->cpus); + /* Wait for in_sync to be set. */ + while (READ_ONCE(sync->in_sync) == 0) + __udelay(1); } - if (rc) { + if (sync->in_sync != 1) + /* Didn't work. Clear per-cpu in sync bit again. */ disable_sync_clock(NULL); - stp_sync->in_sync = -EAGAIN; - } else - stp_sync->in_sync = 1; - xchg(&first, 0); + /* Apply clock delta to per-CPU fields of this CPU. */ + clock_sync_local(sync->clock_delta); + return 0; } diff --git a/arch/s390/kernel/topology.c b/arch/s390/kernel/topology.c index e959c02..93dcbae 100644 --- a/arch/s390/kernel/topology.c +++ b/arch/s390/kernel/topology.c @@ -7,6 +7,7 @@ #define pr_fmt(fmt) KMSG_COMPONENT ": " fmt #include <linux/workqueue.h> +#include <linux/bootmem.h> #include <linux/cpuset.h> #include <linux/device.h> #include <linux/export.h> @@ -41,15 +42,17 @@ static bool topology_enabled = true; static DECLARE_WORK(topology_work, topology_work_fn); /* - * Socket/Book linked lists and per_cpu(cpu_topology) updates are + * Socket/Book linked lists and cpu_topology updates are * protected by "sched_domains_mutex". */ static struct mask_info socket_info; static struct mask_info book_info; static struct mask_info drawer_info; -DEFINE_PER_CPU(struct cpu_topology_s390, cpu_topology); -EXPORT_PER_CPU_SYMBOL_GPL(cpu_topology); +struct cpu_topology_s390 cpu_topology[NR_CPUS]; +EXPORT_SYMBOL_GPL(cpu_topology); + +cpumask_t cpus_with_topology; static cpumask_t cpu_group_map(struct mask_info *info, unsigned int cpu) { @@ -97,7 +100,7 @@ static void add_cpus_to_mask(struct topology_core *tl_core, if (lcpu < 0) continue; for (i = 0; i <= smp_cpu_mtid; i++) { - topo = &per_cpu(cpu_topology, lcpu + i); + topo = &cpu_topology[lcpu + i]; topo->drawer_id = drawer->id; topo->book_id = book->id; topo->socket_id = socket->id; @@ -106,6 +109,7 @@ static void add_cpus_to_mask(struct topology_core *tl_core, cpumask_set_cpu(lcpu + i, &drawer->mask); cpumask_set_cpu(lcpu + i, &book->mask); cpumask_set_cpu(lcpu + i, &socket->mask); + cpumask_set_cpu(lcpu + i, &cpus_with_topology); smp_cpu_set_polarization(lcpu + i, tl_core->pp); } } @@ -220,7 +224,7 @@ static void update_cpu_masks(void) int cpu; for_each_possible_cpu(cpu) { - topo = &per_cpu(cpu_topology, cpu); + topo = &cpu_topology[cpu]; topo->thread_mask = cpu_thread_map(cpu); topo->core_mask = cpu_group_map(&socket_info, cpu); topo->book_mask = cpu_group_map(&book_info, cpu); @@ -231,6 +235,8 @@ static void update_cpu_masks(void) topo->socket_id = cpu; topo->book_id = cpu; topo->drawer_id = cpu; + if (cpu_present(cpu)) + cpumask_set_cpu(cpu, &cpus_with_topology); } } numa_update_cpu_topology(); @@ -241,12 +247,12 @@ void store_topology(struct sysinfo_15_1_x *info) stsi(info, 15, 1, min(topology_max_mnest, 4)); } -int arch_update_cpu_topology(void) +static int __arch_update_cpu_topology(void) { struct sysinfo_15_1_x *info = tl_info; - struct device *dev; - int cpu, rc = 0; + int rc = 0; + cpumask_clear(&cpus_with_topology); if (MACHINE_HAS_TOPOLOGY) { rc = 1; store_topology(info); @@ -255,6 +261,15 @@ int arch_update_cpu_topology(void) update_cpu_masks(); if (!MACHINE_HAS_TOPOLOGY) topology_update_polarization_simple(); + return rc; +} + +int arch_update_cpu_topology(void) +{ + struct device *dev; + int cpu, rc; + + rc = __arch_update_cpu_topology(); for_each_online_cpu(cpu) { dev = get_cpu_device(cpu); kobject_uevent(&dev->kobj, KOBJ_CHANGE); @@ -394,23 +409,23 @@ int topology_cpu_init(struct cpu *cpu) static const struct cpumask *cpu_thread_mask(int cpu) { - return &per_cpu(cpu_topology, cpu).thread_mask; + return &cpu_topology[cpu].thread_mask; } const struct cpumask *cpu_coregroup_mask(int cpu) { - return &per_cpu(cpu_topology, cpu).core_mask; + return &cpu_topology[cpu].core_mask; } static const struct cpumask *cpu_book_mask(int cpu) { - return &per_cpu(cpu_topology, cpu).book_mask; + return &cpu_topology[cpu].book_mask; } static const struct cpumask *cpu_drawer_mask(int cpu) { - return &per_cpu(cpu_topology, cpu).drawer_mask; + return &cpu_topology[cpu].drawer_mask; } static int __init early_parse_topology(char *p) @@ -438,19 +453,20 @@ static void __init alloc_masks(struct sysinfo_15_1_x *info, nr_masks *= info->mag[TOPOLOGY_NR_MAG - offset - 1 - i]; nr_masks = max(nr_masks, 1); for (i = 0; i < nr_masks; i++) { - mask->next = kzalloc(sizeof(*mask->next), GFP_KERNEL); + mask->next = memblock_virt_alloc(sizeof(*mask->next), 8); mask = mask->next; } } -static int __init s390_topology_init(void) +void __init topology_init_early(void) { struct sysinfo_15_1_x *info; int i; + set_sched_topology(s390_topology); if (!MACHINE_HAS_TOPOLOGY) - return 0; - tl_info = (struct sysinfo_15_1_x *)__get_free_page(GFP_KERNEL); + goto out; + tl_info = memblock_virt_alloc(sizeof(*tl_info), PAGE_SIZE); info = tl_info; store_topology(info); pr_info("The CPU configuration topology of the machine is:"); @@ -460,10 +476,9 @@ static int __init s390_topology_init(void) alloc_masks(info, &socket_info, 1); alloc_masks(info, &book_info, 2); alloc_masks(info, &drawer_info, 3); - set_sched_topology(s390_topology); - return 0; +out: + __arch_update_cpu_topology(); } -early_initcall(s390_topology_init); static int __init topology_init(void) { diff --git a/arch/s390/kernel/vdso32/clock_gettime.S b/arch/s390/kernel/vdso32/clock_gettime.S index 5eec9af..a5769b8 100644 --- a/arch/s390/kernel/vdso32/clock_gettime.S +++ b/arch/s390/kernel/vdso32/clock_gettime.S @@ -99,8 +99,27 @@ __kernel_clock_gettime: tml %r4,0x0001 /* pending update ? loop */ jnz 11b stcke 0(%r15) /* Store TOD clock */ - lm %r0,%r1,1(%r15) - s %r0,__VDSO_XTIME_STAMP(%r5) /* TOD - cycle_last */ + lm %r0,%r1,__VDSO_TS_END(%r5) /* TOD steering end time */ + s %r0,1(%r15) /* no - ts_steering_end */ + sl %r1,5(%r15) + brc 3,22f + ahi %r0,-1 +22: ltr %r0,%r0 /* past end of steering? */ + jm 24f + srdl %r0,15 /* 1 per 2^16 */ + tm __VDSO_TS_DIR+3(%r5),0x01 /* steering direction? */ + jz 23f + lcr %r0,%r0 /* negative TOD offset */ + lcr %r1,%r1 + je 23f + ahi %r0,-1 +23: a %r0,1(%r15) /* add TOD timestamp */ + al %r1,5(%r15) + brc 12,25f + ahi %r0,1 + j 25f +24: lm %r0,%r1,1(%r15) /* load TOD timestamp */ +25: s %r0,__VDSO_XTIME_STAMP(%r5) /* TOD - cycle_last */ sl %r1,__VDSO_XTIME_STAMP+4(%r5) brc 3,12f ahi %r0,-1 diff --git a/arch/s390/kernel/vdso32/gettimeofday.S b/arch/s390/kernel/vdso32/gettimeofday.S index 719de61..63b86dc 100644 --- a/arch/s390/kernel/vdso32/gettimeofday.S +++ b/arch/s390/kernel/vdso32/gettimeofday.S @@ -31,8 +31,27 @@ __kernel_gettimeofday: tml %r4,0x0001 /* pending update ? loop */ jnz 1b stcke 0(%r15) /* Store TOD clock */ - lm %r0,%r1,1(%r15) - s %r0,__VDSO_XTIME_STAMP(%r5) /* TOD - cycle_last */ + lm %r0,%r1,__VDSO_TS_END(%r5) /* TOD steering end time */ + s %r0,1(%r15) + sl %r1,5(%r15) + brc 3,14f + ahi %r0,-1 +14: ltr %r0,%r0 /* past end of steering? */ + jm 16f + srdl %r0,15 /* 1 per 2^16 */ + tm __VDSO_TS_DIR+3(%r5),0x01 /* steering direction? */ + jz 15f + lcr %r0,%r0 /* negative TOD offset */ + lcr %r1,%r1 + je 15f + ahi %r0,-1 +15: a %r0,1(%r15) /* add TOD timestamp */ + al %r1,5(%r15) + brc 12,17f + ahi %r0,1 + j 17f +16: lm %r0,%r1,1(%r15) /* load TOD timestamp */ +17: s %r0,__VDSO_XTIME_STAMP(%r5) /* TOD - cycle_last */ sl %r1,__VDSO_XTIME_STAMP+4(%r5) brc 3,3f ahi %r0,-1 diff --git a/arch/s390/kernel/vdso64/clock_gettime.S b/arch/s390/kernel/vdso64/clock_gettime.S index 61541fb..9c3b126 100644 --- a/arch/s390/kernel/vdso64/clock_gettime.S +++ b/arch/s390/kernel/vdso64/clock_gettime.S @@ -83,8 +83,17 @@ __kernel_clock_gettime: tmll %r4,0x0001 /* pending update ? loop */ jnz 5b stcke 0(%r15) /* Store TOD clock */ - lgf %r2,__VDSO_TK_SHIFT(%r5) /* Timekeeper shift */ lg %r1,1(%r15) + lg %r0,__VDSO_TS_END(%r5) /* TOD steering end time */ + slgr %r0,%r1 /* now - ts_steering_end */ + ltgr %r0,%r0 /* past end of steering ? */ + jm 17f + srlg %r0,%r0,15 /* 1 per 2^16 */ + tm __VDSO_TS_DIR+3(%r5),0x01 /* steering direction? */ + jz 18f + lcgr %r0,%r0 /* negative TOD offset */ +18: algr %r1,%r0 /* add steering offset */ +17: lgf %r2,__VDSO_TK_SHIFT(%r5) /* Timekeeper shift */ sg %r1,__VDSO_XTIME_STAMP(%r5) /* TOD - cycle_last */ msgf %r1,__VDSO_TK_MULT(%r5) /* * tk->mult */ alg %r1,__VDSO_XTIME_NSEC(%r5) /* + tk->xtime_nsec */ diff --git a/arch/s390/kernel/vdso64/gettimeofday.S b/arch/s390/kernel/vdso64/gettimeofday.S index 6ce4670..b02e62f 100644 --- a/arch/s390/kernel/vdso64/gettimeofday.S +++ b/arch/s390/kernel/vdso64/gettimeofday.S @@ -31,7 +31,16 @@ __kernel_gettimeofday: jnz 0b stcke 0(%r15) /* Store TOD clock */ lg %r1,1(%r15) - sg %r1,__VDSO_XTIME_STAMP(%r5) /* TOD - cycle_last */ + lg %r0,__VDSO_TS_END(%r5) /* TOD steering end time */ + slgr %r0,%r1 /* now - ts_steering_end */ + ltgr %r0,%r0 /* past end of steering ? */ + jm 6f + srlg %r0,%r0,15 /* 1 per 2^16 */ + tm __VDSO_TS_DIR+3(%r5),0x01 /* steering direction? */ + jz 7f + lcgr %r0,%r0 /* negative TOD offset */ +7: algr %r1,%r0 /* add steering offset */ +6: sg %r1,__VDSO_XTIME_STAMP(%r5) /* TOD - cycle_last */ msgf %r1,__VDSO_TK_MULT(%r5) /* * tk->mult */ alg %r1,__VDSO_XTIME_NSEC(%r5) /* + tk->xtime_nsec */ lg %r0,__VDSO_XTIME_SEC(%r5) /* tk->xtime_sec */ diff --git a/arch/s390/kernel/vtime.c b/arch/s390/kernel/vtime.c index 1bd5dde..6b246aa 100644 --- a/arch/s390/kernel/vtime.c +++ b/arch/s390/kernel/vtime.c @@ -96,7 +96,6 @@ static void update_mt_scaling(void) */ static int do_account_vtime(struct task_struct *tsk, int hardirq_offset) { - struct thread_info *ti = task_thread_info(tsk); u64 timer, clock, user, system, steal; u64 user_scaled, system_scaled; @@ -119,13 +118,13 @@ static int do_account_vtime(struct task_struct *tsk, int hardirq_offset) time_after64(jiffies_64, this_cpu_read(mt_scaling_jiffies))) update_mt_scaling(); - user = S390_lowcore.user_timer - ti->user_timer; + user = S390_lowcore.user_timer - tsk->thread.user_timer; S390_lowcore.steal_timer -= user; - ti->user_timer = S390_lowcore.user_timer; + tsk->thread.user_timer = S390_lowcore.user_timer; - system = S390_lowcore.system_timer - ti->system_timer; + system = S390_lowcore.system_timer - tsk->thread.system_timer; S390_lowcore.steal_timer -= system; - ti->system_timer = S390_lowcore.system_timer; + tsk->thread.system_timer = S390_lowcore.system_timer; user_scaled = user; system_scaled = system; @@ -153,15 +152,11 @@ static int do_account_vtime(struct task_struct *tsk, int hardirq_offset) void vtime_task_switch(struct task_struct *prev) { - struct thread_info *ti; - do_account_vtime(prev, 0); - ti = task_thread_info(prev); - ti->user_timer = S390_lowcore.user_timer; - ti->system_timer = S390_lowcore.system_timer; - ti = task_thread_info(current); - S390_lowcore.user_timer = ti->user_timer; - S390_lowcore.system_timer = ti->system_timer; + prev->thread.user_timer = S390_lowcore.user_timer; + prev->thread.system_timer = S390_lowcore.system_timer; + S390_lowcore.user_timer = current->thread.user_timer; + S390_lowcore.system_timer = current->thread.system_timer; } /* @@ -181,7 +176,6 @@ void vtime_account_user(struct task_struct *tsk) */ void vtime_account_irq_enter(struct task_struct *tsk) { - struct thread_info *ti = task_thread_info(tsk); u64 timer, system, system_scaled; timer = S390_lowcore.last_update_timer; @@ -193,9 +187,9 @@ void vtime_account_irq_enter(struct task_struct *tsk) time_after64(jiffies_64, this_cpu_read(mt_scaling_jiffies))) update_mt_scaling(); - system = S390_lowcore.system_timer - ti->system_timer; + system = S390_lowcore.system_timer - tsk->thread.system_timer; S390_lowcore.steal_timer -= system; - ti->system_timer = S390_lowcore.system_timer; + tsk->thread.system_timer = S390_lowcore.system_timer; system_scaled = system; /* Do MT utilization scaling */ if (smp_cpu_mtid) { diff --git a/arch/s390/kvm/interrupt.c b/arch/s390/kvm/interrupt.c index be4db07..af13f1a 100644 --- a/arch/s390/kvm/interrupt.c +++ b/arch/s390/kvm/interrupt.c @@ -415,7 +415,7 @@ static int __write_machine_check(struct kvm_vcpu *vcpu, int rc; mci.val = mchk->mcic; - /* take care of lazy register loading via vcpu load/put */ + /* take care of lazy register loading */ save_fpu_regs(); save_access_regs(vcpu->run->s.regs.acrs); diff --git a/arch/s390/kvm/kvm-s390.c b/arch/s390/kvm/kvm-s390.c index 9c7a1ec..bec71e9 100644 --- a/arch/s390/kvm/kvm-s390.c +++ b/arch/s390/kvm/kvm-s390.c @@ -1812,22 +1812,7 @@ __u64 kvm_s390_get_cpu_timer(struct kvm_vcpu *vcpu) void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu) { - /* Save host register state */ - save_fpu_regs(); - vcpu->arch.host_fpregs.fpc = current->thread.fpu.fpc; - vcpu->arch.host_fpregs.regs = current->thread.fpu.regs; - - if (MACHINE_HAS_VX) - current->thread.fpu.regs = vcpu->run->s.regs.vrs; - else - current->thread.fpu.regs = vcpu->run->s.regs.fprs; - current->thread.fpu.fpc = vcpu->run->s.regs.fpc; - if (test_fp_ctl(current->thread.fpu.fpc)) - /* User space provided an invalid FPC, let's clear it */ - current->thread.fpu.fpc = 0; - save_access_regs(vcpu->arch.host_acrs); - restore_access_regs(vcpu->run->s.regs.acrs); gmap_enable(vcpu->arch.enabled_gmap); atomic_or(CPUSTAT_RUNNING, &vcpu->arch.sie_block->cpuflags); if (vcpu->arch.cputm_enabled && !is_vcpu_idle(vcpu)) @@ -1844,16 +1829,6 @@ void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu) vcpu->arch.enabled_gmap = gmap_get_enabled(); gmap_disable(vcpu->arch.enabled_gmap); - /* Save guest register state */ - save_fpu_regs(); - vcpu->run->s.regs.fpc = current->thread.fpu.fpc; - - /* Restore host register state */ - current->thread.fpu.fpc = vcpu->arch.host_fpregs.fpc; - current->thread.fpu.regs = vcpu->arch.host_fpregs.regs; - - save_access_regs(vcpu->run->s.regs.acrs); - restore_access_regs(vcpu->arch.host_acrs); } static void kvm_s390_vcpu_initial_reset(struct kvm_vcpu *vcpu) @@ -2243,7 +2218,6 @@ int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu, { memcpy(&vcpu->run->s.regs.acrs, &sregs->acrs, sizeof(sregs->acrs)); memcpy(&vcpu->arch.sie_block->gcr, &sregs->crs, sizeof(sregs->crs)); - restore_access_regs(vcpu->run->s.regs.acrs); return 0; } @@ -2257,11 +2231,9 @@ int kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu, int kvm_arch_vcpu_ioctl_set_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu) { - /* make sure the new values will be lazily loaded */ - save_fpu_regs(); if (test_fp_ctl(fpu->fpc)) return -EINVAL; - current->thread.fpu.fpc = fpu->fpc; + vcpu->run->s.regs.fpc = fpu->fpc; if (MACHINE_HAS_VX) convert_fp_to_vx((__vector128 *) vcpu->run->s.regs.vrs, (freg_t *) fpu->fprs); @@ -2279,7 +2251,7 @@ int kvm_arch_vcpu_ioctl_get_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu) (__vector128 *) vcpu->run->s.regs.vrs); else memcpy(fpu->fprs, vcpu->run->s.regs.fprs, sizeof(fpu->fprs)); - fpu->fpc = current->thread.fpu.fpc; + fpu->fpc = vcpu->run->s.regs.fpc; return 0; } @@ -2740,6 +2712,20 @@ static void sync_regs(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) if (riccb->valid) vcpu->arch.sie_block->ecb3 |= 0x01; } + save_access_regs(vcpu->arch.host_acrs); + restore_access_regs(vcpu->run->s.regs.acrs); + /* save host (userspace) fprs/vrs */ + save_fpu_regs(); + vcpu->arch.host_fpregs.fpc = current->thread.fpu.fpc; + vcpu->arch.host_fpregs.regs = current->thread.fpu.regs; + if (MACHINE_HAS_VX) + current->thread.fpu.regs = vcpu->run->s.regs.vrs; + else + current->thread.fpu.regs = vcpu->run->s.regs.fprs; + current->thread.fpu.fpc = vcpu->run->s.regs.fpc; + if (test_fp_ctl(current->thread.fpu.fpc)) + /* User space provided an invalid FPC, let's clear it */ + current->thread.fpu.fpc = 0; kvm_run->kvm_dirty_regs = 0; } @@ -2758,6 +2744,15 @@ static void store_regs(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) kvm_run->s.regs.pft = vcpu->arch.pfault_token; kvm_run->s.regs.pfs = vcpu->arch.pfault_select; kvm_run->s.regs.pfc = vcpu->arch.pfault_compare; + save_access_regs(vcpu->run->s.regs.acrs); + restore_access_regs(vcpu->arch.host_acrs); + /* Save guest register state */ + save_fpu_regs(); + vcpu->run->s.regs.fpc = current->thread.fpu.fpc; + /* Restore will be done lazily at return */ + current->thread.fpu.fpc = vcpu->arch.host_fpregs.fpc; + current->thread.fpu.regs = vcpu->arch.host_fpregs.regs; + } int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) @@ -2874,7 +2869,7 @@ int kvm_s390_vcpu_store_status(struct kvm_vcpu *vcpu, unsigned long addr) { /* * The guest FPRS and ACRS are in the host FPRS/ACRS due to the lazy - * copying in vcpu load/put. Lets update our copies before we save + * switch in the run ioctl. Let's update our copies before we save * it into the save area */ save_fpu_regs(); diff --git a/arch/s390/lib/mem.S b/arch/s390/lib/mem.S index be9fa65..7422a70 100644 --- a/arch/s390/lib/mem.S +++ b/arch/s390/lib/mem.S @@ -8,6 +8,45 @@ #include <asm/export.h> /* + * void *memmove(void *dest, const void *src, size_t n) + */ +ENTRY(memmove) + ltgr %r4,%r4 + lgr %r1,%r2 + bzr %r14 + clgr %r2,%r3 + jnh .Lmemmove_forward + la %r5,0(%r4,%r3) + clgr %r2,%r5 + jl .Lmemmove_reverse +.Lmemmove_forward: + aghi %r4,-1 + srlg %r0,%r4,8 + ltgr %r0,%r0 + jz .Lmemmove_rest +.Lmemmove_loop: + mvc 0(256,%r1),0(%r3) + la %r1,256(%r1) + la %r3,256(%r3) + brctg %r0,.Lmemmove_loop +.Lmemmove_rest: + larl %r5,.Lmemmove_mvc + ex %r4,0(%r5) + br %r14 +.Lmemmove_reverse: + aghi %r4,-1 +.Lmemmove_reverse_loop: + ic %r0,0(%r4,%r3) + stc %r0,0(%r4,%r1) + brctg %r4,.Lmemmove_reverse_loop + ic %r0,0(%r4,%r3) + stc %r0,0(%r4,%r1) + br %r14 +.Lmemmove_mvc: + mvc 0(1,%r1),0(%r3) +EXPORT_SYMBOL(memmove) + +/* * memset implementation * * This code corresponds to the C construct below. We do distinguish diff --git a/arch/s390/mm/fault.c b/arch/s390/mm/fault.c index 661d9fe..d1faae5 100644 --- a/arch/s390/mm/fault.c +++ b/arch/s390/mm/fault.c @@ -733,6 +733,7 @@ block: * return to userspace schedule() to block. */ __set_current_state(TASK_UNINTERRUPTIBLE); set_tsk_need_resched(tsk); + set_preempt_need_resched(); } } out: diff --git a/arch/s390/mm/vmem.c b/arch/s390/mm/vmem.c index 1848292..45becc8 100644 --- a/arch/s390/mm/vmem.c +++ b/arch/s390/mm/vmem.c @@ -34,7 +34,7 @@ static void __ref *vmem_alloc_pages(unsigned int order) if (slab_is_available()) return (void *)__get_free_pages(GFP_KERNEL, order); - return alloc_bootmem_align(size, size); + return (void *) memblock_alloc(size, size); } static inline pud_t *vmem_pud_alloc(void) @@ -61,17 +61,16 @@ pmd_t *vmem_pmd_alloc(void) pte_t __ref *vmem_pte_alloc(void) { + unsigned long size = PTRS_PER_PTE * sizeof(pte_t); pte_t *pte; if (slab_is_available()) pte = (pte_t *) page_table_alloc(&init_mm); else - pte = alloc_bootmem_align(PTRS_PER_PTE * sizeof(pte_t), - PTRS_PER_PTE * sizeof(pte_t)); + pte = (pte_t *) memblock_alloc(size, size); if (!pte) return NULL; - clear_table((unsigned long *) pte, _PAGE_INVALID, - PTRS_PER_PTE * sizeof(pte_t)); + clear_table((unsigned long *) pte, _PAGE_INVALID, size); return pte; } diff --git a/arch/s390/numa/mode_emu.c b/arch/s390/numa/mode_emu.c index 37e0bb8..cfd0838 100644 --- a/arch/s390/numa/mode_emu.c +++ b/arch/s390/numa/mode_emu.c @@ -21,6 +21,7 @@ #include <linux/kernel.h> #include <linux/cpumask.h> #include <linux/memblock.h> +#include <linux/bootmem.h> #include <linux/node.h> #include <linux/memory.h> #include <linux/slab.h> @@ -307,13 +308,11 @@ fail: /* * Allocate and initialize core to node mapping */ -static void create_core_to_node_map(void) +static void __ref create_core_to_node_map(void) { int i; - emu_cores = kzalloc(sizeof(*emu_cores), GFP_KERNEL); - if (emu_cores == NULL) - panic("Could not allocate cores to node memory"); + emu_cores = memblock_virt_alloc(sizeof(*emu_cores), 8); for (i = 0; i < ARRAY_SIZE(emu_cores->to_node_id); i++) emu_cores->to_node_id[i] = NODE_ID_FREE; } @@ -354,13 +353,13 @@ static struct toptree *toptree_from_topology(void) phys = toptree_new(TOPTREE_ID_PHYS, 1); - for_each_online_cpu(cpu) { - top = &per_cpu(cpu_topology, cpu); + for_each_cpu(cpu, &cpus_with_topology) { + top = &cpu_topology[cpu]; node = toptree_get_child(phys, 0); drawer = toptree_get_child(node, top->drawer_id); book = toptree_get_child(drawer, top->book_id); mc = toptree_get_child(book, top->socket_id); - core = toptree_get_child(mc, top->core_id); + core = toptree_get_child(mc, smp_get_base_cpu(cpu)); if (!drawer || !book || !mc || !core) panic("NUMA emulation could not allocate memory"); cpumask_set_cpu(cpu, &core->mask); @@ -378,7 +377,7 @@ static void topology_add_core(struct toptree *core) int cpu; for_each_cpu(cpu, &core->mask) { - top = &per_cpu(cpu_topology, cpu); + top = &cpu_topology[cpu]; cpumask_copy(&top->thread_mask, &core->mask); cpumask_copy(&top->core_mask, &core_mc(core)->mask); cpumask_copy(&top->book_mask, &core_book(core)->mask); @@ -425,6 +424,27 @@ static void print_node_to_core_map(void) } } +static void pin_all_possible_cpus(void) +{ + int core_id, node_id, cpu; + static int initialized; + + if (initialized) + return; + print_node_to_core_map(); + node_id = 0; + for_each_possible_cpu(cpu) { + core_id = smp_get_base_cpu(cpu); + if (emu_cores->to_node_id[core_id] != NODE_ID_FREE) + continue; + pin_core_to_node(core_id, node_id); + cpu_topology[cpu].node_id = node_id; + node_id = (node_id + 1) % emu_nodes; + } + print_node_to_core_map(); + initialized = 1; +} + /* * Transfer physical topology into a NUMA topology and modify CPU masks * according to the NUMA topology. @@ -442,7 +462,7 @@ static void emu_update_cpu_topology(void) toptree_free(phys); toptree_to_topology(numa); toptree_free(numa); - print_node_to_core_map(); + pin_all_possible_cpus(); } /* diff --git a/arch/s390/numa/toptree.c b/arch/s390/numa/toptree.c index 902d350..26f622b 100644 --- a/arch/s390/numa/toptree.c +++ b/arch/s390/numa/toptree.c @@ -7,6 +7,7 @@ */ #include <linux/kernel.h> +#include <linux/bootmem.h> #include <linux/cpumask.h> #include <linux/list.h> #include <linux/list_sort.h> @@ -25,10 +26,14 @@ * RETURNS: * Pointer to the new tree node or NULL on error */ -struct toptree *toptree_alloc(int level, int id) +struct toptree __ref *toptree_alloc(int level, int id) { - struct toptree *res = kzalloc(sizeof(struct toptree), GFP_KERNEL); + struct toptree *res; + if (slab_is_available()) + res = kzalloc(sizeof(*res), GFP_KERNEL); + else + res = memblock_virt_alloc(sizeof(*res), 8); if (!res) return res; @@ -65,7 +70,7 @@ static void toptree_remove(struct toptree *cand) * cleanly using toptree_remove. Possible children are freed * recursively. In the end @cand itself is freed. */ -void toptree_free(struct toptree *cand) +void __ref toptree_free(struct toptree *cand) { struct toptree *child, *tmp; @@ -73,7 +78,10 @@ void toptree_free(struct toptree *cand) toptree_remove(cand); toptree_for_each_child_safe(child, tmp, cand) toptree_free(child); - kfree(cand); + if (slab_is_available()) + kfree(cand); + else + memblock_free_early((unsigned long)cand, sizeof(*cand)); } /** diff --git a/arch/s390/pci/pci.c b/arch/s390/pci/pci.c index 15ffc19..64e1734 100644 --- a/arch/s390/pci/pci.c +++ b/arch/s390/pci/pci.c @@ -722,6 +722,11 @@ struct dev_pm_ops pcibios_pm_ops = { static int zpci_alloc_domain(struct zpci_dev *zdev) { + if (zpci_unique_uid) { + zdev->domain = (u16) zdev->uid; + return 0; + } + spin_lock(&zpci_domain_lock); zdev->domain = find_first_zero_bit(zpci_domain, ZPCI_NR_DEVICES); if (zdev->domain == ZPCI_NR_DEVICES) { @@ -735,6 +740,9 @@ static int zpci_alloc_domain(struct zpci_dev *zdev) static void zpci_free_domain(struct zpci_dev *zdev) { + if (zpci_unique_uid) + return; + spin_lock(&zpci_domain_lock); clear_bit(zdev->domain, zpci_domain); spin_unlock(&zpci_domain_lock); diff --git a/arch/s390/pci/pci_clp.c b/arch/s390/pci/pci_clp.c index 1a4512c..e3ef63b 100644 --- a/arch/s390/pci/pci_clp.c +++ b/arch/s390/pci/pci_clp.c @@ -22,6 +22,8 @@ #include <asm/clp.h> #include <uapi/asm/clp.h> +bool zpci_unique_uid; + static inline void zpci_err_clp(unsigned int rsp, int rc) { struct { @@ -315,6 +317,7 @@ static int clp_list_pci(struct clp_req_rsp_list_pci *rrb, goto out; } + zpci_unique_uid = rrb->response.uid_checking; WARN_ON_ONCE(rrb->response.entry_size != sizeof(struct clp_fh_list_entry)); diff --git a/arch/s390/pci/pci_debug.c b/arch/s390/pci/pci_debug.c index 38993b1..c2f786f 100644 --- a/arch/s390/pci/pci_debug.c +++ b/arch/s390/pci/pci_debug.c @@ -69,7 +69,7 @@ static void pci_sw_counter_show(struct seq_file *m) int i; for (i = 0; i < ARRAY_SIZE(pci_sw_names); i++, counter++) - seq_printf(m, "%26s:\t%llu\n", pci_sw_names[i], + seq_printf(m, "%26s:\t%lu\n", pci_sw_names[i], atomic64_read(counter)); } diff --git a/arch/s390/pci/pci_dma.c b/arch/s390/pci/pci_dma.c index 6b2f72f..1d7a9c7 100644 --- a/arch/s390/pci/pci_dma.c +++ b/arch/s390/pci/pci_dma.c @@ -181,14 +181,17 @@ static int __dma_purge_tlb(struct zpci_dev *zdev, dma_addr_t dma_addr, /* * With zdev->tlb_refresh == 0, rpcit is not required to establish new * translations when previously invalid translation-table entries are - * validated. With lazy unmap, it also is skipped for previously valid + * validated. With lazy unmap, rpcit is skipped for previously valid * entries, but a global rpcit is then required before any address can * be re-used, i.e. after each iommu bitmap wrap-around. */ - if (!zdev->tlb_refresh && - (!s390_iommu_strict || - ((flags & ZPCI_PTE_VALID_MASK) == ZPCI_PTE_VALID))) - return 0; + if ((flags & ZPCI_PTE_VALID_MASK) == ZPCI_PTE_VALID) { + if (!zdev->tlb_refresh) + return 0; + } else { + if (!s390_iommu_strict) + return 0; + } return zpci_refresh_trans((u64) zdev->fh << 32, dma_addr, PAGE_ALIGN(size)); @@ -257,7 +260,7 @@ static dma_addr_t dma_alloc_address(struct device *dev, int size) spin_lock_irqsave(&zdev->iommu_bitmap_lock, flags); offset = __dma_alloc_iommu(dev, zdev->next_bit, size); if (offset == -1) { - if (!zdev->tlb_refresh && !s390_iommu_strict) { + if (!s390_iommu_strict) { /* global flush before DMA addresses are reused */ if (zpci_refresh_global(zdev)) goto out_error; @@ -292,7 +295,7 @@ static void dma_free_address(struct device *dev, dma_addr_t dma_addr, int size) if (!zdev->iommu_bitmap) goto out; - if (zdev->tlb_refresh || s390_iommu_strict) + if (s390_iommu_strict) bitmap_clear(zdev->iommu_bitmap, offset, size); else bitmap_set(zdev->lazy_bitmap, offset, size); @@ -388,8 +391,6 @@ static void *s390_dma_alloc(struct device *dev, size_t size, return NULL; pa = page_to_phys(page); - memset((void *) pa, 0, size); - map = s390_dma_map_pages(dev, page, 0, size, DMA_BIDIRECTIONAL, 0); if (dma_mapping_error(dev, map)) { free_pages(pa, get_order(size)); @@ -419,6 +420,7 @@ static int __s390_dma_map_sg(struct device *dev, struct scatterlist *sg, size_t size, dma_addr_t *handle, enum dma_data_direction dir) { + unsigned long nr_pages = PAGE_ALIGN(size) >> PAGE_SHIFT; struct zpci_dev *zdev = to_zpci(to_pci_dev(dev)); dma_addr_t dma_addr_base, dma_addr; int flags = ZPCI_PTE_VALID; @@ -426,8 +428,7 @@ static int __s390_dma_map_sg(struct device *dev, struct scatterlist *sg, unsigned long pa = 0; int ret; - size = PAGE_ALIGN(size); - dma_addr_base = dma_alloc_address(dev, size >> PAGE_SHIFT); + dma_addr_base = dma_alloc_address(dev, nr_pages); if (dma_addr_base == DMA_ERROR_CODE) return -ENOMEM; @@ -436,26 +437,27 @@ static int __s390_dma_map_sg(struct device *dev, struct scatterlist *sg, flags |= ZPCI_TABLE_PROTECTED; for (s = sg; dma_addr < dma_addr_base + size; s = sg_next(s)) { - pa = page_to_phys(sg_page(s)) + s->offset; - ret = __dma_update_trans(zdev, pa, dma_addr, s->length, flags); + pa = page_to_phys(sg_page(s)); + ret = __dma_update_trans(zdev, pa, dma_addr, + s->offset + s->length, flags); if (ret) goto unmap; - dma_addr += s->length; + dma_addr += s->offset + s->length; } ret = __dma_purge_tlb(zdev, dma_addr_base, size, flags); if (ret) goto unmap; *handle = dma_addr_base; - atomic64_add(size >> PAGE_SHIFT, &zdev->mapped_pages); + atomic64_add(nr_pages, &zdev->mapped_pages); return ret; unmap: dma_update_trans(zdev, 0, dma_addr_base, dma_addr - dma_addr_base, ZPCI_PTE_INVALID); - dma_free_address(dev, dma_addr_base, size >> PAGE_SHIFT); + dma_free_address(dev, dma_addr_base, nr_pages); zpci_err("map error:\n"); zpci_err_dma(ret, pa); return ret; @@ -564,7 +566,7 @@ int zpci_dma_init_device(struct zpci_dev *zdev) rc = -ENOMEM; goto free_dma_table; } - if (!zdev->tlb_refresh && !s390_iommu_strict) { + if (!s390_iommu_strict) { zdev->lazy_bitmap = vzalloc(zdev->iommu_pages / 8); if (!zdev->lazy_bitmap) { rc = -ENOMEM; diff --git a/arch/s390/tools/Makefile b/arch/s390/tools/Makefile index 6d9814c..4b5e1e4 100644 --- a/arch/s390/tools/Makefile +++ b/arch/s390/tools/Makefile @@ -9,7 +9,5 @@ define filechk_facilities.h $(obj)/gen_facilities endef -$(obj)/gen_facilities.o: $(srctree)/arch/s390/tools/gen_facilities.c - include/generated/facilities.h: $(obj)/gen_facilities FORCE $(call filechk,facilities.h) diff --git a/arch/s390/tools/gen_facilities.c b/arch/s390/tools/gen_facilities.c index fe4e6c9..8cc53b1 100644 --- a/arch/s390/tools/gen_facilities.c +++ b/arch/s390/tools/gen_facilities.c @@ -7,13 +7,83 @@ * */ -#define S390_GEN_FACILITIES_C - #include <strings.h> #include <string.h> #include <stdlib.h> #include <stdio.h> -#include <asm/facilities_src.h> + +struct facility_def { + char *name; + int *bits; +}; + +static struct facility_def facility_defs[] = { + { + /* + * FACILITIES_ALS contains the list of facilities that are + * required to run a kernel that is compiled e.g. with + * -march=<machine>. + */ + .name = "FACILITIES_ALS", + .bits = (int[]){ +#ifdef CONFIG_HAVE_MARCH_Z900_FEATURES + 0, /* N3 instructions */ + 1, /* z/Arch mode installed */ +#endif +#ifdef CONFIG_HAVE_MARCH_Z990_FEATURES + 18, /* long displacement facility */ +#endif +#ifdef CONFIG_HAVE_MARCH_Z9_109_FEATURES + 7, /* stfle */ + 17, /* message security assist */ + 21, /* extended-immediate facility */ + 25, /* store clock fast */ +#endif +#ifdef CONFIG_HAVE_MARCH_Z10_FEATURES + 27, /* mvcos */ + 32, /* compare and swap and store */ + 33, /* compare and swap and store 2 */ + 34, /* general extension facility */ + 35, /* execute extensions */ +#endif +#ifdef CONFIG_HAVE_MARCH_Z196_FEATURES + 45, /* fast-BCR, etc. */ +#endif +#ifdef CONFIG_HAVE_MARCH_ZEC12_FEATURES + 49, /* misc-instruction-extensions */ + 52, /* interlocked facility 2 */ +#endif +#ifdef CONFIG_HAVE_MARCH_Z13_FEATURES + 53, /* load-and-zero-rightmost-byte, etc. */ +#endif + -1 /* END */ + } + }, + { + .name = "FACILITIES_KVM", + .bits = (int[]){ + 0, /* N3 instructions */ + 1, /* z/Arch mode installed */ + 2, /* z/Arch mode active */ + 3, /* DAT-enhancement */ + 4, /* idte segment table */ + 5, /* idte region table */ + 6, /* ASN-and-LX reuse */ + 7, /* stfle */ + 8, /* enhanced-DAT 1 */ + 9, /* sense-running-status */ + 10, /* conditional sske */ + 13, /* ipte-range */ + 14, /* nonquiescing key-setting */ + 73, /* transactional execution */ + 75, /* access-exception-fetch/store indication */ + 76, /* msa extension 3 */ + 77, /* msa extension 4 */ + 78, /* enhanced-DAT 2 */ + -1 /* END */ + } + }, +}; static void print_facility_list(struct facility_def *def) { diff --git a/arch/x86/include/asm/e820.h b/arch/x86/include/asm/e820.h index 476b574..ec23d8e 100644 --- a/arch/x86/include/asm/e820.h +++ b/arch/x86/include/asm/e820.h @@ -1,13 +1,17 @@ #ifndef _ASM_X86_E820_H #define _ASM_X86_E820_H -#ifdef CONFIG_EFI +/* + * E820_X_MAX is the maximum size of the extended E820 table. The extended + * table may contain up to 3 extra E820 entries per possible NUMA node, so we + * make room for 3 * MAX_NUMNODES possible entries, beyond the standard 128. + * Also note that E820_X_MAX *must* be defined before we include uapi/asm/e820.h. + */ #include <linux/numa.h> #define E820_X_MAX (E820MAX + 3 * MAX_NUMNODES) -#else /* ! CONFIG_EFI */ -#define E820_X_MAX E820MAX -#endif + #include <uapi/asm/e820.h> + #ifndef __ASSEMBLY__ /* see comment in arch/x86/kernel/e820.c */ extern struct e820map *e820; diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h index bdde807..7892530 100644 --- a/arch/x86/include/asm/kvm_host.h +++ b/arch/x86/include/asm/kvm_host.h @@ -191,6 +191,8 @@ enum { #define PFERR_RSVD_BIT 3 #define PFERR_FETCH_BIT 4 #define PFERR_PK_BIT 5 +#define PFERR_GUEST_FINAL_BIT 32 +#define PFERR_GUEST_PAGE_BIT 33 #define PFERR_PRESENT_MASK (1U << PFERR_PRESENT_BIT) #define PFERR_WRITE_MASK (1U << PFERR_WRITE_BIT) @@ -198,6 +200,13 @@ enum { #define PFERR_RSVD_MASK (1U << PFERR_RSVD_BIT) #define PFERR_FETCH_MASK (1U << PFERR_FETCH_BIT) #define PFERR_PK_MASK (1U << PFERR_PK_BIT) +#define PFERR_GUEST_FINAL_MASK (1ULL << PFERR_GUEST_FINAL_BIT) +#define PFERR_GUEST_PAGE_MASK (1ULL << PFERR_GUEST_PAGE_BIT) + +#define PFERR_NESTED_GUEST_PAGE (PFERR_GUEST_PAGE_MASK | \ + PFERR_USER_MASK | \ + PFERR_WRITE_MASK | \ + PFERR_PRESENT_MASK) /* apic attention bits */ #define KVM_APIC_CHECK_VAPIC 0 @@ -1062,6 +1071,7 @@ unsigned int kvm_mmu_calculate_mmu_pages(struct kvm *kvm); void kvm_mmu_change_mmu_pages(struct kvm *kvm, unsigned int kvm_nr_mmu_pages); int load_pdptrs(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu, unsigned long cr3); +bool pdptrs_changed(struct kvm_vcpu *vcpu); int emulator_write_phys(struct kvm_vcpu *vcpu, gpa_t gpa, const void *val, int bytes); @@ -1124,7 +1134,8 @@ int kvm_set_msr(struct kvm_vcpu *vcpu, struct msr_data *msr); struct x86_emulate_ctxt; int kvm_fast_pio_out(struct kvm_vcpu *vcpu, int size, unsigned short port); -void kvm_emulate_cpuid(struct kvm_vcpu *vcpu); +int kvm_fast_pio_in(struct kvm_vcpu *vcpu, int size, unsigned short port); +int kvm_emulate_cpuid(struct kvm_vcpu *vcpu); int kvm_emulate_halt(struct kvm_vcpu *vcpu); int kvm_vcpu_halt(struct kvm_vcpu *vcpu); int kvm_emulate_wbinvd(struct kvm_vcpu *vcpu); @@ -1203,7 +1214,7 @@ void kvm_vcpu_deactivate_apicv(struct kvm_vcpu *vcpu); int kvm_emulate_hypercall(struct kvm_vcpu *vcpu); -int kvm_mmu_page_fault(struct kvm_vcpu *vcpu, gva_t gva, u32 error_code, +int kvm_mmu_page_fault(struct kvm_vcpu *vcpu, gva_t gva, u64 error_code, void *insn, int insn_len); void kvm_mmu_invlpg(struct kvm_vcpu *vcpu, gva_t gva); void kvm_mmu_new_cr3(struct kvm_vcpu *vcpu); @@ -1358,7 +1369,8 @@ void kvm_arch_async_page_ready(struct kvm_vcpu *vcpu, bool kvm_arch_can_inject_async_page_present(struct kvm_vcpu *vcpu); extern bool kvm_find_async_pf_gfn(struct kvm_vcpu *vcpu, gfn_t gfn); -void kvm_complete_insn_gp(struct kvm_vcpu *vcpu, int err); +int kvm_skip_emulated_instruction(struct kvm_vcpu *vcpu); +int kvm_complete_insn_gp(struct kvm_vcpu *vcpu, int err); int kvm_is_in_guest(void); diff --git a/arch/x86/include/asm/vmx.h b/arch/x86/include/asm/vmx.h index a002b07..2b5b2d4 100644 --- a/arch/x86/include/asm/vmx.h +++ b/arch/x86/include/asm/vmx.h @@ -25,6 +25,7 @@ #define VMX_H +#include <linux/bitops.h> #include <linux/types.h> #include <uapi/asm/vmx.h> @@ -60,6 +61,7 @@ */ #define SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES 0x00000001 #define SECONDARY_EXEC_ENABLE_EPT 0x00000002 +#define SECONDARY_EXEC_DESC 0x00000004 #define SECONDARY_EXEC_RDTSCP 0x00000008 #define SECONDARY_EXEC_VIRTUALIZE_X2APIC_MODE 0x00000010 #define SECONDARY_EXEC_ENABLE_VPID 0x00000020 @@ -110,6 +112,36 @@ #define VMX_MISC_SAVE_EFER_LMA 0x00000020 #define VMX_MISC_ACTIVITY_HLT 0x00000040 +static inline u32 vmx_basic_vmcs_revision_id(u64 vmx_basic) +{ + return vmx_basic & GENMASK_ULL(30, 0); +} + +static inline u32 vmx_basic_vmcs_size(u64 vmx_basic) +{ + return (vmx_basic & GENMASK_ULL(44, 32)) >> 32; +} + +static inline int vmx_misc_preemption_timer_rate(u64 vmx_misc) +{ + return vmx_misc & VMX_MISC_PREEMPTION_TIMER_RATE_MASK; +} + +static inline int vmx_misc_cr3_count(u64 vmx_misc) +{ + return (vmx_misc & GENMASK_ULL(24, 16)) >> 16; +} + +static inline int vmx_misc_max_msr(u64 vmx_misc) +{ + return (vmx_misc & GENMASK_ULL(27, 25)) >> 25; +} + +static inline int vmx_misc_mseg_revid(u64 vmx_misc) +{ + return (vmx_misc & GENMASK_ULL(63, 32)) >> 32; +} + /* VMCS Encodings */ enum vmcs_field { VIRTUAL_PROCESSOR_ID = 0x00000000, @@ -399,10 +431,11 @@ enum vmcs_field { #define IDENTITY_PAGETABLE_PRIVATE_MEMSLOT (KVM_USER_MEM_SLOTS + 2) #define VMX_NR_VPIDS (1 << 16) +#define VMX_VPID_EXTENT_INDIVIDUAL_ADDR 0 #define VMX_VPID_EXTENT_SINGLE_CONTEXT 1 #define VMX_VPID_EXTENT_ALL_CONTEXT 2 +#define VMX_VPID_EXTENT_SINGLE_NON_GLOBAL 3 -#define VMX_EPT_EXTENT_INDIVIDUAL_ADDR 0 #define VMX_EPT_EXTENT_CONTEXT 1 #define VMX_EPT_EXTENT_GLOBAL 2 #define VMX_EPT_EXTENT_SHIFT 24 @@ -419,8 +452,10 @@ enum vmcs_field { #define VMX_EPT_EXTENT_GLOBAL_BIT (1ull << 26) #define VMX_VPID_INVVPID_BIT (1ull << 0) /* (32 - 32) */ +#define VMX_VPID_EXTENT_INDIVIDUAL_ADDR_BIT (1ull << 8) /* (40 - 32) */ #define VMX_VPID_EXTENT_SINGLE_CONTEXT_BIT (1ull << 9) /* (41 - 32) */ #define VMX_VPID_EXTENT_GLOBAL_CONTEXT_BIT (1ull << 10) /* (42 - 32) */ +#define VMX_VPID_EXTENT_SINGLE_NON_GLOBAL_BIT (1ull << 11) /* (43 - 32) */ #define VMX_EPT_DEFAULT_GAW 3 #define VMX_EPT_MAX_GAW 0x4 diff --git a/arch/x86/include/uapi/asm/vmx.h b/arch/x86/include/uapi/asm/vmx.h index 37fee27..1445865 100644 --- a/arch/x86/include/uapi/asm/vmx.h +++ b/arch/x86/include/uapi/asm/vmx.h @@ -65,6 +65,8 @@ #define EXIT_REASON_TPR_BELOW_THRESHOLD 43 #define EXIT_REASON_APIC_ACCESS 44 #define EXIT_REASON_EOI_INDUCED 45 +#define EXIT_REASON_GDTR_IDTR 46 +#define EXIT_REASON_LDTR_TR 47 #define EXIT_REASON_EPT_VIOLATION 48 #define EXIT_REASON_EPT_MISCONFIG 49 #define EXIT_REASON_INVEPT 50 @@ -113,6 +115,8 @@ { EXIT_REASON_MCE_DURING_VMENTRY, "MCE_DURING_VMENTRY" }, \ { EXIT_REASON_TPR_BELOW_THRESHOLD, "TPR_BELOW_THRESHOLD" }, \ { EXIT_REASON_APIC_ACCESS, "APIC_ACCESS" }, \ + { EXIT_REASON_GDTR_IDTR, "GDTR_IDTR" }, \ + { EXIT_REASON_LDTR_TR, "LDTR_TR" }, \ { EXIT_REASON_EPT_VIOLATION, "EPT_VIOLATION" }, \ { EXIT_REASON_EPT_MISCONFIG, "EPT_MISCONFIG" }, \ { EXIT_REASON_INVEPT, "INVEPT" }, \ @@ -129,6 +133,7 @@ { EXIT_REASON_XRSTORS, "XRSTORS" } #define VMX_ABORT_SAVE_GUEST_MSR_FAIL 1 +#define VMX_ABORT_LOAD_HOST_PDPTE_FAIL 2 #define VMX_ABORT_LOAD_HOST_MSR_FAIL 4 #endif /* _UAPIVMX_H */ diff --git a/arch/x86/kernel/cpu/mcheck/mce.c b/arch/x86/kernel/cpu/mcheck/mce.c index 132e1ec..00ef432 100644 --- a/arch/x86/kernel/cpu/mcheck/mce.c +++ b/arch/x86/kernel/cpu/mcheck/mce.c @@ -516,7 +516,7 @@ int mce_available(struct cpuinfo_x86 *c) static void mce_schedule_work(void) { - if (!mce_gen_pool_empty() && keventd_up()) + if (!mce_gen_pool_empty()) schedule_work(&mce_work); } diff --git a/arch/x86/kvm/cpuid.c b/arch/x86/kvm/cpuid.c index 0aefb62..b2d3cf1 100644 --- a/arch/x86/kvm/cpuid.c +++ b/arch/x86/kvm/cpuid.c @@ -16,6 +16,7 @@ #include <linux/export.h> #include <linux/vmalloc.h> #include <linux/uaccess.h> +#include <asm/processor.h> #include <asm/user.h> #include <asm/fpu/xstate.h> #include "cpuid.h" @@ -64,6 +65,11 @@ u64 kvm_supported_xcr0(void) #define F(x) bit(X86_FEATURE_##x) +/* These are scattered features in cpufeatures.h. */ +#define KVM_CPUID_BIT_AVX512_4VNNIW 2 +#define KVM_CPUID_BIT_AVX512_4FMAPS 3 +#define KF(x) bit(KVM_CPUID_BIT_##x) + int kvm_update_cpuid(struct kvm_vcpu *vcpu) { struct kvm_cpuid_entry2 *best; @@ -80,6 +86,10 @@ int kvm_update_cpuid(struct kvm_vcpu *vcpu) best->ecx |= F(OSXSAVE); } + best->edx &= ~F(APIC); + if (vcpu->arch.apic_base & MSR_IA32_APICBASE_ENABLE) + best->edx |= F(APIC); + if (apic) { if (best->ecx & F(TSC_DEADLINE_TIMER)) apic->lapic_timer.timer_mode_mask = 3 << 17; @@ -374,6 +384,10 @@ static inline int __do_cpuid_ent(struct kvm_cpuid_entry2 *entry, u32 function, /* cpuid 7.0.ecx*/ const u32 kvm_cpuid_7_0_ecx_x86_features = F(PKU) | 0 /*OSPKE*/; + /* cpuid 7.0.edx*/ + const u32 kvm_cpuid_7_0_edx_x86_features = + KF(AVX512_4VNNIW) | KF(AVX512_4FMAPS); + /* all calls to cpuid_count() should be made on the same cpu */ get_cpu(); @@ -456,12 +470,14 @@ static inline int __do_cpuid_ent(struct kvm_cpuid_entry2 *entry, u32 function, /* PKU is not yet implemented for shadow paging. */ if (!tdp_enabled) entry->ecx &= ~F(PKU); + entry->edx &= kvm_cpuid_7_0_edx_x86_features; + entry->edx &= get_scattered_cpuid_leaf(7, 0, CPUID_EDX); } else { entry->ebx = 0; entry->ecx = 0; + entry->edx = 0; } entry->eax = 0; - entry->edx = 0; break; } case 9: @@ -861,17 +877,17 @@ void kvm_cpuid(struct kvm_vcpu *vcpu, u32 *eax, u32 *ebx, u32 *ecx, u32 *edx) } EXPORT_SYMBOL_GPL(kvm_cpuid); -void kvm_emulate_cpuid(struct kvm_vcpu *vcpu) +int kvm_emulate_cpuid(struct kvm_vcpu *vcpu) { - u32 function, eax, ebx, ecx, edx; + u32 eax, ebx, ecx, edx; - function = eax = kvm_register_read(vcpu, VCPU_REGS_RAX); + eax = kvm_register_read(vcpu, VCPU_REGS_RAX); ecx = kvm_register_read(vcpu, VCPU_REGS_RCX); kvm_cpuid(vcpu, &eax, &ebx, &ecx, &edx); kvm_register_write(vcpu, VCPU_REGS_RAX, eax); kvm_register_write(vcpu, VCPU_REGS_RBX, ebx); kvm_register_write(vcpu, VCPU_REGS_RCX, ecx); kvm_register_write(vcpu, VCPU_REGS_RDX, edx); - kvm_x86_ops->skip_emulated_instruction(vcpu); + return kvm_skip_emulated_instruction(vcpu); } EXPORT_SYMBOL_GPL(kvm_emulate_cpuid); diff --git a/arch/x86/kvm/emulate.c b/arch/x86/kvm/emulate.c index a3ce9d2..56628a4 100644 --- a/arch/x86/kvm/emulate.c +++ b/arch/x86/kvm/emulate.c @@ -158,9 +158,11 @@ #define Src2GS (OpGS << Src2Shift) #define Src2Mask (OpMask << Src2Shift) #define Mmx ((u64)1 << 40) /* MMX Vector instruction */ +#define AlignMask ((u64)7 << 41) #define Aligned ((u64)1 << 41) /* Explicitly aligned (e.g. MOVDQA) */ -#define Unaligned ((u64)1 << 42) /* Explicitly unaligned (e.g. MOVDQU) */ -#define Avx ((u64)1 << 43) /* Advanced Vector Extensions */ +#define Unaligned ((u64)2 << 41) /* Explicitly unaligned (e.g. MOVDQU) */ +#define Avx ((u64)3 << 41) /* Advanced Vector Extensions */ +#define Aligned16 ((u64)4 << 41) /* Aligned to 16 byte boundary (e.g. FXSAVE) */ #define Fastop ((u64)1 << 44) /* Use opcode::u.fastop */ #define NoWrite ((u64)1 << 45) /* No writeback */ #define SrcWrite ((u64)1 << 46) /* Write back src operand */ @@ -446,6 +448,26 @@ FOP_END; FOP_START(salc) "pushf; sbb %al, %al; popf \n\t" FOP_RET FOP_END; +/* + * XXX: inoutclob user must know where the argument is being expanded. + * Relying on CC_HAVE_ASM_GOTO would allow us to remove _fault. + */ +#define asm_safe(insn, inoutclob...) \ +({ \ + int _fault = 0; \ + \ + asm volatile("1:" insn "\n" \ + "2:\n" \ + ".pushsection .fixup, \"ax\"\n" \ + "3: movl $1, %[_fault]\n" \ + " jmp 2b\n" \ + ".popsection\n" \ + _ASM_EXTABLE(1b, 3b) \ + : [_fault] "+qm"(_fault) inoutclob ); \ + \ + _fault ? X86EMUL_UNHANDLEABLE : X86EMUL_CONTINUE; \ +}) + static int emulator_check_intercept(struct x86_emulate_ctxt *ctxt, enum x86_intercept intercept, enum x86_intercept_stage stage) @@ -632,21 +654,26 @@ static void set_segment_selector(struct x86_emulate_ctxt *ctxt, u16 selector, * depending on whether they're AVX encoded or not. * * Also included is CMPXCHG16B which is not a vector instruction, yet it is - * subject to the same check. + * subject to the same check. FXSAVE and FXRSTOR are checked here too as their + * 512 bytes of data must be aligned to a 16 byte boundary. */ -static bool insn_aligned(struct x86_emulate_ctxt *ctxt, unsigned size) +static unsigned insn_alignment(struct x86_emulate_ctxt *ctxt, unsigned size) { - if (likely(size < 16)) - return false; + u64 alignment = ctxt->d & AlignMask; - if (ctxt->d & Aligned) - return true; - else if (ctxt->d & Unaligned) - return false; - else if (ctxt->d & Avx) - return false; - else - return true; + if (likely(size < 16)) + return 1; + + switch (alignment) { + case Unaligned: + case Avx: + return 1; + case Aligned16: + return 16; + case Aligned: + default: + return size; + } } static __always_inline int __linearize(struct x86_emulate_ctxt *ctxt, @@ -704,7 +731,7 @@ static __always_inline int __linearize(struct x86_emulate_ctxt *ctxt, } break; } - if (insn_aligned(ctxt, size) && ((la & (size - 1)) != 0)) + if (la & (insn_alignment(ctxt, size) - 1)) return emulate_gp(ctxt, 0); return X86EMUL_CONTINUE; bad: @@ -3842,6 +3869,131 @@ static int em_movsxd(struct x86_emulate_ctxt *ctxt) return X86EMUL_CONTINUE; } +static int check_fxsr(struct x86_emulate_ctxt *ctxt) +{ + u32 eax = 1, ebx, ecx = 0, edx; + + ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx); + if (!(edx & FFL(FXSR))) + return emulate_ud(ctxt); + + if (ctxt->ops->get_cr(ctxt, 0) & (X86_CR0_TS | X86_CR0_EM)) + return emulate_nm(ctxt); + + /* + * Don't emulate a case that should never be hit, instead of working + * around a lack of fxsave64/fxrstor64 on old compilers. + */ + if (ctxt->mode >= X86EMUL_MODE_PROT64) + return X86EMUL_UNHANDLEABLE; + + return X86EMUL_CONTINUE; +} + +/* + * FXSAVE and FXRSTOR have 4 different formats depending on execution mode, + * 1) 16 bit mode + * 2) 32 bit mode + * - like (1), but FIP and FDP (foo) are only 16 bit. At least Intel CPUs + * preserve whole 32 bit values, though, so (1) and (2) are the same wrt. + * save and restore + * 3) 64-bit mode with REX.W prefix + * - like (2), but XMM 8-15 are being saved and restored + * 4) 64-bit mode without REX.W prefix + * - like (3), but FIP and FDP are 64 bit + * + * Emulation uses (3) for (1) and (2) and preserves XMM 8-15 to reach the + * desired result. (4) is not emulated. + * + * Note: Guest and host CPUID.(EAX=07H,ECX=0H):EBX[bit 13] (deprecate FPU CS + * and FPU DS) should match. + */ +static int em_fxsave(struct x86_emulate_ctxt *ctxt) +{ + struct fxregs_state fx_state; + size_t size; + int rc; + + rc = check_fxsr(ctxt); + if (rc != X86EMUL_CONTINUE) + return rc; + + ctxt->ops->get_fpu(ctxt); + + rc = asm_safe("fxsave %[fx]", , [fx] "+m"(fx_state)); + + ctxt->ops->put_fpu(ctxt); + + if (rc != X86EMUL_CONTINUE) + return rc; + + if (ctxt->ops->get_cr(ctxt, 4) & X86_CR4_OSFXSR) + size = offsetof(struct fxregs_state, xmm_space[8 * 16/4]); + else + size = offsetof(struct fxregs_state, xmm_space[0]); + + return segmented_write(ctxt, ctxt->memop.addr.mem, &fx_state, size); +} + +static int fxrstor_fixup(struct x86_emulate_ctxt *ctxt, + struct fxregs_state *new) +{ + int rc = X86EMUL_CONTINUE; + struct fxregs_state old; + + rc = asm_safe("fxsave %[fx]", , [fx] "+m"(old)); + if (rc != X86EMUL_CONTINUE) + return rc; + + /* + * 64 bit host will restore XMM 8-15, which is not correct on non-64 + * bit guests. Load the current values in order to preserve 64 bit + * XMMs after fxrstor. + */ +#ifdef CONFIG_X86_64 + /* XXX: accessing XMM 8-15 very awkwardly */ + memcpy(&new->xmm_space[8 * 16/4], &old.xmm_space[8 * 16/4], 8 * 16); +#endif + + /* + * Hardware doesn't save and restore XMM 0-7 without CR4.OSFXSR, but + * does save and restore MXCSR. + */ + if (!(ctxt->ops->get_cr(ctxt, 4) & X86_CR4_OSFXSR)) + memcpy(new->xmm_space, old.xmm_space, 8 * 16); + + return rc; +} + +static int em_fxrstor(struct x86_emulate_ctxt *ctxt) +{ + struct fxregs_state fx_state; + int rc; + + rc = check_fxsr(ctxt); + if (rc != X86EMUL_CONTINUE) + return rc; + + rc = segmented_read(ctxt, ctxt->memop.addr.mem, &fx_state, 512); + if (rc != X86EMUL_CONTINUE) + return rc; + + if (fx_state.mxcsr >> 16) + return emulate_gp(ctxt, 0); + + ctxt->ops->get_fpu(ctxt); + + if (ctxt->mode < X86EMUL_MODE_PROT64) + rc = fxrstor_fixup(ctxt, &fx_state); + + if (rc == X86EMUL_CONTINUE) + rc = asm_safe("fxrstor %[fx]", : [fx] "m"(fx_state)); + + ctxt->ops->put_fpu(ctxt); + + return rc; +} + static bool valid_cr(int nr) { switch (nr) { @@ -4194,7 +4346,9 @@ static const struct gprefix pfx_0f_ae_7 = { }; static const struct group_dual group15 = { { - N, N, N, N, N, N, N, GP(0, &pfx_0f_ae_7), + I(ModRM | Aligned16, em_fxsave), + I(ModRM | Aligned16, em_fxrstor), + N, N, N, N, N, GP(0, &pfx_0f_ae_7), }, { N, N, N, N, N, N, N, N, } }; @@ -5066,21 +5220,13 @@ static bool string_insn_completed(struct x86_emulate_ctxt *ctxt) static int flush_pending_x87_faults(struct x86_emulate_ctxt *ctxt) { - bool fault = false; + int rc; ctxt->ops->get_fpu(ctxt); - asm volatile("1: fwait \n\t" - "2: \n\t" - ".pushsection .fixup,\"ax\" \n\t" - "3: \n\t" - "movb $1, %[fault] \n\t" - "jmp 2b \n\t" - ".popsection \n\t" - _ASM_EXTABLE(1b, 3b) - : [fault]"+qm"(fault)); + rc = asm_safe("fwait"); ctxt->ops->put_fpu(ctxt); - if (unlikely(fault)) + if (unlikely(rc != X86EMUL_CONTINUE)) return emulate_exception(ctxt, MF_VECTOR, 0, false); return X86EMUL_CONTINUE; diff --git a/arch/x86/kvm/hyperv.c b/arch/x86/kvm/hyperv.c index 42b1c83..99cde52 100644 --- a/arch/x86/kvm/hyperv.c +++ b/arch/x86/kvm/hyperv.c @@ -291,7 +291,7 @@ static int synic_get_msr(struct kvm_vcpu_hv_synic *synic, u32 msr, u64 *pdata) return ret; } -int synic_set_irq(struct kvm_vcpu_hv_synic *synic, u32 sint) +static int synic_set_irq(struct kvm_vcpu_hv_synic *synic, u32 sint) { struct kvm_vcpu *vcpu = synic_to_vcpu(synic); struct kvm_lapic_irq irq; diff --git a/arch/x86/kvm/i8254.c b/arch/x86/kvm/i8254.c index 16a7134..a78b445 100644 --- a/arch/x86/kvm/i8254.c +++ b/arch/x86/kvm/i8254.c @@ -212,7 +212,7 @@ static void kvm_pit_ack_irq(struct kvm_irq_ack_notifier *kian) */ smp_mb(); if (atomic_dec_if_positive(&ps->pending) > 0) - kthread_queue_work(&pit->worker, &pit->expired); + kthread_queue_work(pit->worker, &pit->expired); } void __kvm_migrate_pit_timer(struct kvm_vcpu *vcpu) @@ -272,7 +272,7 @@ static enum hrtimer_restart pit_timer_fn(struct hrtimer *data) if (atomic_read(&ps->reinject)) atomic_inc(&ps->pending); - kthread_queue_work(&pt->worker, &pt->expired); + kthread_queue_work(pt->worker, &pt->expired); if (ps->is_periodic) { hrtimer_add_expires_ns(&ps->timer, ps->period); @@ -667,10 +667,8 @@ struct kvm_pit *kvm_create_pit(struct kvm *kvm, u32 flags) pid_nr = pid_vnr(pid); put_pid(pid); - kthread_init_worker(&pit->worker); - pit->worker_task = kthread_run(kthread_worker_fn, &pit->worker, - "kvm-pit/%d", pid_nr); - if (IS_ERR(pit->worker_task)) + pit->worker = kthread_create_worker(0, "kvm-pit/%d", pid_nr); + if (IS_ERR(pit->worker)) goto fail_kthread; kthread_init_work(&pit->expired, pit_do_work); @@ -713,7 +711,7 @@ fail_register_speaker: fail_register_pit: mutex_unlock(&kvm->slots_lock); kvm_pit_set_reinject(pit, false); - kthread_stop(pit->worker_task); + kthread_destroy_worker(pit->worker); fail_kthread: kvm_free_irq_source_id(kvm, pit->irq_source_id); fail_request: @@ -730,8 +728,7 @@ void kvm_free_pit(struct kvm *kvm) kvm_io_bus_unregister_dev(kvm, KVM_PIO_BUS, &pit->speaker_dev); kvm_pit_set_reinject(pit, false); hrtimer_cancel(&pit->pit_state.timer); - kthread_flush_work(&pit->expired); - kthread_stop(pit->worker_task); + kthread_destroy_worker(pit->worker); kvm_free_irq_source_id(kvm, pit->irq_source_id); kfree(pit); } diff --git a/arch/x86/kvm/i8254.h b/arch/x86/kvm/i8254.h index 2f5af07..600bee9 100644 --- a/arch/x86/kvm/i8254.h +++ b/arch/x86/kvm/i8254.h @@ -44,8 +44,7 @@ struct kvm_pit { struct kvm_kpit_state pit_state; int irq_source_id; struct kvm_irq_mask_notifier mask_notifier; - struct kthread_worker worker; - struct task_struct *worker_task; + struct kthread_worker *worker; struct kthread_work expired; }; diff --git a/arch/x86/kvm/lapic.c b/arch/x86/kvm/lapic.c index 6f69340..34a66b2 100644 --- a/arch/x86/kvm/lapic.c +++ b/arch/x86/kvm/lapic.c @@ -342,9 +342,11 @@ void __kvm_apic_update_irr(u32 *pir, void *regs) u32 i, pir_val; for (i = 0; i <= 7; i++) { - pir_val = xchg(&pir[i], 0); - if (pir_val) + pir_val = READ_ONCE(pir[i]); + if (pir_val) { + pir_val = xchg(&pir[i], 0); *((u32 *)(regs + APIC_IRR + i * 0x10)) |= pir_val; + } } } EXPORT_SYMBOL_GPL(__kvm_apic_update_irr); @@ -1090,7 +1092,7 @@ static void apic_send_ipi(struct kvm_lapic *apic) static u32 apic_get_tmcct(struct kvm_lapic *apic) { - ktime_t remaining; + ktime_t remaining, now; s64 ns; u32 tmcct; @@ -1101,7 +1103,8 @@ static u32 apic_get_tmcct(struct kvm_lapic *apic) apic->lapic_timer.period == 0) return 0; - remaining = hrtimer_get_remaining(&apic->lapic_timer.timer); + now = ktime_get(); + remaining = ktime_sub(apic->lapic_timer.target_expiration, now); if (ktime_to_ns(remaining) < 0) remaining = ktime_set(0, 0); @@ -1332,7 +1335,7 @@ static void start_sw_tscdeadline(struct kvm_lapic *apic) local_irq_save(flags); - now = apic->lapic_timer.timer.base->get_time(); + now = ktime_get(); guest_tsc = kvm_read_l1_tsc(vcpu, rdtsc()); if (likely(tscdeadline > guest_tsc)) { ns = (tscdeadline - guest_tsc) * 1000000ULL; @@ -1347,6 +1350,79 @@ static void start_sw_tscdeadline(struct kvm_lapic *apic) local_irq_restore(flags); } +static void start_sw_period(struct kvm_lapic *apic) +{ + if (!apic->lapic_timer.period) + return; + + if (apic_lvtt_oneshot(apic) && + ktime_after(ktime_get(), + apic->lapic_timer.target_expiration)) { + apic_timer_expired(apic); + return; + } + + hrtimer_start(&apic->lapic_timer.timer, + apic->lapic_timer.target_expiration, + HRTIMER_MODE_ABS_PINNED); +} + +static bool set_target_expiration(struct kvm_lapic *apic) +{ + ktime_t now; + u64 tscl = rdtsc(); + + now = ktime_get(); + apic->lapic_timer.period = (u64)kvm_lapic_get_reg(apic, APIC_TMICT) + * APIC_BUS_CYCLE_NS * apic->divide_count; + + if (!apic->lapic_timer.period) + return false; + + /* + * Do not allow the guest to program periodic timers with small + * interval, since the hrtimers are not throttled by the host + * scheduler. + */ + if (apic_lvtt_period(apic)) { + s64 min_period = min_timer_period_us * 1000LL; + + if (apic->lapic_timer.period < min_period) { + pr_info_ratelimited( + "kvm: vcpu %i: requested %lld ns " + "lapic timer period limited to %lld ns\n", + apic->vcpu->vcpu_id, + apic->lapic_timer.period, min_period); + apic->lapic_timer.period = min_period; + } + } + + apic_debug("%s: bus cycle is %" PRId64 "ns, now 0x%016" + PRIx64 ", " + "timer initial count 0x%x, period %lldns, " + "expire @ 0x%016" PRIx64 ".\n", __func__, + APIC_BUS_CYCLE_NS, ktime_to_ns(now), + kvm_lapic_get_reg(apic, APIC_TMICT), + apic->lapic_timer.period, + ktime_to_ns(ktime_add_ns(now, + apic->lapic_timer.period))); + + apic->lapic_timer.tscdeadline = kvm_read_l1_tsc(apic->vcpu, tscl) + + nsec_to_cycles(apic->vcpu, apic->lapic_timer.period); + apic->lapic_timer.target_expiration = ktime_add_ns(now, apic->lapic_timer.period); + + return true; +} + +static void advance_periodic_target_expiration(struct kvm_lapic *apic) +{ + apic->lapic_timer.tscdeadline += + nsec_to_cycles(apic->vcpu, apic->lapic_timer.period); + apic->lapic_timer.target_expiration = + ktime_add_ns(apic->lapic_timer.target_expiration, + apic->lapic_timer.period); +} + bool kvm_lapic_hv_timer_in_use(struct kvm_vcpu *vcpu) { if (!lapic_in_kernel(vcpu)) @@ -1356,52 +1432,59 @@ bool kvm_lapic_hv_timer_in_use(struct kvm_vcpu *vcpu) } EXPORT_SYMBOL_GPL(kvm_lapic_hv_timer_in_use); -static void cancel_hv_tscdeadline(struct kvm_lapic *apic) +static void cancel_hv_timer(struct kvm_lapic *apic) { kvm_x86_ops->cancel_hv_timer(apic->vcpu); apic->lapic_timer.hv_timer_in_use = false; } -void kvm_lapic_expired_hv_timer(struct kvm_vcpu *vcpu) -{ - struct kvm_lapic *apic = vcpu->arch.apic; - - WARN_ON(!apic->lapic_timer.hv_timer_in_use); - WARN_ON(swait_active(&vcpu->wq)); - cancel_hv_tscdeadline(apic); - apic_timer_expired(apic); -} -EXPORT_SYMBOL_GPL(kvm_lapic_expired_hv_timer); - -static bool start_hv_tscdeadline(struct kvm_lapic *apic) +static bool start_hv_timer(struct kvm_lapic *apic) { u64 tscdeadline = apic->lapic_timer.tscdeadline; - if (atomic_read(&apic->lapic_timer.pending) || + if ((atomic_read(&apic->lapic_timer.pending) && + !apic_lvtt_period(apic)) || kvm_x86_ops->set_hv_timer(apic->vcpu, tscdeadline)) { if (apic->lapic_timer.hv_timer_in_use) - cancel_hv_tscdeadline(apic); + cancel_hv_timer(apic); } else { apic->lapic_timer.hv_timer_in_use = true; hrtimer_cancel(&apic->lapic_timer.timer); /* In case the sw timer triggered in the window */ - if (atomic_read(&apic->lapic_timer.pending)) - cancel_hv_tscdeadline(apic); + if (atomic_read(&apic->lapic_timer.pending) && + !apic_lvtt_period(apic)) + cancel_hv_timer(apic); } trace_kvm_hv_timer_state(apic->vcpu->vcpu_id, apic->lapic_timer.hv_timer_in_use); return apic->lapic_timer.hv_timer_in_use; } +void kvm_lapic_expired_hv_timer(struct kvm_vcpu *vcpu) +{ + struct kvm_lapic *apic = vcpu->arch.apic; + + WARN_ON(!apic->lapic_timer.hv_timer_in_use); + WARN_ON(swait_active(&vcpu->wq)); + cancel_hv_timer(apic); + apic_timer_expired(apic); + + if (apic_lvtt_period(apic) && apic->lapic_timer.period) { + advance_periodic_target_expiration(apic); + if (!start_hv_timer(apic)) + start_sw_period(apic); + } +} +EXPORT_SYMBOL_GPL(kvm_lapic_expired_hv_timer); + void kvm_lapic_switch_to_hv_timer(struct kvm_vcpu *vcpu) { struct kvm_lapic *apic = vcpu->arch.apic; WARN_ON(apic->lapic_timer.hv_timer_in_use); - if (apic_lvtt_tscdeadline(apic)) - start_hv_tscdeadline(apic); + start_hv_timer(apic); } EXPORT_SYMBOL_GPL(kvm_lapic_switch_to_hv_timer); @@ -1413,62 +1496,28 @@ void kvm_lapic_switch_to_sw_timer(struct kvm_vcpu *vcpu) if (!apic->lapic_timer.hv_timer_in_use) return; - cancel_hv_tscdeadline(apic); + cancel_hv_timer(apic); if (atomic_read(&apic->lapic_timer.pending)) return; - start_sw_tscdeadline(apic); + if (apic_lvtt_period(apic) || apic_lvtt_oneshot(apic)) + start_sw_period(apic); + else if (apic_lvtt_tscdeadline(apic)) + start_sw_tscdeadline(apic); } EXPORT_SYMBOL_GPL(kvm_lapic_switch_to_sw_timer); static void start_apic_timer(struct kvm_lapic *apic) { - ktime_t now; - atomic_set(&apic->lapic_timer.pending, 0); if (apic_lvtt_period(apic) || apic_lvtt_oneshot(apic)) { - /* lapic timer in oneshot or periodic mode */ - now = apic->lapic_timer.timer.base->get_time(); - apic->lapic_timer.period = (u64)kvm_lapic_get_reg(apic, APIC_TMICT) - * APIC_BUS_CYCLE_NS * apic->divide_count; - - if (!apic->lapic_timer.period) - return; - /* - * Do not allow the guest to program periodic timers with small - * interval, since the hrtimers are not throttled by the host - * scheduler. - */ - if (apic_lvtt_period(apic)) { - s64 min_period = min_timer_period_us * 1000LL; - - if (apic->lapic_timer.period < min_period) { - pr_info_ratelimited( - "kvm: vcpu %i: requested %lld ns " - "lapic timer period limited to %lld ns\n", - apic->vcpu->vcpu_id, - apic->lapic_timer.period, min_period); - apic->lapic_timer.period = min_period; - } - } - - hrtimer_start(&apic->lapic_timer.timer, - ktime_add_ns(now, apic->lapic_timer.period), - HRTIMER_MODE_ABS_PINNED); - - apic_debug("%s: bus cycle is %" PRId64 "ns, now 0x%016" - PRIx64 ", " - "timer initial count 0x%x, period %lldns, " - "expire @ 0x%016" PRIx64 ".\n", __func__, - APIC_BUS_CYCLE_NS, ktime_to_ns(now), - kvm_lapic_get_reg(apic, APIC_TMICT), - apic->lapic_timer.period, - ktime_to_ns(ktime_add_ns(now, - apic->lapic_timer.period))); + if (set_target_expiration(apic) && + !(kvm_x86_ops->set_hv_timer && start_hv_timer(apic))) + start_sw_period(apic); } else if (apic_lvtt_tscdeadline(apic)) { - if (!(kvm_x86_ops->set_hv_timer && start_hv_tscdeadline(apic))) + if (!(kvm_x86_ops->set_hv_timer && start_hv_timer(apic))) start_sw_tscdeadline(apic); } } @@ -1701,13 +1750,22 @@ void kvm_free_lapic(struct kvm_vcpu *vcpu) * LAPIC interface *---------------------------------------------------------------------- */ +u64 kvm_get_lapic_target_expiration_tsc(struct kvm_vcpu *vcpu) +{ + struct kvm_lapic *apic = vcpu->arch.apic; + + if (!lapic_in_kernel(vcpu)) + return 0; + + return apic->lapic_timer.tscdeadline; +} u64 kvm_get_lapic_tscdeadline_msr(struct kvm_vcpu *vcpu) { struct kvm_lapic *apic = vcpu->arch.apic; - if (!lapic_in_kernel(vcpu) || apic_lvtt_oneshot(apic) || - apic_lvtt_period(apic)) + if (!lapic_in_kernel(vcpu) || + !apic_lvtt_tscdeadline(apic)) return 0; return apic->lapic_timer.tscdeadline; @@ -1748,14 +1806,17 @@ void kvm_lapic_set_base(struct kvm_vcpu *vcpu, u64 value) u64 old_value = vcpu->arch.apic_base; struct kvm_lapic *apic = vcpu->arch.apic; - if (!apic) { + if (!apic) value |= MSR_IA32_APICBASE_BSP; - vcpu->arch.apic_base = value; - return; - } vcpu->arch.apic_base = value; + if ((old_value ^ value) & MSR_IA32_APICBASE_ENABLE) + kvm_update_cpuid(vcpu); + + if (!apic) + return; + /* update jump label if enable bit changes */ if ((old_value ^ value) & MSR_IA32_APICBASE_ENABLE) { if (value & MSR_IA32_APICBASE_ENABLE) { @@ -1909,6 +1970,7 @@ static enum hrtimer_restart apic_timer_fn(struct hrtimer *data) apic_timer_expired(apic); if (lapic_is_periodic(apic)) { + advance_periodic_target_expiration(apic); hrtimer_add_expires_ns(&ktimer->timer, ktimer->period); return HRTIMER_RESTART; } else @@ -1993,6 +2055,10 @@ void kvm_inject_apic_timer_irqs(struct kvm_vcpu *vcpu) kvm_apic_local_deliver(apic, APIC_LVTT); if (apic_lvtt_tscdeadline(apic)) apic->lapic_timer.tscdeadline = 0; + if (apic_lvtt_oneshot(apic)) { + apic->lapic_timer.tscdeadline = 0; + apic->lapic_timer.target_expiration = ktime_set(0, 0); + } atomic_set(&apic->lapic_timer.pending, 0); } } diff --git a/arch/x86/kvm/lapic.h b/arch/x86/kvm/lapic.h index f60d01c..e0c8023 100644 --- a/arch/x86/kvm/lapic.h +++ b/arch/x86/kvm/lapic.h @@ -15,6 +15,7 @@ struct kvm_timer { struct hrtimer timer; s64 period; /* unit: ns */ + ktime_t target_expiration; u32 timer_mode; u32 timer_mode_mask; u64 tscdeadline; @@ -85,6 +86,7 @@ int kvm_apic_get_state(struct kvm_vcpu *vcpu, struct kvm_lapic_state *s); int kvm_apic_set_state(struct kvm_vcpu *vcpu, struct kvm_lapic_state *s); int kvm_lapic_find_highest_irr(struct kvm_vcpu *vcpu); +u64 kvm_get_lapic_target_expiration_tsc(struct kvm_vcpu *vcpu); u64 kvm_get_lapic_tscdeadline_msr(struct kvm_vcpu *vcpu); void kvm_set_lapic_tscdeadline_msr(struct kvm_vcpu *vcpu, u64 data); diff --git a/arch/x86/kvm/mmu.c b/arch/x86/kvm/mmu.c index 87c5880..7012de4 100644 --- a/arch/x86/kvm/mmu.c +++ b/arch/x86/kvm/mmu.c @@ -1660,17 +1660,9 @@ int kvm_age_hva(struct kvm *kvm, unsigned long start, unsigned long end) * This has some overhead, but not as much as the cost of swapping * out actively used pages or breaking up actively used hugepages. */ - if (!shadow_accessed_mask) { - /* - * We are holding the kvm->mmu_lock, and we are blowing up - * shadow PTEs. MMU notifier consumers need to be kept at bay. - * This is correct as long as we don't decouple the mmu_lock - * protected regions (like invalidate_range_start|end does). - */ - kvm->mmu_notifier_seq++; + if (!shadow_accessed_mask) return kvm_handle_hva_range(kvm, start, end, 0, kvm_unmap_rmapp); - } return kvm_handle_hva_range(kvm, start, end, 0, kvm_age_rmapp); } @@ -4509,7 +4501,7 @@ static void make_mmu_pages_available(struct kvm_vcpu *vcpu) kvm_mmu_commit_zap_page(vcpu->kvm, &invalid_list); } -int kvm_mmu_page_fault(struct kvm_vcpu *vcpu, gva_t cr2, u32 error_code, +int kvm_mmu_page_fault(struct kvm_vcpu *vcpu, gva_t cr2, u64 error_code, void *insn, int insn_len) { int r, emulation_type = EMULTYPE_RETRY; @@ -4528,12 +4520,28 @@ int kvm_mmu_page_fault(struct kvm_vcpu *vcpu, gva_t cr2, u32 error_code, return r; } - r = vcpu->arch.mmu.page_fault(vcpu, cr2, error_code, false); + r = vcpu->arch.mmu.page_fault(vcpu, cr2, lower_32_bits(error_code), + false); if (r < 0) return r; if (!r) return 1; + /* + * Before emulating the instruction, check if the error code + * was due to a RO violation while translating the guest page. + * This can occur when using nested virtualization with nested + * paging in both guests. If true, we simply unprotect the page + * and resume the guest. + * + * Note: AMD only (since it supports the PFERR_GUEST_PAGE_MASK used + * in PFERR_NEXT_GUEST_PAGE) + */ + if (error_code == PFERR_NESTED_GUEST_PAGE) { + kvm_mmu_unprotect_page(vcpu->kvm, gpa_to_gfn(cr2)); + return 1; + } + if (mmio_info_in_cache(vcpu, cr2, direct)) emulation_type = 0; emulate: @@ -4967,7 +4975,7 @@ void kvm_mmu_invalidate_mmio_sptes(struct kvm *kvm, struct kvm_memslots *slots) * zap all shadow pages. */ if (unlikely((slots->generation & MMIO_GEN_MASK) == 0)) { - printk_ratelimited(KERN_DEBUG "kvm: zapping shadow pages for mmio generation wraparound\n"); + kvm_debug_ratelimited("kvm: zapping shadow pages for mmio generation wraparound\n"); kvm_mmu_invalidate_zap_all_pages(kvm); } } diff --git a/arch/x86/kvm/svm.c b/arch/x86/kvm/svm.c index 8ca1eca..08a4d3a 100644 --- a/arch/x86/kvm/svm.c +++ b/arch/x86/kvm/svm.c @@ -2074,7 +2074,7 @@ static void svm_set_dr7(struct kvm_vcpu *vcpu, unsigned long value) static int pf_interception(struct vcpu_svm *svm) { u64 fault_address = svm->vmcb->control.exit_info_2; - u32 error_code; + u64 error_code; int r = 1; switch (svm->apf_reason) { @@ -2270,7 +2270,7 @@ static int io_interception(struct vcpu_svm *svm) ++svm->vcpu.stat.io_exits; string = (io_info & SVM_IOIO_STR_MASK) != 0; in = (io_info & SVM_IOIO_TYPE_MASK) != 0; - if (string || in) + if (string) return emulate_instruction(vcpu, 0) == EMULATE_DONE; port = io_info >> 16; @@ -2278,7 +2278,8 @@ static int io_interception(struct vcpu_svm *svm) svm->next_rip = svm->vmcb->control.exit_info_2; skip_emulated_instruction(&svm->vcpu); - return kvm_fast_pio_out(vcpu, size, port); + return in ? kvm_fast_pio_in(vcpu, size, port) + : kvm_fast_pio_out(vcpu, size, port); } static int nmi_interception(struct vcpu_svm *svm) @@ -3150,8 +3151,7 @@ static int skinit_interception(struct vcpu_svm *svm) static int wbinvd_interception(struct vcpu_svm *svm) { - kvm_emulate_wbinvd(&svm->vcpu); - return 1; + return kvm_emulate_wbinvd(&svm->vcpu); } static int xsetbv_interception(struct vcpu_svm *svm) @@ -3238,8 +3238,7 @@ static int task_switch_interception(struct vcpu_svm *svm) static int cpuid_interception(struct vcpu_svm *svm) { svm->next_rip = kvm_rip_read(&svm->vcpu) + 2; - kvm_emulate_cpuid(&svm->vcpu); - return 1; + return kvm_emulate_cpuid(&svm->vcpu); } static int iret_interception(struct vcpu_svm *svm) @@ -3275,9 +3274,7 @@ static int rdpmc_interception(struct vcpu_svm *svm) return emulate_on_interception(svm); err = kvm_rdpmc(&svm->vcpu); - kvm_complete_insn_gp(&svm->vcpu, err); - - return 1; + return kvm_complete_insn_gp(&svm->vcpu, err); } static bool check_selective_cr0_intercepted(struct vcpu_svm *svm, @@ -3374,9 +3371,7 @@ static int cr_interception(struct vcpu_svm *svm) } kvm_register_write(&svm->vcpu, reg, val); } - kvm_complete_insn_gp(&svm->vcpu, err); - - return 1; + return kvm_complete_insn_gp(&svm->vcpu, err); } static int dr_interception(struct vcpu_svm *svm) diff --git a/arch/x86/kvm/vmx.c b/arch/x86/kvm/vmx.c index 3980da5..aae43c6 100644 --- a/arch/x86/kvm/vmx.c +++ b/arch/x86/kvm/vmx.c @@ -132,6 +132,22 @@ module_param_named(preemption_timer, enable_preemption_timer, bool, S_IRUGO); #define VMX_MISC_EMULATED_PREEMPTION_TIMER_RATE 5 +#define VMX_VPID_EXTENT_SUPPORTED_MASK \ + (VMX_VPID_EXTENT_INDIVIDUAL_ADDR_BIT | \ + VMX_VPID_EXTENT_SINGLE_CONTEXT_BIT | \ + VMX_VPID_EXTENT_GLOBAL_CONTEXT_BIT | \ + VMX_VPID_EXTENT_SINGLE_NON_GLOBAL_BIT) + +/* + * Hyper-V requires all of these, so mark them as supported even though + * they are just treated the same as all-context. + */ +#define VMX_VPID_EXTENT_SUPPORTED_MASK \ + (VMX_VPID_EXTENT_INDIVIDUAL_ADDR_BIT | \ + VMX_VPID_EXTENT_SINGLE_CONTEXT_BIT | \ + VMX_VPID_EXTENT_GLOBAL_CONTEXT_BIT | \ + VMX_VPID_EXTENT_SINGLE_NON_GLOBAL_BIT) + /* * These 2 parameters are used to config the controls for Pause-Loop Exiting: * ple_gap: upper bound on the amount of time between two successive @@ -446,23 +462,31 @@ struct nested_vmx { u16 vpid02; u16 last_vpid; + /* + * We only store the "true" versions of the VMX capability MSRs. We + * generate the "non-true" versions by setting the must-be-1 bits + * according to the SDM. + */ u32 nested_vmx_procbased_ctls_low; u32 nested_vmx_procbased_ctls_high; - u32 nested_vmx_true_procbased_ctls_low; u32 nested_vmx_secondary_ctls_low; u32 nested_vmx_secondary_ctls_high; u32 nested_vmx_pinbased_ctls_low; u32 nested_vmx_pinbased_ctls_high; u32 nested_vmx_exit_ctls_low; u32 nested_vmx_exit_ctls_high; - u32 nested_vmx_true_exit_ctls_low; u32 nested_vmx_entry_ctls_low; u32 nested_vmx_entry_ctls_high; - u32 nested_vmx_true_entry_ctls_low; u32 nested_vmx_misc_low; u32 nested_vmx_misc_high; u32 nested_vmx_ept_caps; u32 nested_vmx_vpid_caps; + u64 nested_vmx_basic; + u64 nested_vmx_cr0_fixed0; + u64 nested_vmx_cr0_fixed1; + u64 nested_vmx_cr4_fixed0; + u64 nested_vmx_cr4_fixed1; + u64 nested_vmx_vmcs_enum; }; #define POSTED_INTR_ON 0 @@ -520,6 +544,12 @@ static inline void pi_set_sn(struct pi_desc *pi_desc) (unsigned long *)&pi_desc->control); } +static inline void pi_clear_on(struct pi_desc *pi_desc) +{ + clear_bit(POSTED_INTR_ON, + (unsigned long *)&pi_desc->control); +} + static inline int pi_test_on(struct pi_desc *pi_desc) { return test_bit(POSTED_INTR_ON, @@ -920,16 +950,32 @@ static DEFINE_PER_CPU(struct desc_ptr, host_gdt); static DEFINE_PER_CPU(struct list_head, blocked_vcpu_on_cpu); static DEFINE_PER_CPU(spinlock_t, blocked_vcpu_on_cpu_lock); -static unsigned long *vmx_io_bitmap_a; -static unsigned long *vmx_io_bitmap_b; -static unsigned long *vmx_msr_bitmap_legacy; -static unsigned long *vmx_msr_bitmap_longmode; -static unsigned long *vmx_msr_bitmap_legacy_x2apic; -static unsigned long *vmx_msr_bitmap_longmode_x2apic; -static unsigned long *vmx_msr_bitmap_legacy_x2apic_apicv_inactive; -static unsigned long *vmx_msr_bitmap_longmode_x2apic_apicv_inactive; -static unsigned long *vmx_vmread_bitmap; -static unsigned long *vmx_vmwrite_bitmap; +enum { + VMX_IO_BITMAP_A, + VMX_IO_BITMAP_B, + VMX_MSR_BITMAP_LEGACY, + VMX_MSR_BITMAP_LONGMODE, + VMX_MSR_BITMAP_LEGACY_X2APIC_APICV, + VMX_MSR_BITMAP_LONGMODE_X2APIC_APICV, + VMX_MSR_BITMAP_LEGACY_X2APIC, + VMX_MSR_BITMAP_LONGMODE_X2APIC, + VMX_VMREAD_BITMAP, + VMX_VMWRITE_BITMAP, + VMX_BITMAP_NR +}; + +static unsigned long *vmx_bitmap[VMX_BITMAP_NR]; + +#define vmx_io_bitmap_a (vmx_bitmap[VMX_IO_BITMAP_A]) +#define vmx_io_bitmap_b (vmx_bitmap[VMX_IO_BITMAP_B]) +#define vmx_msr_bitmap_legacy (vmx_bitmap[VMX_MSR_BITMAP_LEGACY]) +#define vmx_msr_bitmap_longmode (vmx_bitmap[VMX_MSR_BITMAP_LONGMODE]) +#define vmx_msr_bitmap_legacy_x2apic_apicv (vmx_bitmap[VMX_MSR_BITMAP_LEGACY_X2APIC_APICV]) +#define vmx_msr_bitmap_longmode_x2apic_apicv (vmx_bitmap[VMX_MSR_BITMAP_LONGMODE_X2APIC_APICV]) +#define vmx_msr_bitmap_legacy_x2apic (vmx_bitmap[VMX_MSR_BITMAP_LEGACY_X2APIC]) +#define vmx_msr_bitmap_longmode_x2apic (vmx_bitmap[VMX_MSR_BITMAP_LONGMODE_X2APIC]) +#define vmx_vmread_bitmap (vmx_bitmap[VMX_VMREAD_BITMAP]) +#define vmx_vmwrite_bitmap (vmx_bitmap[VMX_VMWRITE_BITMAP]) static bool cpu_has_load_ia32_efer; static bool cpu_has_load_perf_global_ctrl; @@ -2523,14 +2569,14 @@ static void vmx_set_msr_bitmap(struct kvm_vcpu *vcpu) SECONDARY_EXEC_VIRTUALIZE_X2APIC_MODE)) { if (enable_apicv && kvm_vcpu_apicv_active(vcpu)) { if (is_long_mode(vcpu)) - msr_bitmap = vmx_msr_bitmap_longmode_x2apic; + msr_bitmap = vmx_msr_bitmap_longmode_x2apic_apicv; else - msr_bitmap = vmx_msr_bitmap_legacy_x2apic; + msr_bitmap = vmx_msr_bitmap_legacy_x2apic_apicv; } else { if (is_long_mode(vcpu)) - msr_bitmap = vmx_msr_bitmap_longmode_x2apic_apicv_inactive; + msr_bitmap = vmx_msr_bitmap_longmode_x2apic; else - msr_bitmap = vmx_msr_bitmap_legacy_x2apic_apicv_inactive; + msr_bitmap = vmx_msr_bitmap_legacy_x2apic; } } else { if (is_long_mode(vcpu)) @@ -2706,9 +2752,7 @@ static void nested_vmx_setup_ctls_msrs(struct vcpu_vmx *vmx) vmx->nested.nested_vmx_exit_ctls_high |= VM_EXIT_CLEAR_BNDCFGS; /* We support free control of debug control saving. */ - vmx->nested.nested_vmx_true_exit_ctls_low = - vmx->nested.nested_vmx_exit_ctls_low & - ~VM_EXIT_SAVE_DEBUG_CONTROLS; + vmx->nested.nested_vmx_exit_ctls_low &= ~VM_EXIT_SAVE_DEBUG_CONTROLS; /* entry controls */ rdmsr(MSR_IA32_VMX_ENTRY_CTLS, @@ -2727,9 +2771,7 @@ static void nested_vmx_setup_ctls_msrs(struct vcpu_vmx *vmx) vmx->nested.nested_vmx_entry_ctls_high |= VM_ENTRY_LOAD_BNDCFGS; /* We support free control of debug control loading. */ - vmx->nested.nested_vmx_true_entry_ctls_low = - vmx->nested.nested_vmx_entry_ctls_low & - ~VM_ENTRY_LOAD_DEBUG_CONTROLS; + vmx->nested.nested_vmx_entry_ctls_low &= ~VM_ENTRY_LOAD_DEBUG_CONTROLS; /* cpu-based controls */ rdmsr(MSR_IA32_VMX_PROCBASED_CTLS, @@ -2762,8 +2804,7 @@ static void nested_vmx_setup_ctls_msrs(struct vcpu_vmx *vmx) CPU_BASED_USE_MSR_BITMAPS; /* We support free control of CR3 access interception. */ - vmx->nested.nested_vmx_true_procbased_ctls_low = - vmx->nested.nested_vmx_procbased_ctls_low & + vmx->nested.nested_vmx_procbased_ctls_low &= ~(CPU_BASED_CR3_LOAD_EXITING | CPU_BASED_CR3_STORE_EXITING); /* secondary cpu-based controls */ @@ -2774,6 +2815,7 @@ static void nested_vmx_setup_ctls_msrs(struct vcpu_vmx *vmx) vmx->nested.nested_vmx_secondary_ctls_high &= SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES | SECONDARY_EXEC_RDTSCP | + SECONDARY_EXEC_DESC | SECONDARY_EXEC_VIRTUALIZE_X2APIC_MODE | SECONDARY_EXEC_ENABLE_VPID | SECONDARY_EXEC_APIC_REGISTER_VIRT | @@ -2805,8 +2847,7 @@ static void nested_vmx_setup_ctls_msrs(struct vcpu_vmx *vmx) */ if (enable_vpid) vmx->nested.nested_vmx_vpid_caps = VMX_VPID_INVVPID_BIT | - VMX_VPID_EXTENT_SINGLE_CONTEXT_BIT | - VMX_VPID_EXTENT_GLOBAL_CONTEXT_BIT; + VMX_VPID_EXTENT_SUPPORTED_MASK; else vmx->nested.nested_vmx_vpid_caps = 0; @@ -2823,14 +2864,52 @@ static void nested_vmx_setup_ctls_msrs(struct vcpu_vmx *vmx) VMX_MISC_EMULATED_PREEMPTION_TIMER_RATE | VMX_MISC_ACTIVITY_HLT; vmx->nested.nested_vmx_misc_high = 0; + + /* + * This MSR reports some information about VMX support. We + * should return information about the VMX we emulate for the + * guest, and the VMCS structure we give it - not about the + * VMX support of the underlying hardware. + */ + vmx->nested.nested_vmx_basic = + VMCS12_REVISION | + VMX_BASIC_TRUE_CTLS | + ((u64)VMCS12_SIZE << VMX_BASIC_VMCS_SIZE_SHIFT) | + (VMX_BASIC_MEM_TYPE_WB << VMX_BASIC_MEM_TYPE_SHIFT); + + if (cpu_has_vmx_basic_inout()) + vmx->nested.nested_vmx_basic |= VMX_BASIC_INOUT; + + /* + * These MSRs specify bits which the guest must keep fixed on + * while L1 is in VMXON mode (in L1's root mode, or running an L2). + * We picked the standard core2 setting. + */ +#define VMXON_CR0_ALWAYSON (X86_CR0_PE | X86_CR0_PG | X86_CR0_NE) +#define VMXON_CR4_ALWAYSON X86_CR4_VMXE + vmx->nested.nested_vmx_cr0_fixed0 = VMXON_CR0_ALWAYSON; + vmx->nested.nested_vmx_cr4_fixed0 = VMXON_CR4_ALWAYSON; + + /* These MSRs specify bits which the guest must keep fixed off. */ + rdmsrl(MSR_IA32_VMX_CR0_FIXED1, vmx->nested.nested_vmx_cr0_fixed1); + rdmsrl(MSR_IA32_VMX_CR4_FIXED1, vmx->nested.nested_vmx_cr4_fixed1); + + /* highest index: VMX_PREEMPTION_TIMER_VALUE */ + vmx->nested.nested_vmx_vmcs_enum = 0x2e; +} + +/* + * if fixed0[i] == 1: val[i] must be 1 + * if fixed1[i] == 0: val[i] must be 0 + */ +static inline bool fixed_bits_valid(u64 val, u64 fixed0, u64 fixed1) +{ + return ((val & fixed1) | fixed0) == val; } static inline bool vmx_control_verify(u32 control, u32 low, u32 high) { - /* - * Bits 0 in high must be 0, and bits 1 in low must be 1. - */ - return ((control & high) | low) == control; + return fixed_bits_valid(control, low, high); } static inline u64 vmx_control_msr(u32 low, u32 high) @@ -2838,87 +2917,285 @@ static inline u64 vmx_control_msr(u32 low, u32 high) return low | ((u64)high << 32); } -/* Returns 0 on success, non-0 otherwise. */ -static int vmx_get_vmx_msr(struct kvm_vcpu *vcpu, u32 msr_index, u64 *pdata) +static bool is_bitwise_subset(u64 superset, u64 subset, u64 mask) +{ + superset &= mask; + subset &= mask; + + return (superset | subset) == superset; +} + +static int vmx_restore_vmx_basic(struct vcpu_vmx *vmx, u64 data) +{ + const u64 feature_and_reserved = + /* feature (except bit 48; see below) */ + BIT_ULL(49) | BIT_ULL(54) | BIT_ULL(55) | + /* reserved */ + BIT_ULL(31) | GENMASK_ULL(47, 45) | GENMASK_ULL(63, 56); + u64 vmx_basic = vmx->nested.nested_vmx_basic; + + if (!is_bitwise_subset(vmx_basic, data, feature_and_reserved)) + return -EINVAL; + + /* + * KVM does not emulate a version of VMX that constrains physical + * addresses of VMX structures (e.g. VMCS) to 32-bits. + */ + if (data & BIT_ULL(48)) + return -EINVAL; + + if (vmx_basic_vmcs_revision_id(vmx_basic) != + vmx_basic_vmcs_revision_id(data)) + return -EINVAL; + + if (vmx_basic_vmcs_size(vmx_basic) > vmx_basic_vmcs_size(data)) + return -EINVAL; + + vmx->nested.nested_vmx_basic = data; + return 0; +} + +static int +vmx_restore_control_msr(struct vcpu_vmx *vmx, u32 msr_index, u64 data) +{ + u64 supported; + u32 *lowp, *highp; + + switch (msr_index) { + case MSR_IA32_VMX_TRUE_PINBASED_CTLS: + lowp = &vmx->nested.nested_vmx_pinbased_ctls_low; + highp = &vmx->nested.nested_vmx_pinbased_ctls_high; + break; + case MSR_IA32_VMX_TRUE_PROCBASED_CTLS: + lowp = &vmx->nested.nested_vmx_procbased_ctls_low; + highp = &vmx->nested.nested_vmx_procbased_ctls_high; + break; + case MSR_IA32_VMX_TRUE_EXIT_CTLS: + lowp = &vmx->nested.nested_vmx_exit_ctls_low; + highp = &vmx->nested.nested_vmx_exit_ctls_high; + break; + case MSR_IA32_VMX_TRUE_ENTRY_CTLS: + lowp = &vmx->nested.nested_vmx_entry_ctls_low; + highp = &vmx->nested.nested_vmx_entry_ctls_high; + break; + case MSR_IA32_VMX_PROCBASED_CTLS2: + lowp = &vmx->nested.nested_vmx_secondary_ctls_low; + highp = &vmx->nested.nested_vmx_secondary_ctls_high; + break; + default: + BUG(); + } + + supported = vmx_control_msr(*lowp, *highp); + + /* Check must-be-1 bits are still 1. */ + if (!is_bitwise_subset(data, supported, GENMASK_ULL(31, 0))) + return -EINVAL; + + /* Check must-be-0 bits are still 0. */ + if (!is_bitwise_subset(supported, data, GENMASK_ULL(63, 32))) + return -EINVAL; + + *lowp = data; + *highp = data >> 32; + return 0; +} + +static int vmx_restore_vmx_misc(struct vcpu_vmx *vmx, u64 data) +{ + const u64 feature_and_reserved_bits = + /* feature */ + BIT_ULL(5) | GENMASK_ULL(8, 6) | BIT_ULL(14) | BIT_ULL(15) | + BIT_ULL(28) | BIT_ULL(29) | BIT_ULL(30) | + /* reserved */ + GENMASK_ULL(13, 9) | BIT_ULL(31); + u64 vmx_misc; + + vmx_misc = vmx_control_msr(vmx->nested.nested_vmx_misc_low, + vmx->nested.nested_vmx_misc_high); + + if (!is_bitwise_subset(vmx_misc, data, feature_and_reserved_bits)) + return -EINVAL; + + if ((vmx->nested.nested_vmx_pinbased_ctls_high & + PIN_BASED_VMX_PREEMPTION_TIMER) && + vmx_misc_preemption_timer_rate(data) != + vmx_misc_preemption_timer_rate(vmx_misc)) + return -EINVAL; + + if (vmx_misc_cr3_count(data) > vmx_misc_cr3_count(vmx_misc)) + return -EINVAL; + + if (vmx_misc_max_msr(data) > vmx_misc_max_msr(vmx_misc)) + return -EINVAL; + + if (vmx_misc_mseg_revid(data) != vmx_misc_mseg_revid(vmx_misc)) + return -EINVAL; + + vmx->nested.nested_vmx_misc_low = data; + vmx->nested.nested_vmx_misc_high = data >> 32; + return 0; +} + +static int vmx_restore_vmx_ept_vpid_cap(struct vcpu_vmx *vmx, u64 data) +{ + u64 vmx_ept_vpid_cap; + + vmx_ept_vpid_cap = vmx_control_msr(vmx->nested.nested_vmx_ept_caps, + vmx->nested.nested_vmx_vpid_caps); + + /* Every bit is either reserved or a feature bit. */ + if (!is_bitwise_subset(vmx_ept_vpid_cap, data, -1ULL)) + return -EINVAL; + + vmx->nested.nested_vmx_ept_caps = data; + vmx->nested.nested_vmx_vpid_caps = data >> 32; + return 0; +} + +static int vmx_restore_fixed0_msr(struct vcpu_vmx *vmx, u32 msr_index, u64 data) +{ + u64 *msr; + + switch (msr_index) { + case MSR_IA32_VMX_CR0_FIXED0: + msr = &vmx->nested.nested_vmx_cr0_fixed0; + break; + case MSR_IA32_VMX_CR4_FIXED0: + msr = &vmx->nested.nested_vmx_cr4_fixed0; + break; + default: + BUG(); + } + + /* + * 1 bits (which indicates bits which "must-be-1" during VMX operation) + * must be 1 in the restored value. + */ + if (!is_bitwise_subset(data, *msr, -1ULL)) + return -EINVAL; + + *msr = data; + return 0; +} + +/* + * Called when userspace is restoring VMX MSRs. + * + * Returns 0 on success, non-0 otherwise. + */ +static int vmx_set_vmx_msr(struct kvm_vcpu *vcpu, u32 msr_index, u64 data) { struct vcpu_vmx *vmx = to_vmx(vcpu); switch (msr_index) { case MSR_IA32_VMX_BASIC: + return vmx_restore_vmx_basic(vmx, data); + case MSR_IA32_VMX_PINBASED_CTLS: + case MSR_IA32_VMX_PROCBASED_CTLS: + case MSR_IA32_VMX_EXIT_CTLS: + case MSR_IA32_VMX_ENTRY_CTLS: + /* + * The "non-true" VMX capability MSRs are generated from the + * "true" MSRs, so we do not support restoring them directly. + * + * If userspace wants to emulate VMX_BASIC[55]=0, userspace + * should restore the "true" MSRs with the must-be-1 bits + * set according to the SDM Vol 3. A.2 "RESERVED CONTROLS AND + * DEFAULT SETTINGS". + */ + return -EINVAL; + case MSR_IA32_VMX_TRUE_PINBASED_CTLS: + case MSR_IA32_VMX_TRUE_PROCBASED_CTLS: + case MSR_IA32_VMX_TRUE_EXIT_CTLS: + case MSR_IA32_VMX_TRUE_ENTRY_CTLS: + case MSR_IA32_VMX_PROCBASED_CTLS2: + return vmx_restore_control_msr(vmx, msr_index, data); + case MSR_IA32_VMX_MISC: + return vmx_restore_vmx_misc(vmx, data); + case MSR_IA32_VMX_CR0_FIXED0: + case MSR_IA32_VMX_CR4_FIXED0: + return vmx_restore_fixed0_msr(vmx, msr_index, data); + case MSR_IA32_VMX_CR0_FIXED1: + case MSR_IA32_VMX_CR4_FIXED1: + /* + * These MSRs are generated based on the vCPU's CPUID, so we + * do not support restoring them directly. + */ + return -EINVAL; + case MSR_IA32_VMX_EPT_VPID_CAP: + return vmx_restore_vmx_ept_vpid_cap(vmx, data); + case MSR_IA32_VMX_VMCS_ENUM: + vmx->nested.nested_vmx_vmcs_enum = data; + return 0; + default: /* - * This MSR reports some information about VMX support. We - * should return information about the VMX we emulate for the - * guest, and the VMCS structure we give it - not about the - * VMX support of the underlying hardware. + * The rest of the VMX capability MSRs do not support restore. */ - *pdata = VMCS12_REVISION | VMX_BASIC_TRUE_CTLS | - ((u64)VMCS12_SIZE << VMX_BASIC_VMCS_SIZE_SHIFT) | - (VMX_BASIC_MEM_TYPE_WB << VMX_BASIC_MEM_TYPE_SHIFT); - if (cpu_has_vmx_basic_inout()) - *pdata |= VMX_BASIC_INOUT; + return -EINVAL; + } +} + +/* Returns 0 on success, non-0 otherwise. */ +static int vmx_get_vmx_msr(struct kvm_vcpu *vcpu, u32 msr_index, u64 *pdata) +{ + struct vcpu_vmx *vmx = to_vmx(vcpu); + + switch (msr_index) { + case MSR_IA32_VMX_BASIC: + *pdata = vmx->nested.nested_vmx_basic; break; case MSR_IA32_VMX_TRUE_PINBASED_CTLS: case MSR_IA32_VMX_PINBASED_CTLS: *pdata = vmx_control_msr( vmx->nested.nested_vmx_pinbased_ctls_low, vmx->nested.nested_vmx_pinbased_ctls_high); + if (msr_index == MSR_IA32_VMX_PINBASED_CTLS) + *pdata |= PIN_BASED_ALWAYSON_WITHOUT_TRUE_MSR; break; case MSR_IA32_VMX_TRUE_PROCBASED_CTLS: - *pdata = vmx_control_msr( - vmx->nested.nested_vmx_true_procbased_ctls_low, - vmx->nested.nested_vmx_procbased_ctls_high); - break; case MSR_IA32_VMX_PROCBASED_CTLS: *pdata = vmx_control_msr( vmx->nested.nested_vmx_procbased_ctls_low, vmx->nested.nested_vmx_procbased_ctls_high); + if (msr_index == MSR_IA32_VMX_PROCBASED_CTLS) + *pdata |= CPU_BASED_ALWAYSON_WITHOUT_TRUE_MSR; break; case MSR_IA32_VMX_TRUE_EXIT_CTLS: - *pdata = vmx_control_msr( - vmx->nested.nested_vmx_true_exit_ctls_low, - vmx->nested.nested_vmx_exit_ctls_high); - break; case MSR_IA32_VMX_EXIT_CTLS: *pdata = vmx_control_msr( vmx->nested.nested_vmx_exit_ctls_low, vmx->nested.nested_vmx_exit_ctls_high); + if (msr_index == MSR_IA32_VMX_EXIT_CTLS) + *pdata |= VM_EXIT_ALWAYSON_WITHOUT_TRUE_MSR; break; case MSR_IA32_VMX_TRUE_ENTRY_CTLS: - *pdata = vmx_control_msr( - vmx->nested.nested_vmx_true_entry_ctls_low, - vmx->nested.nested_vmx_entry_ctls_high); - break; case MSR_IA32_VMX_ENTRY_CTLS: *pdata = vmx_control_msr( vmx->nested.nested_vmx_entry_ctls_low, vmx->nested.nested_vmx_entry_ctls_high); + if (msr_index == MSR_IA32_VMX_ENTRY_CTLS) + *pdata |= VM_ENTRY_ALWAYSON_WITHOUT_TRUE_MSR; break; case MSR_IA32_VMX_MISC: *pdata = vmx_control_msr( vmx->nested.nested_vmx_misc_low, vmx->nested.nested_vmx_misc_high); break; - /* - * These MSRs specify bits which the guest must keep fixed (on or off) - * while L1 is in VMXON mode (in L1's root mode, or running an L2). - * We picked the standard core2 setting. - */ -#define VMXON_CR0_ALWAYSON (X86_CR0_PE | X86_CR0_PG | X86_CR0_NE) -#define VMXON_CR4_ALWAYSON X86_CR4_VMXE case MSR_IA32_VMX_CR0_FIXED0: - *pdata = VMXON_CR0_ALWAYSON; + *pdata = vmx->nested.nested_vmx_cr0_fixed0; break; case MSR_IA32_VMX_CR0_FIXED1: - *pdata = -1ULL; + *pdata = vmx->nested.nested_vmx_cr0_fixed1; break; case MSR_IA32_VMX_CR4_FIXED0: - *pdata = VMXON_CR4_ALWAYSON; + *pdata = vmx->nested.nested_vmx_cr4_fixed0; break; case MSR_IA32_VMX_CR4_FIXED1: - *pdata = -1ULL; + *pdata = vmx->nested.nested_vmx_cr4_fixed1; break; case MSR_IA32_VMX_VMCS_ENUM: - *pdata = 0x2e; /* highest index: VMX_PREEMPTION_TIMER_VALUE */ + *pdata = vmx->nested.nested_vmx_vmcs_enum; break; case MSR_IA32_VMX_PROCBASED_CTLS2: *pdata = vmx_control_msr( @@ -3101,7 +3378,11 @@ static int vmx_set_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info) vmx_leave_nested(vcpu); break; case MSR_IA32_VMX_BASIC ... MSR_IA32_VMX_VMFUNC: - return 1; /* they are read-only */ + if (!msr_info->host_initiated) + return 1; /* they are read-only */ + if (!nested_vmx_allowed(vcpu)) + return 1; + return vmx_set_vmx_msr(vcpu, msr_index, data); case MSR_IA32_XSS: if (!vmx_xsaves_supported()) return 1; @@ -3863,6 +4144,40 @@ static void ept_save_pdptrs(struct kvm_vcpu *vcpu) (unsigned long *)&vcpu->arch.regs_dirty); } +static bool nested_guest_cr0_valid(struct kvm_vcpu *vcpu, unsigned long val) +{ + u64 fixed0 = to_vmx(vcpu)->nested.nested_vmx_cr0_fixed0; + u64 fixed1 = to_vmx(vcpu)->nested.nested_vmx_cr0_fixed1; + struct vmcs12 *vmcs12 = get_vmcs12(vcpu); + + if (to_vmx(vcpu)->nested.nested_vmx_secondary_ctls_high & + SECONDARY_EXEC_UNRESTRICTED_GUEST && + nested_cpu_has2(vmcs12, SECONDARY_EXEC_UNRESTRICTED_GUEST)) + fixed0 &= ~(X86_CR0_PE | X86_CR0_PG); + + return fixed_bits_valid(val, fixed0, fixed1); +} + +static bool nested_host_cr0_valid(struct kvm_vcpu *vcpu, unsigned long val) +{ + u64 fixed0 = to_vmx(vcpu)->nested.nested_vmx_cr0_fixed0; + u64 fixed1 = to_vmx(vcpu)->nested.nested_vmx_cr0_fixed1; + + return fixed_bits_valid(val, fixed0, fixed1); +} + +static bool nested_cr4_valid(struct kvm_vcpu *vcpu, unsigned long val) +{ + u64 fixed0 = to_vmx(vcpu)->nested.nested_vmx_cr4_fixed0; + u64 fixed1 = to_vmx(vcpu)->nested.nested_vmx_cr4_fixed1; + + return fixed_bits_valid(val, fixed0, fixed1); +} + +/* No difference in the restrictions on guest and host CR4 in VMX operation. */ +#define nested_guest_cr4_valid nested_cr4_valid +#define nested_host_cr4_valid nested_cr4_valid + static int vmx_set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4); static void ept_update_paging_mode_cr0(unsigned long *hw_cr0, @@ -3991,8 +4306,8 @@ static int vmx_set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4) if (!nested_vmx_allowed(vcpu)) return 1; } - if (to_vmx(vcpu)->nested.vmxon && - ((cr4 & VMXON_CR4_ALWAYSON) != VMXON_CR4_ALWAYSON)) + + if (to_vmx(vcpu)->nested.vmxon && !nested_cr4_valid(vcpu, cr4)) return 1; vcpu->arch.cr4 = cr4; @@ -4569,41 +4884,6 @@ static void __vmx_disable_intercept_for_msr(unsigned long *msr_bitmap, } } -static void __vmx_enable_intercept_for_msr(unsigned long *msr_bitmap, - u32 msr, int type) -{ - int f = sizeof(unsigned long); - - if (!cpu_has_vmx_msr_bitmap()) - return; - - /* - * See Intel PRM Vol. 3, 20.6.9 (MSR-Bitmap Address). Early manuals - * have the write-low and read-high bitmap offsets the wrong way round. - * We can control MSRs 0x00000000-0x00001fff and 0xc0000000-0xc0001fff. - */ - if (msr <= 0x1fff) { - if (type & MSR_TYPE_R) - /* read-low */ - __set_bit(msr, msr_bitmap + 0x000 / f); - - if (type & MSR_TYPE_W) - /* write-low */ - __set_bit(msr, msr_bitmap + 0x800 / f); - - } else if ((msr >= 0xc0000000) && (msr <= 0xc0001fff)) { - msr &= 0x1fff; - if (type & MSR_TYPE_R) - /* read-high */ - __set_bit(msr, msr_bitmap + 0x400 / f); - - if (type & MSR_TYPE_W) - /* write-high */ - __set_bit(msr, msr_bitmap + 0xc00 / f); - - } -} - /* * If a msr is allowed by L0, we should check whether it is allowed by L1. * The corresponding bit will be cleared unless both of L0 and L1 allow it. @@ -4659,48 +4939,18 @@ static void vmx_disable_intercept_for_msr(u32 msr, bool longmode_only) msr, MSR_TYPE_R | MSR_TYPE_W); } -static void vmx_enable_intercept_msr_read_x2apic(u32 msr, bool apicv_active) -{ - if (apicv_active) { - __vmx_enable_intercept_for_msr(vmx_msr_bitmap_legacy_x2apic, - msr, MSR_TYPE_R); - __vmx_enable_intercept_for_msr(vmx_msr_bitmap_longmode_x2apic, - msr, MSR_TYPE_R); - } else { - __vmx_enable_intercept_for_msr(vmx_msr_bitmap_legacy_x2apic_apicv_inactive, - msr, MSR_TYPE_R); - __vmx_enable_intercept_for_msr(vmx_msr_bitmap_longmode_x2apic_apicv_inactive, - msr, MSR_TYPE_R); - } -} - -static void vmx_disable_intercept_msr_read_x2apic(u32 msr, bool apicv_active) +static void vmx_disable_intercept_msr_x2apic(u32 msr, int type, bool apicv_active) { if (apicv_active) { - __vmx_disable_intercept_for_msr(vmx_msr_bitmap_legacy_x2apic, - msr, MSR_TYPE_R); - __vmx_disable_intercept_for_msr(vmx_msr_bitmap_longmode_x2apic, - msr, MSR_TYPE_R); + __vmx_disable_intercept_for_msr(vmx_msr_bitmap_legacy_x2apic_apicv, + msr, type); + __vmx_disable_intercept_for_msr(vmx_msr_bitmap_longmode_x2apic_apicv, + msr, type); } else { - __vmx_disable_intercept_for_msr(vmx_msr_bitmap_legacy_x2apic_apicv_inactive, - msr, MSR_TYPE_R); - __vmx_disable_intercept_for_msr(vmx_msr_bitmap_longmode_x2apic_apicv_inactive, - msr, MSR_TYPE_R); - } -} - -static void vmx_disable_intercept_msr_write_x2apic(u32 msr, bool apicv_active) -{ - if (apicv_active) { __vmx_disable_intercept_for_msr(vmx_msr_bitmap_legacy_x2apic, - msr, MSR_TYPE_W); + msr, type); __vmx_disable_intercept_for_msr(vmx_msr_bitmap_longmode_x2apic, - msr, MSR_TYPE_W); - } else { - __vmx_disable_intercept_for_msr(vmx_msr_bitmap_legacy_x2apic_apicv_inactive, - msr, MSR_TYPE_W); - __vmx_disable_intercept_for_msr(vmx_msr_bitmap_longmode_x2apic_apicv_inactive, - msr, MSR_TYPE_W); + msr, type); } } @@ -4822,9 +5072,15 @@ static void vmx_sync_pir_to_irr(struct kvm_vcpu *vcpu) { struct vcpu_vmx *vmx = to_vmx(vcpu); - if (!pi_test_and_clear_on(&vmx->pi_desc)) + if (!pi_test_on(&vmx->pi_desc)) return; + pi_clear_on(&vmx->pi_desc); + /* + * IOMMU can write to PIR.ON, so the barrier matters even on UP. + * But on x86 this is just a compiler barrier anyway. + */ + smp_mb__after_atomic(); kvm_apic_update_irr(vcpu, vmx->pi_desc.pir); } @@ -5583,7 +5839,7 @@ static int handle_triple_fault(struct kvm_vcpu *vcpu) static int handle_io(struct kvm_vcpu *vcpu) { unsigned long exit_qualification; - int size, in, string; + int size, in, string, ret; unsigned port; exit_qualification = vmcs_readl(EXIT_QUALIFICATION); @@ -5597,9 +5853,14 @@ static int handle_io(struct kvm_vcpu *vcpu) port = exit_qualification >> 16; size = (exit_qualification & 7) + 1; - skip_emulated_instruction(vcpu); - return kvm_fast_pio_out(vcpu, size, port); + ret = kvm_skip_emulated_instruction(vcpu); + + /* + * TODO: we might be squashing a KVM_GUESTDBG_SINGLESTEP-triggered + * KVM_EXIT_DEBUG here. + */ + return kvm_fast_pio_out(vcpu, size, port) && ret; } static void @@ -5613,18 +5874,6 @@ vmx_patch_hypercall(struct kvm_vcpu *vcpu, unsigned char *hypercall) hypercall[2] = 0xc1; } -static bool nested_cr0_valid(struct kvm_vcpu *vcpu, unsigned long val) -{ - unsigned long always_on = VMXON_CR0_ALWAYSON; - struct vmcs12 *vmcs12 = get_vmcs12(vcpu); - - if (to_vmx(vcpu)->nested.nested_vmx_secondary_ctls_high & - SECONDARY_EXEC_UNRESTRICTED_GUEST && - nested_cpu_has2(vmcs12, SECONDARY_EXEC_UNRESTRICTED_GUEST)) - always_on &= ~(X86_CR0_PE | X86_CR0_PG); - return (val & always_on) == always_on; -} - /* called to set cr0 as appropriate for a mov-to-cr0 exit. */ static int handle_set_cr0(struct kvm_vcpu *vcpu, unsigned long val) { @@ -5643,7 +5892,7 @@ static int handle_set_cr0(struct kvm_vcpu *vcpu, unsigned long val) val = (val & ~vmcs12->cr0_guest_host_mask) | (vmcs12->guest_cr0 & vmcs12->cr0_guest_host_mask); - if (!nested_cr0_valid(vcpu, val)) + if (!nested_guest_cr0_valid(vcpu, val)) return 1; if (kvm_set_cr0(vcpu, val)) @@ -5652,8 +5901,9 @@ static int handle_set_cr0(struct kvm_vcpu *vcpu, unsigned long val) return 0; } else { if (to_vmx(vcpu)->nested.vmxon && - ((val & VMXON_CR0_ALWAYSON) != VMXON_CR0_ALWAYSON)) + !nested_host_cr0_valid(vcpu, val)) return 1; + return kvm_set_cr0(vcpu, val); } } @@ -5697,6 +5947,7 @@ static int handle_cr(struct kvm_vcpu *vcpu) int cr; int reg; int err; + int ret; exit_qualification = vmcs_readl(EXIT_QUALIFICATION); cr = exit_qualification & 15; @@ -5708,25 +5959,27 @@ static int handle_cr(struct kvm_vcpu *vcpu) switch (cr) { case 0: err = handle_set_cr0(vcpu, val); - kvm_complete_insn_gp(vcpu, err); - return 1; + return kvm_complete_insn_gp(vcpu, err); case 3: err = kvm_set_cr3(vcpu, val); - kvm_complete_insn_gp(vcpu, err); - return 1; + return kvm_complete_insn_gp(vcpu, err); case 4: err = handle_set_cr4(vcpu, val); - kvm_complete_insn_gp(vcpu, err); - return 1; + return kvm_complete_insn_gp(vcpu, err); case 8: { u8 cr8_prev = kvm_get_cr8(vcpu); u8 cr8 = (u8)val; err = kvm_set_cr8(vcpu, cr8); - kvm_complete_insn_gp(vcpu, err); + ret = kvm_complete_insn_gp(vcpu, err); if (lapic_in_kernel(vcpu)) - return 1; + return ret; if (cr8_prev <= cr8) - return 1; + return ret; + /* + * TODO: we might be squashing a + * KVM_GUESTDBG_SINGLESTEP-triggered + * KVM_EXIT_DEBUG here. + */ vcpu->run->exit_reason = KVM_EXIT_SET_TPR; return 0; } @@ -5735,23 +5988,20 @@ static int handle_cr(struct kvm_vcpu *vcpu) case 2: /* clts */ handle_clts(vcpu); trace_kvm_cr_write(0, kvm_read_cr0(vcpu)); - skip_emulated_instruction(vcpu); vmx_fpu_activate(vcpu); - return 1; + return kvm_skip_emulated_instruction(vcpu); case 1: /*mov from cr*/ switch (cr) { case 3: val = kvm_read_cr3(vcpu); kvm_register_write(vcpu, reg, val); trace_kvm_cr_read(cr, val); - skip_emulated_instruction(vcpu); - return 1; + return kvm_skip_emulated_instruction(vcpu); case 8: val = kvm_get_cr8(vcpu); kvm_register_write(vcpu, reg, val); trace_kvm_cr_read(cr, val); - skip_emulated_instruction(vcpu); - return 1; + return kvm_skip_emulated_instruction(vcpu); } break; case 3: /* lmsw */ @@ -5759,8 +6009,7 @@ static int handle_cr(struct kvm_vcpu *vcpu) trace_kvm_cr_write(0, (kvm_read_cr0(vcpu) & ~0xful) | val); kvm_lmsw(vcpu, val); - skip_emulated_instruction(vcpu); - return 1; + return kvm_skip_emulated_instruction(vcpu); default: break; } @@ -5831,8 +6080,7 @@ static int handle_dr(struct kvm_vcpu *vcpu) if (kvm_set_dr(vcpu, dr, kvm_register_readl(vcpu, reg))) return 1; - skip_emulated_instruction(vcpu); - return 1; + return kvm_skip_emulated_instruction(vcpu); } static u64 vmx_get_dr6(struct kvm_vcpu *vcpu) @@ -5864,8 +6112,7 @@ static void vmx_set_dr7(struct kvm_vcpu *vcpu, unsigned long val) static int handle_cpuid(struct kvm_vcpu *vcpu) { - kvm_emulate_cpuid(vcpu); - return 1; + return kvm_emulate_cpuid(vcpu); } static int handle_rdmsr(struct kvm_vcpu *vcpu) @@ -5886,8 +6133,7 @@ static int handle_rdmsr(struct kvm_vcpu *vcpu) /* FIXME: handling of bits 32:63 of rax, rdx */ vcpu->arch.regs[VCPU_REGS_RAX] = msr_info.data & -1u; vcpu->arch.regs[VCPU_REGS_RDX] = (msr_info.data >> 32) & -1u; - skip_emulated_instruction(vcpu); - return 1; + return kvm_skip_emulated_instruction(vcpu); } static int handle_wrmsr(struct kvm_vcpu *vcpu) @@ -5907,8 +6153,7 @@ static int handle_wrmsr(struct kvm_vcpu *vcpu) } trace_kvm_msr_write(ecx, data); - skip_emulated_instruction(vcpu); - return 1; + return kvm_skip_emulated_instruction(vcpu); } static int handle_tpr_below_threshold(struct kvm_vcpu *vcpu) @@ -5952,8 +6197,7 @@ static int handle_invlpg(struct kvm_vcpu *vcpu) unsigned long exit_qualification = vmcs_readl(EXIT_QUALIFICATION); kvm_mmu_invlpg(vcpu, exit_qualification); - skip_emulated_instruction(vcpu); - return 1; + return kvm_skip_emulated_instruction(vcpu); } static int handle_rdpmc(struct kvm_vcpu *vcpu) @@ -5961,15 +6205,12 @@ static int handle_rdpmc(struct kvm_vcpu *vcpu) int err; err = kvm_rdpmc(vcpu); - kvm_complete_insn_gp(vcpu, err); - - return 1; + return kvm_complete_insn_gp(vcpu, err); } static int handle_wbinvd(struct kvm_vcpu *vcpu) { - kvm_emulate_wbinvd(vcpu); - return 1; + return kvm_emulate_wbinvd(vcpu); } static int handle_xsetbv(struct kvm_vcpu *vcpu) @@ -5978,20 +6219,20 @@ static int handle_xsetbv(struct kvm_vcpu *vcpu) u32 index = kvm_register_read(vcpu, VCPU_REGS_RCX); if (kvm_set_xcr(vcpu, index, new_bv) == 0) - skip_emulated_instruction(vcpu); + return kvm_skip_emulated_instruction(vcpu); return 1; } static int handle_xsaves(struct kvm_vcpu *vcpu) { - skip_emulated_instruction(vcpu); + kvm_skip_emulated_instruction(vcpu); WARN(1, "this should never happen\n"); return 1; } static int handle_xrstors(struct kvm_vcpu *vcpu) { - skip_emulated_instruction(vcpu); + kvm_skip_emulated_instruction(vcpu); WARN(1, "this should never happen\n"); return 1; } @@ -6012,8 +6253,7 @@ static int handle_apic_access(struct kvm_vcpu *vcpu) if ((access_type == TYPE_LINEAR_APIC_INST_WRITE) && (offset == APIC_EOI)) { kvm_lapic_set_eoi(vcpu); - skip_emulated_instruction(vcpu); - return 1; + return kvm_skip_emulated_instruction(vcpu); } } return emulate_instruction(vcpu, 0) == EMULATE_DONE; @@ -6161,9 +6401,8 @@ static int handle_ept_misconfig(struct kvm_vcpu *vcpu) gpa = vmcs_read64(GUEST_PHYSICAL_ADDRESS); if (!kvm_io_bus_write(vcpu, KVM_FAST_MMIO_BUS, gpa, 0, NULL)) { - skip_emulated_instruction(vcpu); trace_kvm_fast_mmio(gpa); - return 1; + return kvm_skip_emulated_instruction(vcpu); } ret = handle_mmio_page_fault(vcpu, gpa, true); @@ -6348,50 +6587,13 @@ static __init int hardware_setup(void) for (i = 0; i < ARRAY_SIZE(vmx_msr_index); ++i) kvm_define_shared_msr(i, vmx_msr_index[i]); - vmx_io_bitmap_a = (unsigned long *)__get_free_page(GFP_KERNEL); - if (!vmx_io_bitmap_a) - return r; + for (i = 0; i < VMX_BITMAP_NR; i++) { + vmx_bitmap[i] = (unsigned long *)__get_free_page(GFP_KERNEL); + if (!vmx_bitmap[i]) + goto out; + } vmx_io_bitmap_b = (unsigned long *)__get_free_page(GFP_KERNEL); - if (!vmx_io_bitmap_b) - goto out; - - vmx_msr_bitmap_legacy = (unsigned long *)__get_free_page(GFP_KERNEL); - if (!vmx_msr_bitmap_legacy) - goto out1; - - vmx_msr_bitmap_legacy_x2apic = - (unsigned long *)__get_free_page(GFP_KERNEL); - if (!vmx_msr_bitmap_legacy_x2apic) - goto out2; - - vmx_msr_bitmap_legacy_x2apic_apicv_inactive = - (unsigned long *)__get_free_page(GFP_KERNEL); - if (!vmx_msr_bitmap_legacy_x2apic_apicv_inactive) - goto out3; - - vmx_msr_bitmap_longmode = (unsigned long *)__get_free_page(GFP_KERNEL); - if (!vmx_msr_bitmap_longmode) - goto out4; - - vmx_msr_bitmap_longmode_x2apic = - (unsigned long *)__get_free_page(GFP_KERNEL); - if (!vmx_msr_bitmap_longmode_x2apic) - goto out5; - - vmx_msr_bitmap_longmode_x2apic_apicv_inactive = - (unsigned long *)__get_free_page(GFP_KERNEL); - if (!vmx_msr_bitmap_longmode_x2apic_apicv_inactive) - goto out6; - - vmx_vmread_bitmap = (unsigned long *)__get_free_page(GFP_KERNEL); - if (!vmx_vmread_bitmap) - goto out7; - - vmx_vmwrite_bitmap = (unsigned long *)__get_free_page(GFP_KERNEL); - if (!vmx_vmwrite_bitmap) - goto out8; - memset(vmx_vmread_bitmap, 0xff, PAGE_SIZE); memset(vmx_vmwrite_bitmap, 0xff, PAGE_SIZE); @@ -6409,7 +6611,7 @@ static __init int hardware_setup(void) if (setup_vmcs_config(&vmcs_config) < 0) { r = -EIO; - goto out9; + goto out; } if (boot_cpu_has(X86_FEATURE_NX)) @@ -6472,39 +6674,34 @@ static __init int hardware_setup(void) vmx_disable_intercept_for_msr(MSR_IA32_SYSENTER_EIP, false); vmx_disable_intercept_for_msr(MSR_IA32_BNDCFGS, true); - memcpy(vmx_msr_bitmap_legacy_x2apic, + memcpy(vmx_msr_bitmap_legacy_x2apic_apicv, vmx_msr_bitmap_legacy, PAGE_SIZE); - memcpy(vmx_msr_bitmap_longmode_x2apic, + memcpy(vmx_msr_bitmap_longmode_x2apic_apicv, vmx_msr_bitmap_longmode, PAGE_SIZE); - memcpy(vmx_msr_bitmap_legacy_x2apic_apicv_inactive, + memcpy(vmx_msr_bitmap_legacy_x2apic, vmx_msr_bitmap_legacy, PAGE_SIZE); - memcpy(vmx_msr_bitmap_longmode_x2apic_apicv_inactive, + memcpy(vmx_msr_bitmap_longmode_x2apic, vmx_msr_bitmap_longmode, PAGE_SIZE); set_bit(0, vmx_vpid_bitmap); /* 0 is reserved for host */ + for (msr = 0x800; msr <= 0x8ff; msr++) { + if (msr == 0x839 /* TMCCT */) + continue; + vmx_disable_intercept_msr_x2apic(msr, MSR_TYPE_R, true); + } + /* - * enable_apicv && kvm_vcpu_apicv_active() + * TPR reads and writes can be virtualized even if virtual interrupt + * delivery is not in use. */ - for (msr = 0x800; msr <= 0x8ff; msr++) - vmx_disable_intercept_msr_read_x2apic(msr, true); + vmx_disable_intercept_msr_x2apic(0x808, MSR_TYPE_W, true); + vmx_disable_intercept_msr_x2apic(0x808, MSR_TYPE_R | MSR_TYPE_W, false); - /* TMCCT */ - vmx_enable_intercept_msr_read_x2apic(0x839, true); - /* TPR */ - vmx_disable_intercept_msr_write_x2apic(0x808, true); /* EOI */ - vmx_disable_intercept_msr_write_x2apic(0x80b, true); + vmx_disable_intercept_msr_x2apic(0x80b, MSR_TYPE_W, true); /* SELF-IPI */ - vmx_disable_intercept_msr_write_x2apic(0x83f, true); - - /* - * (enable_apicv && !kvm_vcpu_apicv_active()) || - * !enable_apicv - */ - /* TPR */ - vmx_disable_intercept_msr_read_x2apic(0x808, false); - vmx_disable_intercept_msr_write_x2apic(0x808, false); + vmx_disable_intercept_msr_x2apic(0x83f, MSR_TYPE_W, true); if (enable_ept) { kvm_mmu_set_mask_ptes(VMX_EPT_READABLE_MASK, @@ -6551,42 +6748,19 @@ static __init int hardware_setup(void) return alloc_kvm_area(); -out9: - free_page((unsigned long)vmx_vmwrite_bitmap); -out8: - free_page((unsigned long)vmx_vmread_bitmap); -out7: - free_page((unsigned long)vmx_msr_bitmap_longmode_x2apic_apicv_inactive); -out6: - free_page((unsigned long)vmx_msr_bitmap_longmode_x2apic); -out5: - free_page((unsigned long)vmx_msr_bitmap_longmode); -out4: - free_page((unsigned long)vmx_msr_bitmap_legacy_x2apic_apicv_inactive); -out3: - free_page((unsigned long)vmx_msr_bitmap_legacy_x2apic); -out2: - free_page((unsigned long)vmx_msr_bitmap_legacy); -out1: - free_page((unsigned long)vmx_io_bitmap_b); out: - free_page((unsigned long)vmx_io_bitmap_a); + for (i = 0; i < VMX_BITMAP_NR; i++) + free_page((unsigned long)vmx_bitmap[i]); return r; } static __exit void hardware_unsetup(void) { - free_page((unsigned long)vmx_msr_bitmap_legacy_x2apic); - free_page((unsigned long)vmx_msr_bitmap_legacy_x2apic_apicv_inactive); - free_page((unsigned long)vmx_msr_bitmap_longmode_x2apic); - free_page((unsigned long)vmx_msr_bitmap_longmode_x2apic_apicv_inactive); - free_page((unsigned long)vmx_msr_bitmap_legacy); - free_page((unsigned long)vmx_msr_bitmap_longmode); - free_page((unsigned long)vmx_io_bitmap_b); - free_page((unsigned long)vmx_io_bitmap_a); - free_page((unsigned long)vmx_vmwrite_bitmap); - free_page((unsigned long)vmx_vmread_bitmap); + int i; + + for (i = 0; i < VMX_BITMAP_NR; i++) + free_page((unsigned long)vmx_bitmap[i]); free_kvm_area(); } @@ -6600,16 +6774,13 @@ static int handle_pause(struct kvm_vcpu *vcpu) if (ple_gap) grow_ple_window(vcpu); - skip_emulated_instruction(vcpu); kvm_vcpu_on_spin(vcpu); - - return 1; + return kvm_skip_emulated_instruction(vcpu); } static int handle_nop(struct kvm_vcpu *vcpu) { - skip_emulated_instruction(vcpu); - return 1; + return kvm_skip_emulated_instruction(vcpu); } static int handle_mwait(struct kvm_vcpu *vcpu) @@ -6916,8 +7087,7 @@ static int nested_vmx_check_vmptr(struct kvm_vcpu *vcpu, int exit_reason, */ if (!PAGE_ALIGNED(vmptr) || (vmptr >> maxphyaddr)) { nested_vmx_failInvalid(vcpu); - skip_emulated_instruction(vcpu); - return 1; + return kvm_skip_emulated_instruction(vcpu); } page = nested_get_page(vcpu, vmptr); @@ -6925,8 +7095,7 @@ static int nested_vmx_check_vmptr(struct kvm_vcpu *vcpu, int exit_reason, *(u32 *)kmap(page) != VMCS12_REVISION) { nested_vmx_failInvalid(vcpu); kunmap(page); - skip_emulated_instruction(vcpu); - return 1; + return kvm_skip_emulated_instruction(vcpu); } kunmap(page); vmx->nested.vmxon_ptr = vmptr; @@ -6935,30 +7104,26 @@ static int nested_vmx_check_vmptr(struct kvm_vcpu *vcpu, int exit_reason, if (!PAGE_ALIGNED(vmptr) || (vmptr >> maxphyaddr)) { nested_vmx_failValid(vcpu, VMXERR_VMCLEAR_INVALID_ADDRESS); - skip_emulated_instruction(vcpu); - return 1; + return kvm_skip_emulated_instruction(vcpu); } if (vmptr == vmx->nested.vmxon_ptr) { nested_vmx_failValid(vcpu, VMXERR_VMCLEAR_VMXON_POINTER); - skip_emulated_instruction(vcpu); - return 1; + return kvm_skip_emulated_instruction(vcpu); } break; case EXIT_REASON_VMPTRLD: if (!PAGE_ALIGNED(vmptr) || (vmptr >> maxphyaddr)) { nested_vmx_failValid(vcpu, VMXERR_VMPTRLD_INVALID_ADDRESS); - skip_emulated_instruction(vcpu); - return 1; + return kvm_skip_emulated_instruction(vcpu); } if (vmptr == vmx->nested.vmxon_ptr) { nested_vmx_failValid(vcpu, VMXERR_VMCLEAR_VMXON_POINTER); - skip_emulated_instruction(vcpu); - return 1; + return kvm_skip_emulated_instruction(vcpu); } break; default: @@ -7014,8 +7179,7 @@ static int handle_vmon(struct kvm_vcpu *vcpu) if (vmx->nested.vmxon) { nested_vmx_failValid(vcpu, VMXERR_VMXON_IN_VMX_ROOT_OPERATION); - skip_emulated_instruction(vcpu); - return 1; + return kvm_skip_emulated_instruction(vcpu); } if ((vmx->msr_ia32_feature_control & VMXON_NEEDED_FEATURES) @@ -7055,9 +7219,8 @@ static int handle_vmon(struct kvm_vcpu *vcpu) vmx->nested.vmxon = true; - skip_emulated_instruction(vcpu); nested_vmx_succeed(vcpu); - return 1; + return kvm_skip_emulated_instruction(vcpu); out_shadow_vmcs: kfree(vmx->nested.cached_vmcs12); @@ -7176,9 +7339,8 @@ static int handle_vmoff(struct kvm_vcpu *vcpu) if (!nested_vmx_check_permission(vcpu)) return 1; free_nested(to_vmx(vcpu)); - skip_emulated_instruction(vcpu); nested_vmx_succeed(vcpu); - return 1; + return kvm_skip_emulated_instruction(vcpu); } /* Emulate the VMCLEAR instruction */ @@ -7217,9 +7379,8 @@ static int handle_vmclear(struct kvm_vcpu *vcpu) nested_free_vmcs02(vmx, vmptr); - skip_emulated_instruction(vcpu); nested_vmx_succeed(vcpu); - return 1; + return kvm_skip_emulated_instruction(vcpu); } static int nested_vmx_run(struct kvm_vcpu *vcpu, bool launch); @@ -7417,7 +7578,6 @@ static int nested_vmx_check_vmcs12(struct kvm_vcpu *vcpu) struct vcpu_vmx *vmx = to_vmx(vcpu); if (vmx->nested.current_vmptr == -1ull) { nested_vmx_failInvalid(vcpu); - skip_emulated_instruction(vcpu); return 0; } return 1; @@ -7431,17 +7591,18 @@ static int handle_vmread(struct kvm_vcpu *vcpu) u32 vmx_instruction_info = vmcs_read32(VMX_INSTRUCTION_INFO); gva_t gva = 0; - if (!nested_vmx_check_permission(vcpu) || - !nested_vmx_check_vmcs12(vcpu)) + if (!nested_vmx_check_permission(vcpu)) return 1; + if (!nested_vmx_check_vmcs12(vcpu)) + return kvm_skip_emulated_instruction(vcpu); + /* Decode instruction info and find the field to read */ field = kvm_register_readl(vcpu, (((vmx_instruction_info) >> 28) & 0xf)); /* Read the field, zero-extended to a u64 field_value */ if (vmcs12_read_any(vcpu, field, &field_value) < 0) { nested_vmx_failValid(vcpu, VMXERR_UNSUPPORTED_VMCS_COMPONENT); - skip_emulated_instruction(vcpu); - return 1; + return kvm_skip_emulated_instruction(vcpu); } /* * Now copy part of this value to register or memory, as requested. @@ -7461,8 +7622,7 @@ static int handle_vmread(struct kvm_vcpu *vcpu) } nested_vmx_succeed(vcpu); - skip_emulated_instruction(vcpu); - return 1; + return kvm_skip_emulated_instruction(vcpu); } @@ -7481,10 +7641,12 @@ static int handle_vmwrite(struct kvm_vcpu *vcpu) u64 field_value = 0; struct x86_exception e; - if (!nested_vmx_check_permission(vcpu) || - !nested_vmx_check_vmcs12(vcpu)) + if (!nested_vmx_check_permission(vcpu)) return 1; + if (!nested_vmx_check_vmcs12(vcpu)) + return kvm_skip_emulated_instruction(vcpu); + if (vmx_instruction_info & (1u << 10)) field_value = kvm_register_readl(vcpu, (((vmx_instruction_info) >> 3) & 0xf)); @@ -7504,19 +7666,16 @@ static int handle_vmwrite(struct kvm_vcpu *vcpu) if (vmcs_field_readonly(field)) { nested_vmx_failValid(vcpu, VMXERR_VMWRITE_READ_ONLY_VMCS_COMPONENT); - skip_emulated_instruction(vcpu); - return 1; + return kvm_skip_emulated_instruction(vcpu); } if (vmcs12_write_any(vcpu, field, field_value) < 0) { nested_vmx_failValid(vcpu, VMXERR_UNSUPPORTED_VMCS_COMPONENT); - skip_emulated_instruction(vcpu); - return 1; + return kvm_skip_emulated_instruction(vcpu); } nested_vmx_succeed(vcpu); - skip_emulated_instruction(vcpu); - return 1; + return kvm_skip_emulated_instruction(vcpu); } /* Emulate the VMPTRLD instruction */ @@ -7537,8 +7696,7 @@ static int handle_vmptrld(struct kvm_vcpu *vcpu) page = nested_get_page(vcpu, vmptr); if (page == NULL) { nested_vmx_failInvalid(vcpu); - skip_emulated_instruction(vcpu); - return 1; + return kvm_skip_emulated_instruction(vcpu); } new_vmcs12 = kmap(page); if (new_vmcs12->revision_id != VMCS12_REVISION) { @@ -7546,8 +7704,7 @@ static int handle_vmptrld(struct kvm_vcpu *vcpu) nested_release_page_clean(page); nested_vmx_failValid(vcpu, VMXERR_VMPTRLD_INCORRECT_VMCS_REVISION_ID); - skip_emulated_instruction(vcpu); - return 1; + return kvm_skip_emulated_instruction(vcpu); } nested_release_vmcs12(vmx); @@ -7571,8 +7728,7 @@ static int handle_vmptrld(struct kvm_vcpu *vcpu) } nested_vmx_succeed(vcpu); - skip_emulated_instruction(vcpu); - return 1; + return kvm_skip_emulated_instruction(vcpu); } /* Emulate the VMPTRST instruction */ @@ -7597,8 +7753,7 @@ static int handle_vmptrst(struct kvm_vcpu *vcpu) return 1; } nested_vmx_succeed(vcpu); - skip_emulated_instruction(vcpu); - return 1; + return kvm_skip_emulated_instruction(vcpu); } /* Emulate the INVEPT instruction */ @@ -7636,8 +7791,7 @@ static int handle_invept(struct kvm_vcpu *vcpu) if (type >= 32 || !(types & (1 << type))) { nested_vmx_failValid(vcpu, VMXERR_INVALID_OPERAND_TO_INVEPT_INVVPID); - skip_emulated_instruction(vcpu); - return 1; + return kvm_skip_emulated_instruction(vcpu); } /* According to the Intel VMX instruction reference, the memory @@ -7668,8 +7822,7 @@ static int handle_invept(struct kvm_vcpu *vcpu) break; } - skip_emulated_instruction(vcpu); - return 1; + return kvm_skip_emulated_instruction(vcpu); } static int handle_invvpid(struct kvm_vcpu *vcpu) @@ -7694,13 +7847,13 @@ static int handle_invvpid(struct kvm_vcpu *vcpu) vmx_instruction_info = vmcs_read32(VMX_INSTRUCTION_INFO); type = kvm_register_readl(vcpu, (vmx_instruction_info >> 28) & 0xf); - types = (vmx->nested.nested_vmx_vpid_caps >> 8) & 0x7; + types = (vmx->nested.nested_vmx_vpid_caps & + VMX_VPID_EXTENT_SUPPORTED_MASK) >> 8; if (type >= 32 || !(types & (1 << type))) { nested_vmx_failValid(vcpu, VMXERR_INVALID_OPERAND_TO_INVEPT_INVVPID); - skip_emulated_instruction(vcpu); - return 1; + return kvm_skip_emulated_instruction(vcpu); } /* according to the intel vmx instruction reference, the memory @@ -7716,23 +7869,26 @@ static int handle_invvpid(struct kvm_vcpu *vcpu) } switch (type) { + case VMX_VPID_EXTENT_INDIVIDUAL_ADDR: case VMX_VPID_EXTENT_SINGLE_CONTEXT: - /* - * Old versions of KVM use the single-context version so we - * have to support it; just treat it the same as all-context. - */ + case VMX_VPID_EXTENT_SINGLE_NON_GLOBAL: + if (!vpid) { + nested_vmx_failValid(vcpu, + VMXERR_INVALID_OPERAND_TO_INVEPT_INVVPID); + return kvm_skip_emulated_instruction(vcpu); + } + break; case VMX_VPID_EXTENT_ALL_CONTEXT: - __vmx_flush_tlb(vcpu, to_vmx(vcpu)->nested.vpid02); - nested_vmx_succeed(vcpu); break; default: - /* Trap individual address invalidation invvpid calls */ - BUG_ON(1); - break; + WARN_ON_ONCE(1); + return kvm_skip_emulated_instruction(vcpu); } - skip_emulated_instruction(vcpu); - return 1; + __vmx_flush_tlb(vcpu, vmx->nested.vpid02); + nested_vmx_succeed(vcpu); + + return kvm_skip_emulated_instruction(vcpu); } static int handle_pml_full(struct kvm_vcpu *vcpu) @@ -8071,6 +8227,8 @@ static bool nested_vmx_exit_handled(struct kvm_vcpu *vcpu) return nested_cpu_has(vmcs12, CPU_BASED_MOV_DR_EXITING); case EXIT_REASON_IO_INSTRUCTION: return nested_vmx_exit_handled_io(vcpu, vmcs12); + case EXIT_REASON_GDTR_IDTR: case EXIT_REASON_LDTR_TR: + return nested_cpu_has2(vmcs12, SECONDARY_EXEC_DESC); case EXIT_REASON_MSR_READ: case EXIT_REASON_MSR_WRITE: return nested_vmx_exit_handled_msr(vcpu, vmcs12, exit_reason); @@ -8620,11 +8778,6 @@ static void vmx_handle_external_intr(struct kvm_vcpu *vcpu) u32 exit_intr_info = vmcs_read32(VM_EXIT_INTR_INFO); register void *__sp asm(_ASM_SP); - /* - * If external interrupt exists, IF bit is set in rflags/eflags on the - * interrupt stack frame, and interrupt will be enabled on a return - * from interrupt handler. - */ if ((exit_intr_info & (INTR_INFO_VALID_MASK | INTR_INFO_INTR_TYPE_MASK)) == (INTR_INFO_VALID_MASK | INTR_TYPE_EXT_INTR)) { unsigned int vector; @@ -8809,7 +8962,7 @@ static void atomic_switch_perf_msrs(struct vcpu_vmx *vmx) msrs[i].host); } -void vmx_arm_hv_timer(struct kvm_vcpu *vcpu) +static void vmx_arm_hv_timer(struct kvm_vcpu *vcpu) { struct vcpu_vmx *vmx = to_vmx(vcpu); u64 tscl; @@ -9279,6 +9432,50 @@ static void vmcs_set_secondary_exec_control(u32 new_ctl) (new_ctl & ~mask) | (cur_ctl & mask)); } +/* + * Generate MSR_IA32_VMX_CR{0,4}_FIXED1 according to CPUID. Only set bits + * (indicating "allowed-1") if they are supported in the guest's CPUID. + */ +static void nested_vmx_cr_fixed1_bits_update(struct kvm_vcpu *vcpu) +{ + struct vcpu_vmx *vmx = to_vmx(vcpu); + struct kvm_cpuid_entry2 *entry; + + vmx->nested.nested_vmx_cr0_fixed1 = 0xffffffff; + vmx->nested.nested_vmx_cr4_fixed1 = X86_CR4_PCE; + +#define cr4_fixed1_update(_cr4_mask, _reg, _cpuid_mask) do { \ + if (entry && (entry->_reg & (_cpuid_mask))) \ + vmx->nested.nested_vmx_cr4_fixed1 |= (_cr4_mask); \ +} while (0) + + entry = kvm_find_cpuid_entry(vcpu, 0x1, 0); + cr4_fixed1_update(X86_CR4_VME, edx, bit(X86_FEATURE_VME)); + cr4_fixed1_update(X86_CR4_PVI, edx, bit(X86_FEATURE_VME)); + cr4_fixed1_update(X86_CR4_TSD, edx, bit(X86_FEATURE_TSC)); + cr4_fixed1_update(X86_CR4_DE, edx, bit(X86_FEATURE_DE)); + cr4_fixed1_update(X86_CR4_PSE, edx, bit(X86_FEATURE_PSE)); + cr4_fixed1_update(X86_CR4_PAE, edx, bit(X86_FEATURE_PAE)); + cr4_fixed1_update(X86_CR4_MCE, edx, bit(X86_FEATURE_MCE)); + cr4_fixed1_update(X86_CR4_PGE, edx, bit(X86_FEATURE_PGE)); + cr4_fixed1_update(X86_CR4_OSFXSR, edx, bit(X86_FEATURE_FXSR)); + cr4_fixed1_update(X86_CR4_OSXMMEXCPT, edx, bit(X86_FEATURE_XMM)); + cr4_fixed1_update(X86_CR4_VMXE, ecx, bit(X86_FEATURE_VMX)); + cr4_fixed1_update(X86_CR4_SMXE, ecx, bit(X86_FEATURE_SMX)); + cr4_fixed1_update(X86_CR4_PCIDE, ecx, bit(X86_FEATURE_PCID)); + cr4_fixed1_update(X86_CR4_OSXSAVE, ecx, bit(X86_FEATURE_XSAVE)); + + entry = kvm_find_cpuid_entry(vcpu, 0x7, 0); + cr4_fixed1_update(X86_CR4_FSGSBASE, ebx, bit(X86_FEATURE_FSGSBASE)); + cr4_fixed1_update(X86_CR4_SMEP, ebx, bit(X86_FEATURE_SMEP)); + cr4_fixed1_update(X86_CR4_SMAP, ebx, bit(X86_FEATURE_SMAP)); + cr4_fixed1_update(X86_CR4_PKE, ecx, bit(X86_FEATURE_PKU)); + /* TODO: Use X86_CR4_UMIP and X86_FEATURE_UMIP macros */ + cr4_fixed1_update(bit(11), ecx, bit(2)); + +#undef cr4_fixed1_update +} + static void vmx_cpuid_update(struct kvm_vcpu *vcpu) { struct kvm_cpuid_entry2 *best; @@ -9320,6 +9517,9 @@ static void vmx_cpuid_update(struct kvm_vcpu *vcpu) else to_vmx(vcpu)->msr_ia32_feature_control_valid_bits &= ~FEATURE_CONTROL_VMXON_ENABLED_OUTSIDE_SMX; + + if (nested_vmx_allowed(vcpu)) + nested_vmx_cr_fixed1_bits_update(vcpu); } static void vmx_set_supported_cpuid(u32 func, struct kvm_cpuid_entry2 *entry) @@ -9774,6 +9974,49 @@ static int nested_vmx_store_msr(struct kvm_vcpu *vcpu, u64 gpa, u32 count) return 0; } +static bool nested_cr3_valid(struct kvm_vcpu *vcpu, unsigned long val) +{ + unsigned long invalid_mask; + + invalid_mask = (~0ULL) << cpuid_maxphyaddr(vcpu); + return (val & invalid_mask) == 0; +} + +/* + * Load guest's/host's cr3 at nested entry/exit. nested_ept is true if we are + * emulating VM entry into a guest with EPT enabled. + * Returns 0 on success, 1 on failure. Invalid state exit qualification code + * is assigned to entry_failure_code on failure. + */ +static int nested_vmx_load_cr3(struct kvm_vcpu *vcpu, unsigned long cr3, bool nested_ept, + unsigned long *entry_failure_code) +{ + if (cr3 != kvm_read_cr3(vcpu) || (!nested_ept && pdptrs_changed(vcpu))) { + if (!nested_cr3_valid(vcpu, cr3)) { + *entry_failure_code = ENTRY_FAIL_DEFAULT; + return 1; + } + + /* + * If PAE paging and EPT are both on, CR3 is not used by the CPU and + * must not be dereferenced. + */ + if (!is_long_mode(vcpu) && is_pae(vcpu) && is_paging(vcpu) && + !nested_ept) { + if (!load_pdptrs(vcpu, vcpu->arch.walk_mmu, cr3)) { + *entry_failure_code = ENTRY_FAIL_PDPTE; + return 1; + } + } + + vcpu->arch.cr3 = cr3; + __set_bit(VCPU_EXREG_CR3, (ulong *)&vcpu->arch.regs_avail); + } + + kvm_mmu_reset_context(vcpu); + return 0; +} + /* * prepare_vmcs02 is called when the L1 guest hypervisor runs its nested * L2 guest. L1 has a vmcs for L2 (vmcs12), and this function "merges" it @@ -9782,11 +10025,15 @@ static int nested_vmx_store_msr(struct kvm_vcpu *vcpu, u64 gpa, u32 count) * needs. In addition to modifying the active vmcs (which is vmcs02), this * function also has additional necessary side-effects, like setting various * vcpu->arch fields. + * Returns 0 on success, 1 on failure. Invalid state exit qualification code + * is assigned to entry_failure_code on failure. */ -static void prepare_vmcs02(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12) +static int prepare_vmcs02(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12, + unsigned long *entry_failure_code) { struct vcpu_vmx *vmx = to_vmx(vcpu); u32 exec_control; + bool nested_ept_enabled = false; vmcs_write16(GUEST_ES_SELECTOR, vmcs12->guest_es_selector); vmcs_write16(GUEST_CS_SELECTOR, vmcs12->guest_cs_selector); @@ -9951,6 +10198,7 @@ static void prepare_vmcs02(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12) vmcs12->guest_intr_status); } + nested_ept_enabled = (exec_control & SECONDARY_EXEC_ENABLE_EPT) != 0; vmcs_write32(SECONDARY_VM_EXEC_CONTROL, exec_control); } @@ -9964,6 +10212,15 @@ static void prepare_vmcs02(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12) vmx_set_constant_host_state(vmx); /* + * Set the MSR load/store lists to match L0's settings. + */ + vmcs_write32(VM_EXIT_MSR_STORE_COUNT, 0); + vmcs_write32(VM_EXIT_MSR_LOAD_COUNT, vmx->msr_autoload.nr); + vmcs_write64(VM_EXIT_MSR_LOAD_ADDR, __pa(vmx->msr_autoload.host)); + vmcs_write32(VM_ENTRY_MSR_LOAD_COUNT, vmx->msr_autoload.nr); + vmcs_write64(VM_ENTRY_MSR_LOAD_ADDR, __pa(vmx->msr_autoload.guest)); + + /* * HOST_RSP is normally set correctly in vmx_vcpu_run() just before * entry, but only if the current (host) sp changed from the value * we wrote last (vmx->host_rsp). This cache is no longer relevant @@ -10069,15 +10326,6 @@ static void prepare_vmcs02(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12) nested_ept_init_mmu_context(vcpu); } - if (vmcs12->vm_entry_controls & VM_ENTRY_LOAD_IA32_EFER) - vcpu->arch.efer = vmcs12->guest_ia32_efer; - else if (vmcs12->vm_entry_controls & VM_ENTRY_IA32E_MODE) - vcpu->arch.efer |= (EFER_LMA | EFER_LME); - else - vcpu->arch.efer &= ~(EFER_LMA | EFER_LME); - /* Note: modifies VM_ENTRY/EXIT_CONTROLS and GUEST/HOST_IA32_EFER */ - vmx_set_efer(vcpu, vcpu->arch.efer); - /* * This sets GUEST_CR0 to vmcs12->guest_cr0, with possibly a modified * TS bit (for lazy fpu) and bits which we consider mandatory enabled. @@ -10092,8 +10340,20 @@ static void prepare_vmcs02(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12) vmx_set_cr4(vcpu, vmcs12->guest_cr4); vmcs_writel(CR4_READ_SHADOW, nested_read_cr4(vmcs12)); - /* shadow page tables on either EPT or shadow page tables */ - kvm_set_cr3(vcpu, vmcs12->guest_cr3); + if (vmcs12->vm_entry_controls & VM_ENTRY_LOAD_IA32_EFER) + vcpu->arch.efer = vmcs12->guest_ia32_efer; + else if (vmcs12->vm_entry_controls & VM_ENTRY_IA32E_MODE) + vcpu->arch.efer |= (EFER_LMA | EFER_LME); + else + vcpu->arch.efer &= ~(EFER_LMA | EFER_LME); + /* Note: modifies VM_ENTRY/EXIT_CONTROLS and GUEST/HOST_IA32_EFER */ + vmx_set_efer(vcpu, vcpu->arch.efer); + + /* Shadow page tables on either EPT or shadow page tables. */ + if (nested_vmx_load_cr3(vcpu, vmcs12->guest_cr3, nested_ept_enabled, + entry_failure_code)) + return 1; + kvm_mmu_reset_context(vcpu); if (!enable_ept) @@ -10111,6 +10371,7 @@ static void prepare_vmcs02(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12) kvm_register_write(vcpu, VCPU_REGS_RSP, vmcs12->guest_rsp); kvm_register_write(vcpu, VCPU_REGS_RIP, vmcs12->guest_rip); + return 0; } /* @@ -10125,12 +10386,14 @@ static int nested_vmx_run(struct kvm_vcpu *vcpu, bool launch) struct loaded_vmcs *vmcs02; bool ia32e; u32 msr_entry_idx; + unsigned long exit_qualification; - if (!nested_vmx_check_permission(vcpu) || - !nested_vmx_check_vmcs12(vcpu)) + if (!nested_vmx_check_permission(vcpu)) return 1; - skip_emulated_instruction(vcpu); + if (!nested_vmx_check_vmcs12(vcpu)) + goto out; + vmcs12 = get_vmcs12(vcpu); if (enable_shadow_vmcs) @@ -10150,37 +10413,37 @@ static int nested_vmx_run(struct kvm_vcpu *vcpu, bool launch) nested_vmx_failValid(vcpu, launch ? VMXERR_VMLAUNCH_NONCLEAR_VMCS : VMXERR_VMRESUME_NONLAUNCHED_VMCS); - return 1; + goto out; } if (vmcs12->guest_activity_state != GUEST_ACTIVITY_ACTIVE && vmcs12->guest_activity_state != GUEST_ACTIVITY_HLT) { nested_vmx_failValid(vcpu, VMXERR_ENTRY_INVALID_CONTROL_FIELD); - return 1; + goto out; } if (!nested_get_vmcs12_pages(vcpu, vmcs12)) { nested_vmx_failValid(vcpu, VMXERR_ENTRY_INVALID_CONTROL_FIELD); - return 1; + goto out; } if (nested_vmx_check_msr_bitmap_controls(vcpu, vmcs12)) { nested_vmx_failValid(vcpu, VMXERR_ENTRY_INVALID_CONTROL_FIELD); - return 1; + goto out; } if (nested_vmx_check_apicv_controls(vcpu, vmcs12)) { nested_vmx_failValid(vcpu, VMXERR_ENTRY_INVALID_CONTROL_FIELD); - return 1; + goto out; } if (nested_vmx_check_msr_switch_controls(vcpu, vmcs12)) { nested_vmx_failValid(vcpu, VMXERR_ENTRY_INVALID_CONTROL_FIELD); - return 1; + goto out; } if (!vmx_control_verify(vmcs12->cpu_based_vm_exec_control, - vmx->nested.nested_vmx_true_procbased_ctls_low, + vmx->nested.nested_vmx_procbased_ctls_low, vmx->nested.nested_vmx_procbased_ctls_high) || !vmx_control_verify(vmcs12->secondary_vm_exec_control, vmx->nested.nested_vmx_secondary_ctls_low, @@ -10189,33 +10452,34 @@ static int nested_vmx_run(struct kvm_vcpu *vcpu, bool launch) vmx->nested.nested_vmx_pinbased_ctls_low, vmx->nested.nested_vmx_pinbased_ctls_high) || !vmx_control_verify(vmcs12->vm_exit_controls, - vmx->nested.nested_vmx_true_exit_ctls_low, + vmx->nested.nested_vmx_exit_ctls_low, vmx->nested.nested_vmx_exit_ctls_high) || !vmx_control_verify(vmcs12->vm_entry_controls, - vmx->nested.nested_vmx_true_entry_ctls_low, + vmx->nested.nested_vmx_entry_ctls_low, vmx->nested.nested_vmx_entry_ctls_high)) { nested_vmx_failValid(vcpu, VMXERR_ENTRY_INVALID_CONTROL_FIELD); - return 1; + goto out; } - if (((vmcs12->host_cr0 & VMXON_CR0_ALWAYSON) != VMXON_CR0_ALWAYSON) || - ((vmcs12->host_cr4 & VMXON_CR4_ALWAYSON) != VMXON_CR4_ALWAYSON)) { + if (!nested_host_cr0_valid(vcpu, vmcs12->host_cr0) || + !nested_host_cr4_valid(vcpu, vmcs12->host_cr4) || + !nested_cr3_valid(vcpu, vmcs12->host_cr3)) { nested_vmx_failValid(vcpu, VMXERR_ENTRY_INVALID_HOST_STATE_FIELD); - return 1; + goto out; } - if (!nested_cr0_valid(vcpu, vmcs12->guest_cr0) || - ((vmcs12->guest_cr4 & VMXON_CR4_ALWAYSON) != VMXON_CR4_ALWAYSON)) { + if (!nested_guest_cr0_valid(vcpu, vmcs12->guest_cr0) || + !nested_guest_cr4_valid(vcpu, vmcs12->guest_cr4)) { nested_vmx_entry_failure(vcpu, vmcs12, EXIT_REASON_INVALID_STATE, ENTRY_FAIL_DEFAULT); - return 1; + goto out; } if (vmcs12->vmcs_link_pointer != -1ull) { nested_vmx_entry_failure(vcpu, vmcs12, EXIT_REASON_INVALID_STATE, ENTRY_FAIL_VMCS_LINK_PTR); - return 1; + goto out; } /* @@ -10235,7 +10499,7 @@ static int nested_vmx_run(struct kvm_vcpu *vcpu, bool launch) ia32e != !!(vmcs12->guest_ia32_efer & EFER_LME))) { nested_vmx_entry_failure(vcpu, vmcs12, EXIT_REASON_INVALID_STATE, ENTRY_FAIL_DEFAULT); - return 1; + goto out; } } @@ -10253,7 +10517,7 @@ static int nested_vmx_run(struct kvm_vcpu *vcpu, bool launch) ia32e != !!(vmcs12->host_ia32_efer & EFER_LME)) { nested_vmx_entry_failure(vcpu, vmcs12, EXIT_REASON_INVALID_STATE, ENTRY_FAIL_DEFAULT); - return 1; + goto out; } } @@ -10266,6 +10530,12 @@ static int nested_vmx_run(struct kvm_vcpu *vcpu, bool launch) if (!vmcs02) return -ENOMEM; + /* + * After this point, the trap flag no longer triggers a singlestep trap + * on the vm entry instructions. Don't call + * kvm_skip_emulated_instruction. + */ + skip_emulated_instruction(vcpu); enter_guest_mode(vcpu); if (!(vmcs12->vm_entry_controls & VM_ENTRY_LOAD_DEBUG_CONTROLS)) @@ -10280,7 +10550,13 @@ static int nested_vmx_run(struct kvm_vcpu *vcpu, bool launch) vmx_segment_cache_clear(vmx); - prepare_vmcs02(vcpu, vmcs12); + if (prepare_vmcs02(vcpu, vmcs12, &exit_qualification)) { + leave_guest_mode(vcpu); + vmx_load_vmcs01(vcpu); + nested_vmx_entry_failure(vcpu, vmcs12, + EXIT_REASON_INVALID_STATE, exit_qualification); + return 1; + } msr_entry_idx = nested_vmx_load_msr(vcpu, vmcs12->vm_entry_msr_load_addr, @@ -10307,6 +10583,9 @@ static int nested_vmx_run(struct kvm_vcpu *vcpu, bool launch) * the success flag) when L2 exits (see nested_vmx_vmexit()). */ return 1; + +out: + return kvm_skip_emulated_instruction(vcpu); } /* @@ -10612,6 +10891,7 @@ static void load_vmcs12_host_state(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12) { struct kvm_segment seg; + unsigned long entry_failure_code; if (vmcs12->vm_exit_controls & VM_EXIT_LOAD_IA32_EFER) vcpu->arch.efer = vmcs12->host_ia32_efer; @@ -10649,8 +10929,12 @@ static void load_vmcs12_host_state(struct kvm_vcpu *vcpu, nested_ept_uninit_mmu_context(vcpu); - kvm_set_cr3(vcpu, vmcs12->host_cr3); - kvm_mmu_reset_context(vcpu); + /* + * Only PDPTE load can fail as the value of cr3 was checked on entry and + * couldn't have changed. + */ + if (nested_vmx_load_cr3(vcpu, vmcs12->host_cr3, false, &entry_failure_code)) + nested_vmx_abort(vcpu, VMX_ABORT_LOAD_HOST_PDPTE_FAIL); if (!enable_ept) vcpu->arch.walk_mmu->inject_page_fault = kvm_inject_page_fault; @@ -10751,6 +11035,7 @@ static void nested_vmx_vmexit(struct kvm_vcpu *vcpu, u32 exit_reason, { struct vcpu_vmx *vmx = to_vmx(vcpu); struct vmcs12 *vmcs12 = get_vmcs12(vcpu); + u32 vm_inst_error = 0; /* trying to cancel vmlaunch/vmresume is a bug */ WARN_ON_ONCE(vmx->nested.nested_run_pending); @@ -10763,6 +11048,9 @@ static void nested_vmx_vmexit(struct kvm_vcpu *vcpu, u32 exit_reason, vmcs12->vm_exit_msr_store_count)) nested_vmx_abort(vcpu, VMX_ABORT_SAVE_GUEST_MSR_FAIL); + if (unlikely(vmx->fail)) + vm_inst_error = vmcs_read32(VM_INSTRUCTION_ERROR); + vmx_load_vmcs01(vcpu); if ((exit_reason == EXIT_REASON_EXTERNAL_INTERRUPT) @@ -10791,6 +11079,8 @@ static void nested_vmx_vmexit(struct kvm_vcpu *vcpu, u32 exit_reason, load_vmcs12_host_state(vcpu, vmcs12); /* Update any VMCS fields that might have changed while L2 ran */ + vmcs_write32(VM_EXIT_MSR_LOAD_COUNT, vmx->msr_autoload.nr); + vmcs_write32(VM_ENTRY_MSR_LOAD_COUNT, vmx->msr_autoload.nr); vmcs_write64(TSC_OFFSET, vcpu->arch.tsc_offset); if (vmx->hv_deadline_tsc == -1) vmcs_clear_bits(PIN_BASED_VM_EXEC_CONTROL, @@ -10839,7 +11129,7 @@ static void nested_vmx_vmexit(struct kvm_vcpu *vcpu, u32 exit_reason, */ if (unlikely(vmx->fail)) { vmx->fail = 0; - nested_vmx_failValid(vcpu, vmcs_read32(VM_INSTRUCTION_ERROR)); + nested_vmx_failValid(vcpu, vm_inst_error); } else nested_vmx_succeed(vcpu); if (enable_shadow_vmcs) diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index 2912684..1f0d238 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -434,12 +434,14 @@ void kvm_requeue_exception(struct kvm_vcpu *vcpu, unsigned nr) } EXPORT_SYMBOL_GPL(kvm_requeue_exception); -void kvm_complete_insn_gp(struct kvm_vcpu *vcpu, int err) +int kvm_complete_insn_gp(struct kvm_vcpu *vcpu, int err) { if (err) kvm_inject_gp(vcpu, 0); else - kvm_x86_ops->skip_emulated_instruction(vcpu); + return kvm_skip_emulated_instruction(vcpu); + + return 1; } EXPORT_SYMBOL_GPL(kvm_complete_insn_gp); @@ -573,7 +575,7 @@ out: } EXPORT_SYMBOL_GPL(load_pdptrs); -static bool pdptrs_changed(struct kvm_vcpu *vcpu) +bool pdptrs_changed(struct kvm_vcpu *vcpu) { u64 pdpte[ARRAY_SIZE(vcpu->arch.walk_mmu->pdptrs)]; bool changed = true; @@ -599,6 +601,7 @@ out: return changed; } +EXPORT_SYMBOL_GPL(pdptrs_changed); int kvm_set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0) { @@ -2178,7 +2181,6 @@ int kvm_set_msr_common(struct kvm_vcpu *vcpu, struct msr_data *msr_info) break; case MSR_KVM_SYSTEM_TIME_NEW: case MSR_KVM_SYSTEM_TIME: { - u64 gpa_offset; struct kvm_arch *ka = &vcpu->kvm->arch; kvmclock_reset(vcpu); @@ -2200,8 +2202,6 @@ int kvm_set_msr_common(struct kvm_vcpu *vcpu, struct msr_data *msr_info) if (!(data & 1)) break; - gpa_offset = data & ~(PAGE_MASK | 1); - if (kvm_gfn_to_hva_cache_init(vcpu->kvm, &vcpu->arch.pv_time, data & ~1ULL, sizeof(struct pvclock_vcpu_time_info))) @@ -2296,7 +2296,7 @@ int kvm_set_msr_common(struct kvm_vcpu *vcpu, struct msr_data *msr_info) if (kvm_pmu_is_valid_msr(vcpu, msr)) return kvm_pmu_set_msr(vcpu, msr_info); if (!ignore_msrs) { - vcpu_unimpl(vcpu, "unhandled wrmsr: 0x%x data 0x%llx\n", + vcpu_debug_ratelimited(vcpu, "unhandled wrmsr: 0x%x data 0x%llx\n", msr, data); return 1; } else { @@ -2508,7 +2508,8 @@ int kvm_get_msr_common(struct kvm_vcpu *vcpu, struct msr_data *msr_info) if (kvm_pmu_is_valid_msr(vcpu, msr_info->index)) return kvm_pmu_get_msr(vcpu, msr_info->index, &msr_info->data); if (!ignore_msrs) { - vcpu_unimpl(vcpu, "unhandled rdmsr: 0x%x\n", msr_info->index); + vcpu_debug_ratelimited(vcpu, "unhandled rdmsr: 0x%x\n", + msr_info->index); return 1; } else { vcpu_unimpl(vcpu, "ignored rdmsr: 0x%x\n", msr_info->index); @@ -2812,7 +2813,7 @@ void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu) } if (kvm_lapic_hv_timer_in_use(vcpu) && kvm_x86_ops->set_hv_timer(vcpu, - kvm_get_lapic_tscdeadline_msr(vcpu))) + kvm_get_lapic_target_expiration_tsc(vcpu))) kvm_lapic_switch_to_sw_timer(vcpu); /* * On a host with synchronized TSC, there is no need to update @@ -4832,7 +4833,7 @@ static void emulator_invlpg(struct x86_emulate_ctxt *ctxt, ulong address) kvm_mmu_invlpg(emul_to_vcpu(ctxt), address); } -int kvm_emulate_wbinvd_noskip(struct kvm_vcpu *vcpu) +static int kvm_emulate_wbinvd_noskip(struct kvm_vcpu *vcpu) { if (!need_emulate_wbinvd(vcpu)) return X86EMUL_CONTINUE; @@ -4852,8 +4853,8 @@ int kvm_emulate_wbinvd_noskip(struct kvm_vcpu *vcpu) int kvm_emulate_wbinvd(struct kvm_vcpu *vcpu) { - kvm_x86_ops->skip_emulated_instruction(vcpu); - return kvm_emulate_wbinvd_noskip(vcpu); + kvm_emulate_wbinvd_noskip(vcpu); + return kvm_skip_emulated_instruction(vcpu); } EXPORT_SYMBOL_GPL(kvm_emulate_wbinvd); @@ -5451,7 +5452,6 @@ static void kvm_vcpu_check_singlestep(struct kvm_vcpu *vcpu, unsigned long rflag kvm_run->exit_reason = KVM_EXIT_DEBUG; *r = EMULATE_USER_EXIT; } else { - vcpu->arch.emulate_ctxt.eflags &= ~X86_EFLAGS_TF; /* * "Certain debug exceptions may clear bit 0-3. The * remaining contents of the DR6 register are never @@ -5464,6 +5464,17 @@ static void kvm_vcpu_check_singlestep(struct kvm_vcpu *vcpu, unsigned long rflag } } +int kvm_skip_emulated_instruction(struct kvm_vcpu *vcpu) +{ + unsigned long rflags = kvm_x86_ops->get_rflags(vcpu); + int r = EMULATE_DONE; + + kvm_x86_ops->skip_emulated_instruction(vcpu); + kvm_vcpu_check_singlestep(vcpu, rflags, &r); + return r == EMULATE_DONE; +} +EXPORT_SYMBOL_GPL(kvm_skip_emulated_instruction); + static bool kvm_vcpu_check_breakpoint(struct kvm_vcpu *vcpu, int *r) { if (unlikely(vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP) && @@ -5649,6 +5660,49 @@ int kvm_fast_pio_out(struct kvm_vcpu *vcpu, int size, unsigned short port) } EXPORT_SYMBOL_GPL(kvm_fast_pio_out); +static int complete_fast_pio_in(struct kvm_vcpu *vcpu) +{ + unsigned long val; + + /* We should only ever be called with arch.pio.count equal to 1 */ + BUG_ON(vcpu->arch.pio.count != 1); + + /* For size less than 4 we merge, else we zero extend */ + val = (vcpu->arch.pio.size < 4) ? kvm_register_read(vcpu, VCPU_REGS_RAX) + : 0; + + /* + * Since vcpu->arch.pio.count == 1 let emulator_pio_in_emulated perform + * the copy and tracing + */ + emulator_pio_in_emulated(&vcpu->arch.emulate_ctxt, vcpu->arch.pio.size, + vcpu->arch.pio.port, &val, 1); + kvm_register_write(vcpu, VCPU_REGS_RAX, val); + + return 1; +} + +int kvm_fast_pio_in(struct kvm_vcpu *vcpu, int size, unsigned short port) +{ + unsigned long val; + int ret; + + /* For size less than 4 we merge, else we zero extend */ + val = (size < 4) ? kvm_register_read(vcpu, VCPU_REGS_RAX) : 0; + + ret = emulator_pio_in_emulated(&vcpu->arch.emulate_ctxt, size, port, + &val, 1); + if (ret) { + kvm_register_write(vcpu, VCPU_REGS_RAX, val); + return ret; + } + + vcpu->arch.complete_userspace_io = complete_fast_pio_in; + + return 0; +} +EXPORT_SYMBOL_GPL(kvm_fast_pio_in); + static int kvmclock_cpu_down_prep(unsigned int cpu) { __this_cpu_write(cpu_tsc_khz, 0); @@ -5998,8 +6052,12 @@ EXPORT_SYMBOL_GPL(kvm_vcpu_halt); int kvm_emulate_halt(struct kvm_vcpu *vcpu) { - kvm_x86_ops->skip_emulated_instruction(vcpu); - return kvm_vcpu_halt(vcpu); + int ret = kvm_skip_emulated_instruction(vcpu); + /* + * TODO: we might be squashing a GUESTDBG_SINGLESTEP-triggered + * KVM_EXIT_DEBUG here. + */ + return kvm_vcpu_halt(vcpu) && ret; } EXPORT_SYMBOL_GPL(kvm_emulate_halt); @@ -6030,9 +6088,9 @@ void kvm_vcpu_deactivate_apicv(struct kvm_vcpu *vcpu) int kvm_emulate_hypercall(struct kvm_vcpu *vcpu) { unsigned long nr, a0, a1, a2, a3, ret; - int op_64_bit, r = 1; + int op_64_bit, r; - kvm_x86_ops->skip_emulated_instruction(vcpu); + r = kvm_skip_emulated_instruction(vcpu); if (kvm_hv_hypercall_enabled(vcpu->kvm)) return kvm_hv_hypercall(vcpu); diff --git a/arch/x86/pci/xen.c b/arch/x86/pci/xen.c index bedfab98..e1fb269 100644 --- a/arch/x86/pci/xen.c +++ b/arch/x86/pci/xen.c @@ -264,8 +264,8 @@ static int xen_hvm_setup_msi_irqs(struct pci_dev *dev, int nvec, int type) return 0; error: - dev_err(&dev->dev, - "Xen PCI frontend has not registered MSI/MSI-X support!\n"); + dev_err(&dev->dev, "Failed to create MSI%s! ret=%d!\n", + type == PCI_CAP_ID_MSI ? "" : "-X", irq); return irq; } diff --git a/arch/x86/xen/pci-swiotlb-xen.c b/arch/x86/xen/pci-swiotlb-xen.c index 0e98e5d..a9fafb5 100644 --- a/arch/x86/xen/pci-swiotlb-xen.c +++ b/arch/x86/xen/pci-swiotlb-xen.c @@ -19,7 +19,6 @@ int xen_swiotlb __read_mostly; static struct dma_map_ops xen_swiotlb_dma_ops = { - .mapping_error = xen_swiotlb_dma_mapping_error, .alloc = xen_swiotlb_alloc_coherent, .free = xen_swiotlb_free_coherent, .sync_single_for_cpu = xen_swiotlb_sync_single_for_cpu, diff --git a/arch/x86/xen/setup.c b/arch/x86/xen/setup.c index f8960fc..8c394e30 100644 --- a/arch/x86/xen/setup.c +++ b/arch/x86/xen/setup.c @@ -41,7 +41,7 @@ struct xen_memory_region xen_extra_mem[XEN_EXTRA_MEM_MAX_REGIONS] __initdata; unsigned long xen_released_pages; /* E820 map used during setting up memory. */ -static struct e820entry xen_e820_map[E820MAX] __initdata; +static struct e820entry xen_e820_map[E820_X_MAX] __initdata; static u32 xen_e820_map_entries __initdata; /* @@ -750,7 +750,7 @@ char * __init xen_memory_setup(void) max_pfn = min(max_pfn, xen_start_info->nr_pages); mem_end = PFN_PHYS(max_pfn); - memmap.nr_entries = E820MAX; + memmap.nr_entries = ARRAY_SIZE(xen_e820_map); set_xen_guest_handle(memmap.buffer, xen_e820_map); op = xen_initial_domain() ? @@ -923,7 +923,7 @@ char * __init xen_auto_xlated_memory_setup(void) int i; int rc; - memmap.nr_entries = E820MAX; + memmap.nr_entries = ARRAY_SIZE(xen_e820_map); set_xen_guest_handle(memmap.buffer, xen_e820_map); rc = HYPERVISOR_memory_op(XENMEM_memory_map, &memmap); diff --git a/block/blk-core.c b/block/blk-core.c index bd642a4..61ba08c 100644 --- a/block/blk-core.c +++ b/block/blk-core.c @@ -1154,6 +1154,7 @@ static struct request *__get_request(struct request_list *rl, unsigned int op, blk_rq_init(q, rq); blk_rq_set_rl(rq, rl); + blk_rq_set_prio(rq, ioc); rq->cmd_flags = op; rq->rq_flags = rq_flags; @@ -1626,7 +1627,8 @@ void init_request_from_bio(struct request *req, struct bio *bio) req->errors = 0; req->__sector = bio->bi_iter.bi_sector; - req->ioprio = bio_prio(bio); + if (ioprio_valid(bio_prio(bio))) + req->ioprio = bio_prio(bio); blk_rq_bio_prep(req->q, req, bio); } diff --git a/block/blk-mq-cpumap.c b/block/blk-mq-cpumap.c index 19b1d9c..8e61e86 100644 --- a/block/blk-mq-cpumap.c +++ b/block/blk-mq-cpumap.c @@ -87,6 +87,7 @@ int blk_mq_map_queues(struct blk_mq_tag_set *set) free_cpumask_var(cpus); return 0; } +EXPORT_SYMBOL_GPL(blk_mq_map_queues); /* * We have no quick way of doing reverse lookups. This is only used at diff --git a/block/blk-mq.h b/block/blk-mq.h index 3a54dd3..63e9116 100644 --- a/block/blk-mq.h +++ b/block/blk-mq.h @@ -42,7 +42,6 @@ void blk_mq_disable_hotplug(void); /* * CPU -> queue mappings */ -int blk_mq_map_queues(struct blk_mq_tag_set *set); extern int blk_mq_hw_queue_to_node(unsigned int *map, unsigned int); static inline struct blk_mq_hw_ctx *blk_mq_map_queue(struct request_queue *q, diff --git a/block/bsg-lib.c b/block/bsg-lib.c index b2a61e3..9d652a9 100644 --- a/block/bsg-lib.c +++ b/block/bsg-lib.c @@ -32,8 +32,13 @@ * bsg_destroy_job - routine to teardown/delete a bsg job * @job: bsg_job that is to be torn down */ -static void bsg_destroy_job(struct bsg_job *job) +static void bsg_destroy_job(struct kref *kref) { + struct bsg_job *job = container_of(kref, struct bsg_job, kref); + struct request *rq = job->req; + + blk_end_request_all(rq, rq->errors); + put_device(job->dev); /* release reference for the request */ kfree(job->request_payload.sg_list); @@ -41,6 +46,18 @@ static void bsg_destroy_job(struct bsg_job *job) kfree(job); } +void bsg_job_put(struct bsg_job *job) +{ + kref_put(&job->kref, bsg_destroy_job); +} +EXPORT_SYMBOL_GPL(bsg_job_put); + +int bsg_job_get(struct bsg_job *job) +{ + return kref_get_unless_zero(&job->kref); +} +EXPORT_SYMBOL_GPL(bsg_job_get); + /** * bsg_job_done - completion routine for bsg requests * @job: bsg_job that is complete @@ -83,8 +100,7 @@ static void bsg_softirq_done(struct request *rq) { struct bsg_job *job = rq->special; - blk_end_request_all(rq, rq->errors); - bsg_destroy_job(job); + bsg_job_put(job); } static int bsg_map_buffer(struct bsg_buffer *buf, struct request *req) @@ -142,6 +158,7 @@ static int bsg_create_job(struct device *dev, struct request *req) job->dev = dev; /* take a reference for the request */ get_device(job->dev); + kref_init(&job->kref); return 0; failjob_rls_rqst_payload: diff --git a/drivers/ata/ahci.c b/drivers/ata/ahci.c index 74f4c66..2fc5240 100644 --- a/drivers/ata/ahci.c +++ b/drivers/ata/ahci.c @@ -46,6 +46,8 @@ #include <scsi/scsi_host.h> #include <scsi/scsi_cmnd.h> #include <linux/libata.h> +#include <linux/ahci-remap.h> +#include <linux/io-64-nonatomic-lo-hi.h> #include "ahci.h" #define DRV_NAME "ahci" @@ -1400,6 +1402,40 @@ static irqreturn_t ahci_thunderx_irq_handler(int irq, void *dev_instance) } #endif +static void ahci_remap_check(struct pci_dev *pdev, int bar, + struct ahci_host_priv *hpriv) +{ + int i, count = 0; + u32 cap; + + /* + * Check if this device might have remapped nvme devices. + */ + if (pdev->vendor != PCI_VENDOR_ID_INTEL || + pci_resource_len(pdev, bar) < SZ_512K || + bar != AHCI_PCI_BAR_STANDARD || + !(readl(hpriv->mmio + AHCI_VSCAP) & 1)) + return; + + cap = readq(hpriv->mmio + AHCI_REMAP_CAP); + for (i = 0; i < AHCI_MAX_REMAP; i++) { + if ((cap & (1 << i)) == 0) + continue; + if (readl(hpriv->mmio + ahci_remap_dcc(i)) + != PCI_CLASS_STORAGE_EXPRESS) + continue; + + /* We've found a remapped device */ + count++; + } + + if (!count) + return; + + dev_warn(&pdev->dev, "Found %d remapped NVMe devices.\n", count); + dev_warn(&pdev->dev, "Switch your BIOS from RAID to AHCI mode to use them.\n"); +} + static int ahci_get_irq_vector(struct ata_host *host, int port) { return pci_irq_vector(to_pci_dev(host->dev), port); @@ -1541,6 +1577,9 @@ static int ahci_init_one(struct pci_dev *pdev, const struct pci_device_id *ent) hpriv->mmio = pcim_iomap_table(pdev)[ahci_pci_bar]; + /* detect remapped nvme devices */ + ahci_remap_check(pdev, ahci_pci_bar, hpriv); + /* must set flag prior to save config in order to take effect */ if (ahci_broken_devslp(pdev)) hpriv->flags |= AHCI_HFLAG_NO_DEVSLP; diff --git a/drivers/ata/ahci_qoriq.c b/drivers/ata/ahci_qoriq.c index 1eba8df..9884c8c 100644 --- a/drivers/ata/ahci_qoriq.c +++ b/drivers/ata/ahci_qoriq.c @@ -46,11 +46,13 @@ #define LS1021A_AXICC_ADDR 0xC0 #define SATA_ECC_DISABLE 0x00020000 +#define LS1046A_SATA_ECC_DIS 0x80000000 enum ahci_qoriq_type { AHCI_LS1021A, AHCI_LS1043A, AHCI_LS2080A, + AHCI_LS1046A, }; struct ahci_qoriq_priv { @@ -63,6 +65,7 @@ static const struct of_device_id ahci_qoriq_of_match[] = { { .compatible = "fsl,ls1021a-ahci", .data = (void *)AHCI_LS1021A}, { .compatible = "fsl,ls1043a-ahci", .data = (void *)AHCI_LS1043A}, { .compatible = "fsl,ls2080a-ahci", .data = (void *)AHCI_LS2080A}, + { .compatible = "fsl,ls1046a-ahci", .data = (void *)AHCI_LS1046A}, {}, }; MODULE_DEVICE_TABLE(of, ahci_qoriq_of_match); @@ -175,6 +178,13 @@ static int ahci_qoriq_phy_init(struct ahci_host_priv *hpriv) writel(AHCI_PORT_TRANS_CFG, reg_base + PORT_TRANS); writel(AHCI_PORT_AXICC_CFG, reg_base + PORT_AXICC); break; + + case AHCI_LS1046A: + writel(LS1046A_SATA_ECC_DIS, qpriv->ecc_addr); + writel(AHCI_PORT_PHY_1_CFG, reg_base + PORT_PHY1); + writel(AHCI_PORT_TRANS_CFG, reg_base + PORT_TRANS); + writel(AHCI_PORT_AXICC_CFG, reg_base + PORT_AXICC); + break; } return 0; @@ -204,9 +214,9 @@ static int ahci_qoriq_probe(struct platform_device *pdev) qoriq_priv->type = (enum ahci_qoriq_type)of_id->data; - if (qoriq_priv->type == AHCI_LS1021A) { - res = platform_get_resource_byname(pdev, IORESOURCE_MEM, - "sata-ecc"); + res = platform_get_resource_byname(pdev, IORESOURCE_MEM, + "sata-ecc"); + if (res) { qoriq_priv->ecc_addr = devm_ioremap_resource(dev, res); if (IS_ERR(qoriq_priv->ecc_addr)) return PTR_ERR(qoriq_priv->ecc_addr); diff --git a/drivers/ata/libahci.c b/drivers/ata/libahci.c index 0d028ea..ee7db31 100644 --- a/drivers/ata/libahci.c +++ b/drivers/ata/libahci.c @@ -140,6 +140,7 @@ EXPORT_SYMBOL_GPL(ahci_shost_attrs); struct device_attribute *ahci_sdev_attrs[] = { &dev_attr_sw_activity, &dev_attr_unload_heads, + &dev_attr_ncq_prio_enable, NULL }; EXPORT_SYMBOL_GPL(ahci_sdev_attrs); diff --git a/drivers/ata/libata-core.c b/drivers/ata/libata-core.c index 59ce0dd..9cd0a2d 100644 --- a/drivers/ata/libata-core.c +++ b/drivers/ata/libata-core.c @@ -739,6 +739,7 @@ u64 ata_tf_read_block(const struct ata_taskfile *tf, struct ata_device *dev) * @n_block: Number of blocks * @tf_flags: RW/FUA etc... * @tag: tag + * @class: IO priority class * * LOCKING: * None. @@ -753,7 +754,7 @@ u64 ata_tf_read_block(const struct ata_taskfile *tf, struct ata_device *dev) */ int ata_build_rw_tf(struct ata_taskfile *tf, struct ata_device *dev, u64 block, u32 n_block, unsigned int tf_flags, - unsigned int tag) + unsigned int tag, int class) { tf->flags |= ATA_TFLAG_ISADDR | ATA_TFLAG_DEVICE; tf->flags |= tf_flags; @@ -785,6 +786,12 @@ int ata_build_rw_tf(struct ata_taskfile *tf, struct ata_device *dev, tf->device = ATA_LBA; if (tf->flags & ATA_TFLAG_FUA) tf->device |= 1 << 7; + + if (dev->flags & ATA_DFLAG_NCQ_PRIO) { + if (class == IOPRIO_CLASS_RT) + tf->hob_nsect |= ATA_PRIO_HIGH << + ATA_SHIFT_PRIO; + } } else if (dev->flags & ATA_DFLAG_LBA) { tf->flags |= ATA_TFLAG_LBA; @@ -2156,6 +2163,37 @@ static void ata_dev_config_ncq_non_data(struct ata_device *dev) } } +static void ata_dev_config_ncq_prio(struct ata_device *dev) +{ + struct ata_port *ap = dev->link->ap; + unsigned int err_mask; + + if (!(dev->flags & ATA_DFLAG_NCQ_PRIO_ENABLE)) { + dev->flags &= ~ATA_DFLAG_NCQ_PRIO; + return; + } + + err_mask = ata_read_log_page(dev, + ATA_LOG_SATA_ID_DEV_DATA, + ATA_LOG_SATA_SETTINGS, + ap->sector_buf, + 1); + if (err_mask) { + ata_dev_dbg(dev, + "failed to get Identify Device data, Emask 0x%x\n", + err_mask); + return; + } + + if (ap->sector_buf[ATA_LOG_NCQ_PRIO_OFFSET] & BIT(3)) { + dev->flags |= ATA_DFLAG_NCQ_PRIO; + } else { + dev->flags &= ~ATA_DFLAG_NCQ_PRIO; + ata_dev_dbg(dev, "SATA page does not support priority\n"); + } + +} + static int ata_dev_config_ncq(struct ata_device *dev, char *desc, size_t desc_sz) { @@ -2205,6 +2243,8 @@ static int ata_dev_config_ncq(struct ata_device *dev, ata_dev_config_ncq_send_recv(dev); if (ata_id_has_ncq_non_data(dev->id)) ata_dev_config_ncq_non_data(dev); + if (ata_id_has_ncq_prio(dev->id)) + ata_dev_config_ncq_prio(dev); } return 0; diff --git a/drivers/ata/libata-scsi.c b/drivers/ata/libata-scsi.c index 8e575fb..1f863e7 100644 --- a/drivers/ata/libata-scsi.c +++ b/drivers/ata/libata-scsi.c @@ -50,6 +50,7 @@ #include <linux/uaccess.h> #include <linux/suspend.h> #include <asm/unaligned.h> +#include <linux/ioprio.h> #include "libata.h" #include "libata-transport.h" @@ -270,6 +271,83 @@ DEVICE_ATTR(unload_heads, S_IRUGO | S_IWUSR, ata_scsi_park_show, ata_scsi_park_store); EXPORT_SYMBOL_GPL(dev_attr_unload_heads); +static ssize_t ata_ncq_prio_enable_show(struct device *device, + struct device_attribute *attr, + char *buf) +{ + struct scsi_device *sdev = to_scsi_device(device); + struct ata_port *ap; + struct ata_device *dev; + bool ncq_prio_enable; + int rc = 0; + + ap = ata_shost_to_port(sdev->host); + + spin_lock_irq(ap->lock); + dev = ata_scsi_find_dev(ap, sdev); + if (!dev) { + rc = -ENODEV; + goto unlock; + } + + ncq_prio_enable = dev->flags & ATA_DFLAG_NCQ_PRIO_ENABLE; + +unlock: + spin_unlock_irq(ap->lock); + + return rc ? rc : snprintf(buf, 20, "%u\n", ncq_prio_enable); +} + +static ssize_t ata_ncq_prio_enable_store(struct device *device, + struct device_attribute *attr, + const char *buf, size_t len) +{ + struct scsi_device *sdev = to_scsi_device(device); + struct ata_port *ap; + struct ata_device *dev; + long int input; + int rc; + + rc = kstrtol(buf, 10, &input); + if (rc) + return rc; + if ((input < 0) || (input > 1)) + return -EINVAL; + + ap = ata_shost_to_port(sdev->host); + dev = ata_scsi_find_dev(ap, sdev); + if (unlikely(!dev)) + return -ENODEV; + + spin_lock_irq(ap->lock); + if (input) + dev->flags |= ATA_DFLAG_NCQ_PRIO_ENABLE; + else + dev->flags &= ~ATA_DFLAG_NCQ_PRIO_ENABLE; + + dev->link->eh_info.action |= ATA_EH_REVALIDATE; + dev->link->eh_info.flags |= ATA_EHI_QUIET; + ata_port_schedule_eh(ap); + spin_unlock_irq(ap->lock); + + ata_port_wait_eh(ap); + + if (input) { + spin_lock_irq(ap->lock); + if (!(dev->flags & ATA_DFLAG_NCQ_PRIO)) { + dev->flags &= ~ATA_DFLAG_NCQ_PRIO_ENABLE; + rc = -EIO; + } + spin_unlock_irq(ap->lock); + } + + return rc ? rc : len; +} + +DEVICE_ATTR(ncq_prio_enable, S_IRUGO | S_IWUSR, + ata_ncq_prio_enable_show, ata_ncq_prio_enable_store); +EXPORT_SYMBOL_GPL(dev_attr_ncq_prio_enable); + void ata_scsi_set_sense(struct ata_device *dev, struct scsi_cmnd *cmd, u8 sk, u8 asc, u8 ascq) { @@ -401,6 +479,7 @@ EXPORT_SYMBOL_GPL(dev_attr_sw_activity); struct device_attribute *ata_common_sdev_attrs[] = { &dev_attr_unload_heads, + &dev_attr_ncq_prio_enable, NULL }; EXPORT_SYMBOL_GPL(ata_common_sdev_attrs); @@ -1756,6 +1835,8 @@ static unsigned int ata_scsi_rw_xlat(struct ata_queued_cmd *qc) { struct scsi_cmnd *scmd = qc->scsicmd; const u8 *cdb = scmd->cmnd; + struct request *rq = scmd->request; + int class = IOPRIO_PRIO_CLASS(req_get_ioprio(rq)); unsigned int tf_flags = 0; u64 block; u32 n_block; @@ -1822,7 +1903,8 @@ static unsigned int ata_scsi_rw_xlat(struct ata_queued_cmd *qc) qc->nbytes = n_block * scmd->device->sector_size; rc = ata_build_rw_tf(&qc->tf, qc->dev, block, n_block, tf_flags, - qc->tag); + qc->tag, class); + if (likely(rc == 0)) return 0; diff --git a/drivers/ata/libata.h b/drivers/ata/libata.h index 3b301a4..8f3a559 100644 --- a/drivers/ata/libata.h +++ b/drivers/ata/libata.h @@ -66,7 +66,7 @@ extern u64 ata_tf_to_lba48(const struct ata_taskfile *tf); extern struct ata_queued_cmd *ata_qc_new_init(struct ata_device *dev, int tag); extern int ata_build_rw_tf(struct ata_taskfile *tf, struct ata_device *dev, u64 block, u32 n_block, unsigned int tf_flags, - unsigned int tag); + unsigned int tag, int class); extern u64 ata_tf_read_block(const struct ata_taskfile *tf, struct ata_device *dev); extern unsigned ata_exec_internal(struct ata_device *dev, diff --git a/drivers/ata/pata_imx.c b/drivers/ata/pata_imx.c index 139d207..d4caa23 100644 --- a/drivers/ata/pata_imx.c +++ b/drivers/ata/pata_imx.c @@ -11,19 +11,26 @@ * * TODO: * - dmaengine support - * - check if timing stuff needed */ -#include <linux/kernel.h> -#include <linux/module.h> -#include <linux/blkdev.h> -#include <scsi/scsi_host.h> + #include <linux/ata.h> +#include <linux/clk.h> #include <linux/libata.h> +#include <linux/module.h> #include <linux/platform_device.h> -#include <linux/clk.h> #define DRV_NAME "pata_imx" +#define PATA_IMX_ATA_TIME_OFF 0x00 +#define PATA_IMX_ATA_TIME_ON 0x01 +#define PATA_IMX_ATA_TIME_1 0x02 +#define PATA_IMX_ATA_TIME_2W 0x03 +#define PATA_IMX_ATA_TIME_2R 0x04 +#define PATA_IMX_ATA_TIME_AX 0x05 +#define PATA_IMX_ATA_TIME_PIO_RDX 0x06 +#define PATA_IMX_ATA_TIME_4 0x07 +#define PATA_IMX_ATA_TIME_9 0x08 + #define PATA_IMX_ATA_CONTROL 0x24 #define PATA_IMX_ATA_CTRL_FIFO_RST_B (1<<7) #define PATA_IMX_ATA_CTRL_ATA_RST_B (1<<6) @@ -33,6 +40,10 @@ #define PATA_IMX_DRIVE_DATA 0xA0 #define PATA_IMX_DRIVE_CONTROL 0xD8 +static u32 pio_t4[] = { 30, 20, 15, 10, 10 }; +static u32 pio_t9[] = { 20, 15, 10, 10, 10 }; +static u32 pio_tA[] = { 35, 35, 35, 35, 35 }; + struct pata_imx_priv { struct clk *clk; /* timings/interrupt/control regs */ @@ -40,28 +51,49 @@ struct pata_imx_priv { u32 ata_ctl; }; -static int pata_imx_set_mode(struct ata_link *link, struct ata_device **unused) +static void pata_imx_set_timing(struct ata_device *adev, + struct pata_imx_priv *priv) +{ + struct ata_timing timing; + unsigned long clkrate; + u32 T, mode; + + clkrate = clk_get_rate(priv->clk); + + if (adev->pio_mode < XFER_PIO_0 || adev->pio_mode > XFER_PIO_4 || + !clkrate) + return; + + T = 1000000000 / clkrate; + ata_timing_compute(adev, adev->pio_mode, &timing, T * 1000, 0); + + mode = adev->pio_mode - XFER_PIO_0; + + writeb(3, priv->host_regs + PATA_IMX_ATA_TIME_OFF); + writeb(3, priv->host_regs + PATA_IMX_ATA_TIME_ON); + writeb(timing.setup, priv->host_regs + PATA_IMX_ATA_TIME_1); + writeb(timing.act8b, priv->host_regs + PATA_IMX_ATA_TIME_2W); + writeb(timing.act8b, priv->host_regs + PATA_IMX_ATA_TIME_2R); + writeb(1, priv->host_regs + PATA_IMX_ATA_TIME_PIO_RDX); + + writeb(pio_t4[mode] / T + 1, priv->host_regs + PATA_IMX_ATA_TIME_4); + writeb(pio_t9[mode] / T + 1, priv->host_regs + PATA_IMX_ATA_TIME_9); + writeb(pio_tA[mode] / T + 1, priv->host_regs + PATA_IMX_ATA_TIME_AX); +} + +static void pata_imx_set_piomode(struct ata_port *ap, struct ata_device *adev) { - struct ata_device *dev; - struct ata_port *ap = link->ap; struct pata_imx_priv *priv = ap->host->private_data; u32 val; - ata_for_each_dev(dev, link, ENABLED) { - dev->pio_mode = dev->xfer_mode = XFER_PIO_0; - dev->xfer_shift = ATA_SHIFT_PIO; - dev->flags |= ATA_DFLAG_PIO; - - val = __raw_readl(priv->host_regs + PATA_IMX_ATA_CONTROL); - if (ata_pio_need_iordy(dev)) - val |= PATA_IMX_ATA_CTRL_IORDY_EN; - else - val &= ~PATA_IMX_ATA_CTRL_IORDY_EN; - __raw_writel(val, priv->host_regs + PATA_IMX_ATA_CONTROL); + pata_imx_set_timing(adev, priv); - ata_dev_info(dev, "configured for PIO\n"); - } - return 0; + val = __raw_readl(priv->host_regs + PATA_IMX_ATA_CONTROL); + if (ata_pio_need_iordy(adev)) + val |= PATA_IMX_ATA_CTRL_IORDY_EN; + else + val &= ~PATA_IMX_ATA_CTRL_IORDY_EN; + __raw_writel(val, priv->host_regs + PATA_IMX_ATA_CONTROL); } static struct scsi_host_template pata_imx_sht = { @@ -72,7 +104,7 @@ static struct ata_port_operations pata_imx_port_ops = { .inherits = &ata_sff_port_ops, .sff_data_xfer = ata_sff_data_xfer_noirq, .cable_detect = ata_cable_unknown, - .set_mode = pata_imx_set_mode, + .set_piomode = pata_imx_set_piomode, }; static void pata_imx_setup_port(struct ata_ioports *ioaddr) @@ -128,7 +160,7 @@ static int pata_imx_probe(struct platform_device *pdev) ap = host->ports[0]; ap->ops = &pata_imx_port_ops; - ap->pio_mask = ATA_PIO0; + ap->pio_mask = ATA_PIO4; ap->flags |= ATA_FLAG_SLAVE_POSS; io_res = platform_get_resource(pdev, IORESOURCE_MEM, 0); diff --git a/drivers/block/cciss_scsi.c b/drivers/block/cciss_scsi.c index 1537302..a18de9d 100644 --- a/drivers/block/cciss_scsi.c +++ b/drivers/block/cciss_scsi.c @@ -260,43 +260,6 @@ scsi_cmd_stack_free(ctlr_info_t *h) } #if 0 -static int xmargin=8; -static int amargin=60; - -static void -print_bytes (unsigned char *c, int len, int hex, int ascii) -{ - - int i; - unsigned char *x; - - if (hex) - { - x = c; - for (i=0;i<len;i++) - { - if ((i % xmargin) == 0 && i>0) printk("\n"); - if ((i % xmargin) == 0) printk("0x%04x:", i); - printk(" %02x", *x); - x++; - } - printk("\n"); - } - if (ascii) - { - x = c; - for (i=0;i<len;i++) - { - if ((i % amargin) == 0 && i>0) printk("\n"); - if ((i % amargin) == 0) printk("0x%04x:", i); - if (*x > 26 && *x < 128) printk("%c", *x); - else printk("."); - x++; - } - printk("\n"); - } -} - static void print_cmd(CommandList_struct *cp) { @@ -305,30 +268,13 @@ print_cmd(CommandList_struct *cp) printk("sgtot:%d\n", cp->Header.SGTotal); printk("Tag:0x%08x/0x%08x\n", cp->Header.Tag.upper, cp->Header.Tag.lower); - printk("LUN:0x%02x%02x%02x%02x%02x%02x%02x%02x\n", - cp->Header.LUN.LunAddrBytes[0], - cp->Header.LUN.LunAddrBytes[1], - cp->Header.LUN.LunAddrBytes[2], - cp->Header.LUN.LunAddrBytes[3], - cp->Header.LUN.LunAddrBytes[4], - cp->Header.LUN.LunAddrBytes[5], - cp->Header.LUN.LunAddrBytes[6], - cp->Header.LUN.LunAddrBytes[7]); + printk("LUN:0x%8phN\n", cp->Header.LUN.LunAddrBytes); printk("CDBLen:%d\n", cp->Request.CDBLen); printk("Type:%d\n",cp->Request.Type.Type); printk("Attr:%d\n",cp->Request.Type.Attribute); printk(" Dir:%d\n",cp->Request.Type.Direction); printk("Timeout:%d\n",cp->Request.Timeout); - printk( "CDB: %02x %02x %02x %02x %02x %02x %02x %02x" - " %02x %02x %02x %02x %02x %02x %02x %02x\n", - cp->Request.CDB[0], cp->Request.CDB[1], - cp->Request.CDB[2], cp->Request.CDB[3], - cp->Request.CDB[4], cp->Request.CDB[5], - cp->Request.CDB[6], cp->Request.CDB[7], - cp->Request.CDB[8], cp->Request.CDB[9], - cp->Request.CDB[10], cp->Request.CDB[11], - cp->Request.CDB[12], cp->Request.CDB[13], - cp->Request.CDB[14], cp->Request.CDB[15]), + printk("CDB: %16ph\n", cp->Request.CDB); printk("edesc.Addr: 0x%08x/0%08x, Len = %d\n", cp->ErrDesc.Addr.upper, cp->ErrDesc.Addr.lower, cp->ErrDesc.Len); @@ -340,9 +286,7 @@ print_cmd(CommandList_struct *cp) printk("offense size:%d\n", cp->err_info->MoreErrInfo.Invalid_Cmd.offense_size); printk("offense byte:%d\n", cp->err_info->MoreErrInfo.Invalid_Cmd.offense_num); printk("offense value:%d\n", cp->err_info->MoreErrInfo.Invalid_Cmd.offense_value); - } - #endif static int @@ -782,8 +726,10 @@ static void complete_scsi_command(CommandList_struct *c, int timeout, "reported\n", c); break; case CMD_INVALID: { - /* print_bytes(c, sizeof(*c), 1, 0); - print_cmd(c); */ + /* + print_hex_dump(KERN_INFO, "", DUMP_PREFIX_OFFSET, 16, 1, c, sizeof(*c), false); + print_cmd(c); + */ /* We get CMD_INVALID if you address a non-existent tape drive instead of a selection timeout (no response). You will see this if you yank out a tape drive, then try to access it. This is kind of a shame @@ -985,8 +931,10 @@ cciss_scsi_interpret_error(ctlr_info_t *h, CommandList_struct *c) dev_warn(&h->pdev->dev, "%p is reported invalid (probably means " "target device no longer present)\n", c); - /* print_bytes((unsigned char *) c, sizeof(*c), 1, 0); - print_cmd(c); */ + /* + print_hex_dump(KERN_INFO, "", DUMP_PREFIX_OFFSET, 16, 1, c, sizeof(*c), false); + print_cmd(c); + */ } break; case CMD_PROTOCOL_ERR: diff --git a/drivers/block/xen-blkback/xenbus.c b/drivers/block/xen-blkback/xenbus.c index 3cc6d1d..415e79b 100644 --- a/drivers/block/xen-blkback/xenbus.c +++ b/drivers/block/xen-blkback/xenbus.c @@ -533,13 +533,11 @@ static void xen_blkbk_discard(struct xenbus_transaction xbt, struct backend_info struct xenbus_device *dev = be->dev; struct xen_blkif *blkif = be->blkif; int err; - int state = 0, discard_enable; + int state = 0; struct block_device *bdev = be->blkif->vbd.bdev; struct request_queue *q = bdev_get_queue(bdev); - err = xenbus_scanf(XBT_NIL, dev->nodename, "discard-enable", "%d", - &discard_enable); - if (err == 1 && !discard_enable) + if (!xenbus_read_unsigned(dev->nodename, "discard-enable", 1)) return; if (blk_queue_discard(q)) { @@ -1039,30 +1037,24 @@ static int connect_ring(struct backend_info *be) xenbus_dev_fatal(dev, err, "unknown fe protocol %s", protocol); return -ENOSYS; } - err = xenbus_scanf(XBT_NIL, dev->otherend, - "feature-persistent", "%u", &pers_grants); - if (err <= 0) - pers_grants = 0; - + pers_grants = xenbus_read_unsigned(dev->otherend, "feature-persistent", + 0); be->blkif->vbd.feature_gnt_persistent = pers_grants; be->blkif->vbd.overflow_max_grants = 0; /* * Read the number of hardware queues from frontend. */ - err = xenbus_scanf(XBT_NIL, dev->otherend, "multi-queue-num-queues", - "%u", &requested_num_queues); - if (err < 0) { - requested_num_queues = 1; - } else { - if (requested_num_queues > xenblk_max_queues - || requested_num_queues == 0) { - /* Buggy or malicious guest. */ - xenbus_dev_fatal(dev, err, - "guest requested %u queues, exceeding the maximum of %u.", - requested_num_queues, xenblk_max_queues); - return -ENOSYS; - } + requested_num_queues = xenbus_read_unsigned(dev->otherend, + "multi-queue-num-queues", + 1); + if (requested_num_queues > xenblk_max_queues + || requested_num_queues == 0) { + /* Buggy or malicious guest. */ + xenbus_dev_fatal(dev, err, + "guest requested %u queues, exceeding the maximum of %u.", + requested_num_queues, xenblk_max_queues); + return -ENOSYS; } be->blkif->nr_rings = requested_num_queues; if (xen_blkif_alloc_rings(be->blkif)) diff --git a/drivers/block/xen-blkfront.c b/drivers/block/xen-blkfront.c index c000fdf..b2bdfa8 100644 --- a/drivers/block/xen-blkfront.c +++ b/drivers/block/xen-blkfront.c @@ -1758,17 +1758,13 @@ static int talk_to_blkback(struct xenbus_device *dev, const char *message = NULL; struct xenbus_transaction xbt; int err; - unsigned int i, max_page_order = 0; - unsigned int ring_page_order = 0; + unsigned int i, max_page_order; + unsigned int ring_page_order; - err = xenbus_scanf(XBT_NIL, info->xbdev->otherend, - "max-ring-page-order", "%u", &max_page_order); - if (err != 1) - info->nr_ring_pages = 1; - else { - ring_page_order = min(xen_blkif_max_ring_order, max_page_order); - info->nr_ring_pages = 1 << ring_page_order; - } + max_page_order = xenbus_read_unsigned(info->xbdev->otherend, + "max-ring-page-order", 0); + ring_page_order = min(xen_blkif_max_ring_order, max_page_order); + info->nr_ring_pages = 1 << ring_page_order; for (i = 0; i < info->nr_rings; i++) { struct blkfront_ring_info *rinfo = &info->rinfo[i]; @@ -1877,18 +1873,14 @@ again: static int negotiate_mq(struct blkfront_info *info) { - unsigned int backend_max_queues = 0; - int err; + unsigned int backend_max_queues; unsigned int i; BUG_ON(info->nr_rings); /* Check if backend supports multiple queues. */ - err = xenbus_scanf(XBT_NIL, info->xbdev->otherend, - "multi-queue-max-queues", "%u", &backend_max_queues); - if (err < 0) - backend_max_queues = 1; - + backend_max_queues = xenbus_read_unsigned(info->xbdev->otherend, + "multi-queue-max-queues", 1); info->nr_rings = min(backend_max_queues, xen_blkif_max_queues); /* We need at least one ring. */ if (!info->nr_rings) @@ -2196,7 +2188,6 @@ static void blkfront_setup_discard(struct blkfront_info *info) int err; unsigned int discard_granularity; unsigned int discard_alignment; - unsigned int discard_secure; info->feature_discard = 1; err = xenbus_gather(XBT_NIL, info->xbdev->otherend, @@ -2207,10 +2198,9 @@ static void blkfront_setup_discard(struct blkfront_info *info) info->discard_granularity = discard_granularity; info->discard_alignment = discard_alignment; } - err = xenbus_scanf(XBT_NIL, info->xbdev->otherend, - "discard-secure", "%u", &discard_secure); - if (err > 0) - info->feature_secdiscard = !!discard_secure; + info->feature_secdiscard = + !!xenbus_read_unsigned(info->xbdev->otherend, "discard-secure", + 0); } static int blkfront_setup_indirect(struct blkfront_ring_info *rinfo) @@ -2302,16 +2292,11 @@ out_of_memory: */ static void blkfront_gather_backend_features(struct blkfront_info *info) { - int err; - int barrier, flush, discard, persistent; unsigned int indirect_segments; info->feature_flush = 0; info->feature_fua = 0; - err = xenbus_scanf(XBT_NIL, info->xbdev->otherend, - "feature-barrier", "%d", &barrier); - /* * If there's no "feature-barrier" defined, then it means * we're dealing with a very old backend which writes @@ -2319,7 +2304,7 @@ static void blkfront_gather_backend_features(struct blkfront_info *info) * * If there are barriers, then we use flush. */ - if (err > 0 && barrier) { + if (xenbus_read_unsigned(info->xbdev->otherend, "feature-barrier", 0)) { info->feature_flush = 1; info->feature_fua = 1; } @@ -2328,35 +2313,23 @@ static void blkfront_gather_backend_features(struct blkfront_info *info) * And if there is "feature-flush-cache" use that above * barriers. */ - err = xenbus_scanf(XBT_NIL, info->xbdev->otherend, - "feature-flush-cache", "%d", &flush); - - if (err > 0 && flush) { + if (xenbus_read_unsigned(info->xbdev->otherend, "feature-flush-cache", + 0)) { info->feature_flush = 1; info->feature_fua = 0; } - err = xenbus_scanf(XBT_NIL, info->xbdev->otherend, - "feature-discard", "%d", &discard); - - if (err > 0 && discard) + if (xenbus_read_unsigned(info->xbdev->otherend, "feature-discard", 0)) blkfront_setup_discard(info); - err = xenbus_scanf(XBT_NIL, info->xbdev->otherend, - "feature-persistent", "%d", &persistent); - if (err <= 0) - info->feature_persistent = 0; - else - info->feature_persistent = persistent; + info->feature_persistent = + xenbus_read_unsigned(info->xbdev->otherend, + "feature-persistent", 0); - err = xenbus_scanf(XBT_NIL, info->xbdev->otherend, - "feature-max-indirect-segments", "%u", - &indirect_segments); - if (err <= 0) - info->max_indirect_segments = 0; - else - info->max_indirect_segments = min(indirect_segments, - xen_blkif_max_segments); + indirect_segments = xenbus_read_unsigned(info->xbdev->otherend, + "feature-max-indirect-segments", 0); + info->max_indirect_segments = min(indirect_segments, + xen_blkif_max_segments); } /* @@ -2421,11 +2394,9 @@ static void blkfront_connect(struct blkfront_info *info) * provide this. Assume physical sector size to be the same as * sector_size in that case. */ - err = xenbus_scanf(XBT_NIL, info->xbdev->otherend, - "physical-sector-size", "%u", &physical_sector_size); - if (err != 1) - physical_sector_size = sector_size; - + physical_sector_size = xenbus_read_unsigned(info->xbdev->otherend, + "physical-sector-size", + sector_size); blkfront_gather_backend_features(info); for (i = 0; i < info->nr_rings; i++) { err = blkfront_setup_indirect(&info->rinfo[i]); diff --git a/drivers/char/tpm/xen-tpmfront.c b/drivers/char/tpm/xen-tpmfront.c index 62028f4..50072cc 100644 --- a/drivers/char/tpm/xen-tpmfront.c +++ b/drivers/char/tpm/xen-tpmfront.c @@ -337,18 +337,14 @@ static int tpmfront_resume(struct xenbus_device *dev) static void backend_changed(struct xenbus_device *dev, enum xenbus_state backend_state) { - int val; - switch (backend_state) { case XenbusStateInitialised: case XenbusStateConnected: if (dev->state == XenbusStateConnected) break; - if (xenbus_scanf(XBT_NIL, dev->otherend, - "feature-protocol-v2", "%d", &val) < 0) - val = 0; - if (!val) { + if (!xenbus_read_unsigned(dev->otherend, "feature-protocol-v2", + 0)) { xenbus_dev_fatal(dev, -EINVAL, "vTPM protocol 2 required"); return; diff --git a/drivers/firmware/efi/arm-runtime.c b/drivers/firmware/efi/arm-runtime.c index 7c75a8d..349dc3e 100644 --- a/drivers/firmware/efi/arm-runtime.c +++ b/drivers/firmware/efi/arm-runtime.c @@ -39,7 +39,7 @@ static struct mm_struct efi_mm = { .mmlist = LIST_HEAD_INIT(efi_mm.mmlist), }; -#ifdef CONFIG_ARM64_PTDUMP +#ifdef CONFIG_ARM64_PTDUMP_DEBUGFS #include <asm/ptdump.h> static struct ptdump_info efi_ptdump_info = { @@ -53,7 +53,7 @@ static struct ptdump_info efi_ptdump_info = { static int __init ptdump_init(void) { - return ptdump_register(&efi_ptdump_info, "efi_page_tables"); + return ptdump_debugfs_register(&efi_ptdump_info, "efi_page_tables"); } device_initcall(ptdump_init); diff --git a/drivers/hwmon/Kconfig b/drivers/hwmon/Kconfig index 45cef3d..190d270 100644 --- a/drivers/hwmon/Kconfig +++ b/drivers/hwmon/Kconfig @@ -907,6 +907,17 @@ config SENSORS_MCP3021 This driver can also be built as a module. If so, the module will be called mcp3021. +config SENSORS_TC654 + tristate "Microchip TC654/TC655 and compatibles" + depends on I2C + help + If you say yes here you get support for TC654 and TC655. + The TC654 and TC655 are PWM mode fan speed controllers with + FanSense technology for use with brushless DC fans. + + This driver can also be built as a module. If so, the module + will be called tc654. + config SENSORS_MENF21BMC_HWMON tristate "MEN 14F021P00 BMC Hardware Monitoring" depends on MFD_MENF21BMC @@ -1068,8 +1079,8 @@ config SENSORS_LM90 LM86, LM89 and LM99, Analog Devices ADM1032, ADT7461, and ADT7461A, Maxim MAX6646, MAX6647, MAX6648, MAX6649, MAX6657, MAX6658, MAX6659, MAX6680, MAX6681, MAX6692, MAX6695, MAX6696, ON Semiconductor NCT1008, - Winbond/Nuvoton W83L771W/G/AWG/ASG, Philips SA56004, and GMT G781 - sensor chips. + Winbond/Nuvoton W83L771W/G/AWG/ASG, Philips SA56004, GMT G781, and + Texas Instruments TMP451 sensor chips. This driver can also be built as a module. If so, the module will be called lm90. @@ -1591,6 +1602,17 @@ config SENSORS_TMP103 This driver can also be built as a module. If so, the module will be called tmp103. +config SENSORS_TMP108 + tristate "Texas Instruments TMP108" + depends on I2C + select REGMAP_I2C + help + If you say yes here you get support for Texas Instruments TMP108 + sensor chips. + + This driver can also be built as a module. If so, the module + will be called tmp108. + config SENSORS_TMP401 tristate "Texas Instruments TMP401 and compatibles" depends on I2C diff --git a/drivers/hwmon/Makefile b/drivers/hwmon/Makefile index aecf4ba..d2cb7e8 100644 --- a/drivers/hwmon/Makefile +++ b/drivers/hwmon/Makefile @@ -122,6 +122,7 @@ obj-$(CONFIG_SENSORS_MAX6697) += max6697.o obj-$(CONFIG_SENSORS_MAX31790) += max31790.o obj-$(CONFIG_SENSORS_MC13783_ADC)+= mc13783-adc.o obj-$(CONFIG_SENSORS_MCP3021) += mcp3021.o +obj-$(CONFIG_SENSORS_TC654) += tc654.o obj-$(CONFIG_SENSORS_MENF21BMC_HWMON) += menf21bmc_hwmon.o obj-$(CONFIG_SENSORS_NCT6683) += nct6683.o obj-$(CONFIG_SENSORS_NCT6775) += nct6775.o @@ -152,6 +153,7 @@ obj-$(CONFIG_SENSORS_TC74) += tc74.o obj-$(CONFIG_SENSORS_THMC50) += thmc50.o obj-$(CONFIG_SENSORS_TMP102) += tmp102.o obj-$(CONFIG_SENSORS_TMP103) += tmp103.o +obj-$(CONFIG_SENSORS_TMP108) += tmp108.o obj-$(CONFIG_SENSORS_TMP401) += tmp401.o obj-$(CONFIG_SENSORS_TMP421) += tmp421.o obj-$(CONFIG_SENSORS_TWL4030_MADC)+= twl4030-madc-hwmon.o diff --git a/drivers/hwmon/adm1025.c b/drivers/hwmon/adm1025.c index d6c767a..1abb460 100644 --- a/drivers/hwmon/adm1025.c +++ b/drivers/hwmon/adm1025.c @@ -93,7 +93,7 @@ static const int in_scale[6] = { 2500, 2250, 3300, 5000, 12000, 3300 }; #define IN_FROM_REG(reg, scale) (((reg) * (scale) + 96) / 192) #define IN_TO_REG(val, scale) ((val) <= 0 ? 0 : \ - (val) * 192 >= (scale) * 255 ? 255 : \ + (val) >= (scale) * 255 / 192 ? 255 : \ ((val) * 192 + (scale) / 2) / (scale)) #define TEMP_FROM_REG(reg) ((reg) * 1000) diff --git a/drivers/hwmon/adm1026.c b/drivers/hwmon/adm1026.c index e67b9a5..b2a5d9e 100644 --- a/drivers/hwmon/adm1026.c +++ b/drivers/hwmon/adm1026.c @@ -197,8 +197,9 @@ static int adm1026_scaling[] = { /* .001 Volts */ }; #define NEG12_OFFSET 16000 #define SCALE(val, from, to) (((val)*(to) + ((from)/2))/(from)) -#define INS_TO_REG(n, val) (clamp_val(SCALE(val, adm1026_scaling[n], 192),\ - 0, 255)) +#define INS_TO_REG(n, val) \ + SCALE(clamp_val(val, 0, 255 * adm1026_scaling[n] / 192), \ + adm1026_scaling[n], 192) #define INS_FROM_REG(n, val) (SCALE(val, 192, adm1026_scaling[n])) /* @@ -215,11 +216,11 @@ static int adm1026_scaling[] = { /* .001 Volts */ #define DIV_TO_REG(val) ((val) >= 8 ? 3 : (val) >= 4 ? 2 : (val) >= 2 ? 1 : 0) /* Temperature is reported in 1 degC increments */ -#define TEMP_TO_REG(val) (clamp_val(((val) + ((val) < 0 ? -500 : 500)) \ - / 1000, -127, 127)) +#define TEMP_TO_REG(val) DIV_ROUND_CLOSEST(clamp_val(val, -128000, 127000), \ + 1000) #define TEMP_FROM_REG(val) ((val) * 1000) -#define OFFSET_TO_REG(val) (clamp_val(((val) + ((val) < 0 ? -500 : 500)) \ - / 1000, -127, 127)) +#define OFFSET_TO_REG(val) DIV_ROUND_CLOSEST(clamp_val(val, -128000, 127000), \ + 1000) #define OFFSET_FROM_REG(val) ((val) * 1000) #define PWM_TO_REG(val) (clamp_val(val, 0, 255)) @@ -233,7 +234,8 @@ static int adm1026_scaling[] = { /* .001 Volts */ * indicates that the DAC could be used to drive the fans, but in our * example board (Arima HDAMA) it isn't connected to the fans at all. */ -#define DAC_TO_REG(val) (clamp_val(((((val) * 255) + 500) / 2500), 0, 255)) +#define DAC_TO_REG(val) DIV_ROUND_CLOSEST(clamp_val(val, 0, 2500) * 255, \ + 2500) #define DAC_FROM_REG(val) (((val) * 2500) / 255) /* @@ -593,7 +595,10 @@ static ssize_t set_in16_min(struct device *dev, struct device_attribute *attr, return err; mutex_lock(&data->update_lock); - data->in_min[16] = INS_TO_REG(16, val + NEG12_OFFSET); + data->in_min[16] = INS_TO_REG(16, + clamp_val(val, INT_MIN, + INT_MAX - NEG12_OFFSET) + + NEG12_OFFSET); adm1026_write_value(client, ADM1026_REG_IN_MIN[16], data->in_min[16]); mutex_unlock(&data->update_lock); return count; @@ -618,7 +623,10 @@ static ssize_t set_in16_max(struct device *dev, struct device_attribute *attr, return err; mutex_lock(&data->update_lock); - data->in_max[16] = INS_TO_REG(16, val+NEG12_OFFSET); + data->in_max[16] = INS_TO_REG(16, + clamp_val(val, INT_MIN, + INT_MAX - NEG12_OFFSET) + + NEG12_OFFSET); adm1026_write_value(client, ADM1026_REG_IN_MAX[16], data->in_max[16]); mutex_unlock(&data->update_lock); return count; diff --git a/drivers/hwmon/adm9240.c b/drivers/hwmon/adm9240.c index 2fe1828..72bf248 100644 --- a/drivers/hwmon/adm9240.c +++ b/drivers/hwmon/adm9240.c @@ -98,13 +98,15 @@ static inline unsigned int IN_FROM_REG(u8 reg, int n) static inline u8 IN_TO_REG(unsigned long val, int n) { - return clamp_val(SCALE(val, 192, nom_mv[n]), 0, 255); + val = clamp_val(val, 0, nom_mv[n] * 255 / 192); + return SCALE(val, 192, nom_mv[n]); } /* temperature range: -40..125, 127 disables temperature alarm */ static inline s8 TEMP_TO_REG(long val) { - return clamp_val(SCALE(val, 1, 1000), -40, 127); + val = clamp_val(val, -40000, 127000); + return SCALE(val, 1, 1000); } /* two fans, each with low fan speed limit */ @@ -122,7 +124,8 @@ static inline unsigned int FAN_FROM_REG(u8 reg, u8 div) /* analog out 0..1250mV */ static inline u8 AOUT_TO_REG(unsigned long val) { - return clamp_val(SCALE(val, 255, 1250), 0, 255); + val = clamp_val(val, 0, 1250); + return SCALE(val, 255, 1250); } static inline unsigned int AOUT_FROM_REG(u8 reg) diff --git a/drivers/hwmon/adt7411.c b/drivers/hwmon/adt7411.c index 812fbc0..bdeaece 100644 --- a/drivers/hwmon/adt7411.c +++ b/drivers/hwmon/adt7411.c @@ -55,7 +55,7 @@ struct adt7411_data { struct mutex device_lock; /* for "atomic" device accesses */ struct mutex update_lock; unsigned long next_update; - int vref_cached; + long vref_cached; struct i2c_client *client; bool use_ext_temp; }; @@ -114,85 +114,6 @@ static int adt7411_modify_bit(struct i2c_client *client, u8 reg, u8 bit, return ret; } -static ssize_t adt7411_show_vdd(struct device *dev, - struct device_attribute *attr, char *buf) -{ - struct adt7411_data *data = dev_get_drvdata(dev); - struct i2c_client *client = data->client; - int ret = adt7411_read_10_bit(client, ADT7411_REG_INT_TEMP_VDD_LSB, - ADT7411_REG_VDD_MSB, 2); - - return ret < 0 ? ret : sprintf(buf, "%u\n", ret * 7000 / 1024); -} - -static ssize_t adt7411_show_temp(struct device *dev, - struct device_attribute *attr, char *buf) -{ - int nr = to_sensor_dev_attr(attr)->index; - struct adt7411_data *data = dev_get_drvdata(dev); - struct i2c_client *client = data->client; - int val; - struct { - u8 low; - u8 high; - } reg[2] = { - { ADT7411_REG_INT_TEMP_VDD_LSB, ADT7411_REG_INT_TEMP_MSB }, - { ADT7411_REG_EXT_TEMP_AIN14_LSB, - ADT7411_REG_EXT_TEMP_AIN1_MSB }, - }; - - val = adt7411_read_10_bit(client, reg[nr].low, reg[nr].high, 0); - if (val < 0) - return val; - - val = val & 0x200 ? val - 0x400 : val; /* 10 bit signed */ - - return sprintf(buf, "%d\n", val * 250); -} - -static ssize_t adt7411_show_input(struct device *dev, - struct device_attribute *attr, char *buf) -{ - int nr = to_sensor_dev_attr(attr)->index; - struct adt7411_data *data = dev_get_drvdata(dev); - struct i2c_client *client = data->client; - int val; - u8 lsb_reg, lsb_shift; - - mutex_lock(&data->update_lock); - if (time_after_eq(jiffies, data->next_update)) { - val = i2c_smbus_read_byte_data(client, ADT7411_REG_CFG3); - if (val < 0) - goto exit_unlock; - - if (val & ADT7411_CFG3_REF_VDD) { - val = adt7411_read_10_bit(client, - ADT7411_REG_INT_TEMP_VDD_LSB, - ADT7411_REG_VDD_MSB, 2); - if (val < 0) - goto exit_unlock; - - data->vref_cached = val * 7000 / 1024; - } else { - data->vref_cached = 2250; - } - - data->next_update = jiffies + HZ; - } - - lsb_reg = ADT7411_REG_EXT_TEMP_AIN14_LSB + (nr >> 2); - lsb_shift = 2 * (nr & 0x03); - val = adt7411_read_10_bit(client, lsb_reg, - ADT7411_REG_EXT_TEMP_AIN1_MSB + nr, lsb_shift); - if (val < 0) - goto exit_unlock; - - val = sprintf(buf, "%u\n", val * data->vref_cached / 1024); - exit_unlock: - mutex_unlock(&data->update_lock); - return val; -} - static ssize_t adt7411_show_bit(struct device *dev, struct device_attribute *attr, char *buf) { @@ -228,65 +149,157 @@ static ssize_t adt7411_set_bit(struct device *dev, return ret < 0 ? ret : count; } - #define ADT7411_BIT_ATTR(__name, __reg, __bit) \ SENSOR_DEVICE_ATTR_2(__name, S_IRUGO | S_IWUSR, adt7411_show_bit, \ adt7411_set_bit, __bit, __reg) -static SENSOR_DEVICE_ATTR(temp1_input, S_IRUGO, adt7411_show_temp, NULL, 0); -static SENSOR_DEVICE_ATTR(temp2_input, S_IRUGO, adt7411_show_temp, NULL, 1); -static DEVICE_ATTR(in0_input, S_IRUGO, adt7411_show_vdd, NULL); -static SENSOR_DEVICE_ATTR(in1_input, S_IRUGO, adt7411_show_input, NULL, 0); -static SENSOR_DEVICE_ATTR(in2_input, S_IRUGO, adt7411_show_input, NULL, 1); -static SENSOR_DEVICE_ATTR(in3_input, S_IRUGO, adt7411_show_input, NULL, 2); -static SENSOR_DEVICE_ATTR(in4_input, S_IRUGO, adt7411_show_input, NULL, 3); -static SENSOR_DEVICE_ATTR(in5_input, S_IRUGO, adt7411_show_input, NULL, 4); -static SENSOR_DEVICE_ATTR(in6_input, S_IRUGO, adt7411_show_input, NULL, 5); -static SENSOR_DEVICE_ATTR(in7_input, S_IRUGO, adt7411_show_input, NULL, 6); -static SENSOR_DEVICE_ATTR(in8_input, S_IRUGO, adt7411_show_input, NULL, 7); static ADT7411_BIT_ATTR(no_average, ADT7411_REG_CFG2, ADT7411_CFG2_DISABLE_AVG); static ADT7411_BIT_ATTR(fast_sampling, ADT7411_REG_CFG3, ADT7411_CFG3_ADC_CLK_225); static ADT7411_BIT_ATTR(adc_ref_vdd, ADT7411_REG_CFG3, ADT7411_CFG3_REF_VDD); static struct attribute *adt7411_attrs[] = { - &sensor_dev_attr_temp1_input.dev_attr.attr, - &sensor_dev_attr_temp2_input.dev_attr.attr, - &dev_attr_in0_input.attr, - &sensor_dev_attr_in1_input.dev_attr.attr, - &sensor_dev_attr_in2_input.dev_attr.attr, - &sensor_dev_attr_in3_input.dev_attr.attr, - &sensor_dev_attr_in4_input.dev_attr.attr, - &sensor_dev_attr_in5_input.dev_attr.attr, - &sensor_dev_attr_in6_input.dev_attr.attr, - &sensor_dev_attr_in7_input.dev_attr.attr, - &sensor_dev_attr_in8_input.dev_attr.attr, &sensor_dev_attr_no_average.dev_attr.attr, &sensor_dev_attr_fast_sampling.dev_attr.attr, &sensor_dev_attr_adc_ref_vdd.dev_attr.attr, NULL }; +ATTRIBUTE_GROUPS(adt7411); -static umode_t adt7411_attrs_visible(struct kobject *kobj, - struct attribute *attr, int index) +static int adt7411_read_in_vdd(struct device *dev, u32 attr, long *val) { - struct device *dev = container_of(kobj, struct device, kobj); struct adt7411_data *data = dev_get_drvdata(dev); - bool visible = true; + struct i2c_client *client = data->client; + int ret; - if (attr == &sensor_dev_attr_temp2_input.dev_attr.attr) - visible = data->use_ext_temp; - else if (attr == &sensor_dev_attr_in1_input.dev_attr.attr || - attr == &sensor_dev_attr_in2_input.dev_attr.attr) - visible = !data->use_ext_temp; + switch (attr) { + case hwmon_in_input: + ret = adt7411_read_10_bit(client, ADT7411_REG_INT_TEMP_VDD_LSB, + ADT7411_REG_VDD_MSB, 2); + if (ret < 0) + return ret; + *val = ret * 7000 / 1024; + return 0; + default: + return -EOPNOTSUPP; + } +} + +static int adt7411_read_in_chan(struct device *dev, u32 attr, int channel, + long *val) +{ + struct adt7411_data *data = dev_get_drvdata(dev); + struct i2c_client *client = data->client; - return visible ? attr->mode : 0; + int ret; + int lsb_reg, lsb_shift; + int nr = channel - 1; + + mutex_lock(&data->update_lock); + if (time_after_eq(jiffies, data->next_update)) { + ret = i2c_smbus_read_byte_data(client, ADT7411_REG_CFG3); + if (ret < 0) + goto exit_unlock; + + if (ret & ADT7411_CFG3_REF_VDD) { + ret = adt7411_read_in_vdd(dev, hwmon_in_input, + &data->vref_cached); + if (ret < 0) + goto exit_unlock; + } else { + data->vref_cached = 2250; + } + + data->next_update = jiffies + HZ; + } + + switch (attr) { + case hwmon_in_input: + lsb_reg = ADT7411_REG_EXT_TEMP_AIN14_LSB + (nr >> 2); + lsb_shift = 2 * (nr & 0x03); + ret = adt7411_read_10_bit(client, lsb_reg, + ADT7411_REG_EXT_TEMP_AIN1_MSB + nr, + lsb_shift); + if (ret < 0) + goto exit_unlock; + *val = ret * data->vref_cached / 1024; + ret = 0; + break; + default: + ret = -EOPNOTSUPP; + break; + } + exit_unlock: + mutex_unlock(&data->update_lock); + return ret; } -static const struct attribute_group adt7411_group = { - .attrs = adt7411_attrs, - .is_visible = adt7411_attrs_visible, -}; -__ATTRIBUTE_GROUPS(adt7411); +static int adt7411_read_in(struct device *dev, u32 attr, int channel, + long *val) +{ + if (channel == 0) + return adt7411_read_in_vdd(dev, attr, val); + else + return adt7411_read_in_chan(dev, attr, channel, val); +} + +static int adt7411_read_temp(struct device *dev, u32 attr, int channel, + long *val) +{ + struct adt7411_data *data = dev_get_drvdata(dev); + struct i2c_client *client = data->client; + int ret, regl, regh; + + switch (attr) { + case hwmon_temp_input: + regl = channel ? ADT7411_REG_EXT_TEMP_AIN14_LSB : + ADT7411_REG_INT_TEMP_VDD_LSB; + regh = channel ? ADT7411_REG_EXT_TEMP_AIN1_MSB : + ADT7411_REG_INT_TEMP_MSB; + ret = adt7411_read_10_bit(client, regl, regh, 0); + if (ret < 0) + return ret; + ret = ret & 0x200 ? ret - 0x400 : ret; /* 10 bit signed */ + *val = ret * 250; + return 0; + default: + return -EOPNOTSUPP; + } +} + +static int adt7411_read(struct device *dev, enum hwmon_sensor_types type, + u32 attr, int channel, long *val) +{ + switch (type) { + case hwmon_in: + return adt7411_read_in(dev, attr, channel, val); + case hwmon_temp: + return adt7411_read_temp(dev, attr, channel, val); + default: + return -EOPNOTSUPP; + } +} + +static umode_t adt7411_is_visible(const void *_data, + enum hwmon_sensor_types type, + u32 attr, int channel) +{ + const struct adt7411_data *data = _data; + + switch (type) { + case hwmon_in: + if (channel > 0 && channel < 3) + return data->use_ext_temp ? 0 : S_IRUGO; + else + return S_IRUGO; + case hwmon_temp: + if (channel == 1) + return data->use_ext_temp ? S_IRUGO : 0; + else + return S_IRUGO; + default: + return 0; + } +} static int adt7411_detect(struct i2c_client *client, struct i2c_board_info *info) @@ -358,6 +371,51 @@ static int adt7411_init_device(struct adt7411_data *data) return i2c_smbus_write_byte_data(data->client, ADT7411_REG_CFG1, val); } +static const u32 adt7411_in_config[] = { + HWMON_I_INPUT, + HWMON_I_INPUT, + HWMON_I_INPUT, + HWMON_I_INPUT, + HWMON_I_INPUT, + HWMON_I_INPUT, + HWMON_I_INPUT, + HWMON_I_INPUT, + HWMON_I_INPUT, + 0 +}; + +static const struct hwmon_channel_info adt7411_in = { + .type = hwmon_in, + .config = adt7411_in_config, +}; + +static const u32 adt7411_temp_config[] = { + HWMON_T_INPUT, + HWMON_T_INPUT, + 0 +}; + +static const struct hwmon_channel_info adt7411_temp = { + .type = hwmon_temp, + .config = adt7411_temp_config, +}; + +static const struct hwmon_channel_info *adt7411_info[] = { + &adt7411_in, + &adt7411_temp, + NULL +}; + +static const struct hwmon_ops adt7411_hwmon_ops = { + .is_visible = adt7411_is_visible, + .read = adt7411_read, +}; + +static const struct hwmon_chip_info adt7411_chip_info = { + .ops = &adt7411_hwmon_ops, + .info = adt7411_info, +}; + static int adt7411_probe(struct i2c_client *client, const struct i2c_device_id *id) { @@ -382,9 +440,10 @@ static int adt7411_probe(struct i2c_client *client, /* force update on first occasion */ data->next_update = jiffies; - hwmon_dev = devm_hwmon_device_register_with_groups(dev, client->name, - data, - adt7411_groups); + hwmon_dev = devm_hwmon_device_register_with_info(dev, client->name, + data, + &adt7411_chip_info, + adt7411_groups); return PTR_ERR_OR_ZERO(hwmon_dev); } diff --git a/drivers/hwmon/adt7462.c b/drivers/hwmon/adt7462.c index 5929e12..19f2a6d 100644 --- a/drivers/hwmon/adt7462.c +++ b/drivers/hwmon/adt7462.c @@ -810,8 +810,8 @@ static ssize_t set_temp_min(struct device *dev, if (kstrtol(buf, 10, &temp) || !temp_enabled(data, attr->index)) return -EINVAL; + temp = clamp_val(temp, -64000, 191000); temp = DIV_ROUND_CLOSEST(temp, 1000) + 64; - temp = clamp_val(temp, 0, 255); mutex_lock(&data->lock); data->temp_min[attr->index] = temp; @@ -848,8 +848,8 @@ static ssize_t set_temp_max(struct device *dev, if (kstrtol(buf, 10, &temp) || !temp_enabled(data, attr->index)) return -EINVAL; + temp = clamp_val(temp, -64000, 191000); temp = DIV_ROUND_CLOSEST(temp, 1000) + 64; - temp = clamp_val(temp, 0, 255); mutex_lock(&data->lock); data->temp_max[attr->index] = temp; @@ -912,9 +912,9 @@ static ssize_t set_volt_max(struct device *dev, if (kstrtol(buf, 10, &temp) || !x) return -EINVAL; + temp = clamp_val(temp, 0, 255 * x / 1000); temp *= 1000; /* convert mV to uV */ temp = DIV_ROUND_CLOSEST(temp, x); - temp = clamp_val(temp, 0, 255); mutex_lock(&data->lock); data->volt_max[attr->index] = temp; @@ -954,9 +954,9 @@ static ssize_t set_volt_min(struct device *dev, if (kstrtol(buf, 10, &temp) || !x) return -EINVAL; + temp = clamp_val(temp, 0, 255 * x / 1000); temp *= 1000; /* convert mV to uV */ temp = DIV_ROUND_CLOSEST(temp, x); - temp = clamp_val(temp, 0, 255); mutex_lock(&data->lock); data->volt_min[attr->index] = temp; @@ -1220,8 +1220,8 @@ static ssize_t set_pwm_hyst(struct device *dev, if (kstrtol(buf, 10, &temp)) return -EINVAL; + temp = clamp_val(temp, 0, 15000); temp = DIV_ROUND_CLOSEST(temp, 1000); - temp = clamp_val(temp, 0, 15); /* package things up */ temp &= ADT7462_PWM_HYST_MASK; @@ -1306,8 +1306,8 @@ static ssize_t set_pwm_tmin(struct device *dev, if (kstrtol(buf, 10, &temp)) return -EINVAL; + temp = clamp_val(temp, -64000, 191000); temp = DIV_ROUND_CLOSEST(temp, 1000) + 64; - temp = clamp_val(temp, 0, 255); mutex_lock(&data->lock); data->pwm_tmin[attr->index] = temp; diff --git a/drivers/hwmon/adt7470.c b/drivers/hwmon/adt7470.c index 6e60ca5..c9a1d9c 100644 --- a/drivers/hwmon/adt7470.c +++ b/drivers/hwmon/adt7470.c @@ -483,8 +483,8 @@ static ssize_t set_temp_min(struct device *dev, if (kstrtol(buf, 10, &temp)) return -EINVAL; + temp = clamp_val(temp, -128000, 127000); temp = DIV_ROUND_CLOSEST(temp, 1000); - temp = clamp_val(temp, -128, 127); mutex_lock(&data->lock); data->temp_min[attr->index] = temp; @@ -517,8 +517,8 @@ static ssize_t set_temp_max(struct device *dev, if (kstrtol(buf, 10, &temp)) return -EINVAL; + temp = clamp_val(temp, -128000, 127000); temp = DIV_ROUND_CLOSEST(temp, 1000); - temp = clamp_val(temp, -128, 127); mutex_lock(&data->lock); data->temp_max[attr->index] = temp; @@ -880,8 +880,8 @@ static ssize_t set_pwm_tmin(struct device *dev, if (kstrtol(buf, 10, &temp)) return -EINVAL; + temp = clamp_val(temp, -128000, 127000); temp = DIV_ROUND_CLOSEST(temp, 1000); - temp = clamp_val(temp, -128, 127); mutex_lock(&data->lock); data->pwm_tmin[attr->index] = temp; diff --git a/drivers/hwmon/amc6821.c b/drivers/hwmon/amc6821.c index 12e851a..46b4e35 100644 --- a/drivers/hwmon/amc6821.c +++ b/drivers/hwmon/amc6821.c @@ -188,8 +188,8 @@ static struct amc6821_data *amc6821_update_device(struct device *dev) !data->valid) { for (i = 0; i < TEMP_IDX_LEN; i++) - data->temp[i] = i2c_smbus_read_byte_data(client, - temp_reg[i]); + data->temp[i] = (int8_t)i2c_smbus_read_byte_data( + client, temp_reg[i]); data->stat1 = i2c_smbus_read_byte_data(client, AMC6821_REG_STAT1); diff --git a/drivers/hwmon/coretemp.c b/drivers/hwmon/coretemp.c index 6a27eb2..3ac4c03 100644 --- a/drivers/hwmon/coretemp.c +++ b/drivers/hwmon/coretemp.c @@ -51,6 +51,7 @@ static int force_tjmax; module_param_named(tjmax, force_tjmax, int, 0444); MODULE_PARM_DESC(tjmax, "TjMax value in degrees Celsius"); +#define PKG_SYSFS_ATTR_NO 1 /* Sysfs attribute for package temp */ #define BASE_SYSFS_ATTR_NO 2 /* Sysfs Base attr no for coretemp */ #define NUM_REAL_CORES 128 /* Number of Real cores per cpu */ #define CORETEMP_NAME_LENGTH 19 /* String Length of attrs */ @@ -58,7 +59,6 @@ MODULE_PARM_DESC(tjmax, "TjMax value in degrees Celsius"); #define TOTAL_ATTRS (MAX_CORE_ATTRS + 1) #define MAX_CORE_DATA (NUM_REAL_CORES + BASE_SYSFS_ATTR_NO) -#define TO_PHYS_ID(cpu) (cpu_data(cpu).phys_proc_id) #define TO_CORE_ID(cpu) (cpu_data(cpu).cpu_core_id) #define TO_ATTR_NO(cpu) (TO_CORE_ID(cpu) + BASE_SYSFS_ATTR_NO) @@ -102,20 +102,17 @@ struct temp_data { /* Platform Data per Physical CPU */ struct platform_data { - struct device *hwmon_dev; - u16 phys_proc_id; - struct temp_data *core_data[MAX_CORE_DATA]; + struct device *hwmon_dev; + u16 pkg_id; + struct cpumask cpumask; + struct temp_data *core_data[MAX_CORE_DATA]; struct device_attribute name_attr; }; -struct pdev_entry { - struct list_head list; - struct platform_device *pdev; - u16 phys_proc_id; -}; - -static LIST_HEAD(pdev_list); -static DEFINE_MUTEX(pdev_list_mutex); +/* Keep track of how many package pointers we allocated in init() */ +static int max_packages __read_mostly; +/* Array of package pointers. Serialized by cpu hotplug lock */ +static struct platform_device **pkg_devices; static ssize_t show_label(struct device *dev, struct device_attribute *devattr, char *buf) @@ -125,7 +122,7 @@ static ssize_t show_label(struct device *dev, struct temp_data *tdata = pdata->core_data[attr->index]; if (tdata->is_pkg_data) - return sprintf(buf, "Physical id %u\n", pdata->phys_proc_id); + return sprintf(buf, "Package id %u\n", pdata->pkg_id); return sprintf(buf, "Core %u\n", tdata->cpu_core_id); } @@ -138,7 +135,9 @@ static ssize_t show_crit_alarm(struct device *dev, struct platform_data *pdata = dev_get_drvdata(dev); struct temp_data *tdata = pdata->core_data[attr->index]; + mutex_lock(&tdata->update_lock); rdmsr_on_cpu(tdata->cpu, tdata->status_reg, &eax, &edx); + mutex_unlock(&tdata->update_lock); return sprintf(buf, "%d\n", (eax >> 5) & 1); } @@ -435,18 +434,10 @@ static int chk_ucode_version(unsigned int cpu) static struct platform_device *coretemp_get_pdev(unsigned int cpu) { - u16 phys_proc_id = TO_PHYS_ID(cpu); - struct pdev_entry *p; - - mutex_lock(&pdev_list_mutex); + int pkgid = topology_logical_package_id(cpu); - list_for_each_entry(p, &pdev_list, list) - if (p->phys_proc_id == phys_proc_id) { - mutex_unlock(&pdev_list_mutex); - return p->pdev; - } - - mutex_unlock(&pdev_list_mutex); + if (pkgid >= 0 && pkgid < max_packages) + return pkg_devices[pkgid]; return NULL; } @@ -483,21 +474,11 @@ static int create_core_data(struct platform_device *pdev, unsigned int cpu, * The attr number is always core id + 2 * The Pkgtemp will always show up as temp1_*, if available */ - attr_no = pkg_flag ? 1 : TO_ATTR_NO(cpu); + attr_no = pkg_flag ? PKG_SYSFS_ATTR_NO : TO_ATTR_NO(cpu); if (attr_no > MAX_CORE_DATA - 1) return -ERANGE; - /* - * Provide a single set of attributes for all HT siblings of a core - * to avoid duplicate sensors (the processor ID and core ID of all - * HT siblings of a core are the same). - * Skip if a HT sibling of this core is already registered. - * This is not an error. - */ - if (pdata->core_data[attr_no] != NULL) - return 0; - tdata = init_temp_data(cpu, pkg_flag); if (!tdata) return -ENOMEM; @@ -539,21 +520,14 @@ exit_free: return err; } -static void coretemp_add_core(unsigned int cpu, int pkg_flag) +static void +coretemp_add_core(struct platform_device *pdev, unsigned int cpu, int pkg_flag) { - struct platform_device *pdev = coretemp_get_pdev(cpu); - int err; - - if (!pdev) - return; - - err = create_core_data(pdev, cpu, pkg_flag); - if (err) + if (create_core_data(pdev, cpu, pkg_flag)) dev_err(&pdev->dev, "Adding Core %u failed\n", cpu); } -static void coretemp_remove_core(struct platform_data *pdata, - int indx) +static void coretemp_remove_core(struct platform_data *pdata, int indx) { struct temp_data *tdata = pdata->core_data[indx]; @@ -574,7 +548,7 @@ static int coretemp_probe(struct platform_device *pdev) if (!pdata) return -ENOMEM; - pdata->phys_proc_id = pdev->id; + pdata->pkg_id = pdev->id; platform_set_drvdata(pdev, pdata); pdata->hwmon_dev = devm_hwmon_device_register_with_groups(dev, DRVNAME, @@ -602,85 +576,33 @@ static struct platform_driver coretemp_driver = { .remove = coretemp_remove, }; -static int coretemp_device_add(unsigned int cpu) +static struct platform_device *coretemp_device_add(unsigned int cpu) { - int err; + int err, pkgid = topology_logical_package_id(cpu); struct platform_device *pdev; - struct pdev_entry *pdev_entry; - mutex_lock(&pdev_list_mutex); + if (pkgid < 0) + return ERR_PTR(-ENOMEM); - pdev = platform_device_alloc(DRVNAME, TO_PHYS_ID(cpu)); - if (!pdev) { - err = -ENOMEM; - pr_err("Device allocation failed\n"); - goto exit; - } - - pdev_entry = kzalloc(sizeof(struct pdev_entry), GFP_KERNEL); - if (!pdev_entry) { - err = -ENOMEM; - goto exit_device_put; - } + pdev = platform_device_alloc(DRVNAME, pkgid); + if (!pdev) + return ERR_PTR(-ENOMEM); err = platform_device_add(pdev); if (err) { - pr_err("Device addition failed (%d)\n", err); - goto exit_device_free; - } - - pdev_entry->pdev = pdev; - pdev_entry->phys_proc_id = pdev->id; - - list_add_tail(&pdev_entry->list, &pdev_list); - mutex_unlock(&pdev_list_mutex); - - return 0; - -exit_device_free: - kfree(pdev_entry); -exit_device_put: - platform_device_put(pdev); -exit: - mutex_unlock(&pdev_list_mutex); - return err; -} - -static void coretemp_device_remove(unsigned int cpu) -{ - struct pdev_entry *p, *n; - u16 phys_proc_id = TO_PHYS_ID(cpu); - - mutex_lock(&pdev_list_mutex); - list_for_each_entry_safe(p, n, &pdev_list, list) { - if (p->phys_proc_id != phys_proc_id) - continue; - platform_device_unregister(p->pdev); - list_del(&p->list); - kfree(p); + platform_device_put(pdev); + return ERR_PTR(err); } - mutex_unlock(&pdev_list_mutex); -} - -static bool is_any_core_online(struct platform_data *pdata) -{ - int i; - /* Find online cores, except pkgtemp data */ - for (i = MAX_CORE_DATA - 1; i >= 0; --i) { - if (pdata->core_data[i] && - !pdata->core_data[i]->is_pkg_data) { - return true; - } - } - return false; + pkg_devices[pkgid] = pdev; + return pdev; } -static void get_core_online(unsigned int cpu) +static int coretemp_cpu_online(unsigned int cpu) { - struct cpuinfo_x86 *c = &cpu_data(cpu); struct platform_device *pdev = coretemp_get_pdev(cpu); - int err; + struct cpuinfo_x86 *c = &cpu_data(cpu); + struct platform_data *pdata; /* * CPUID.06H.EAX[0] indicates whether the CPU has thermal @@ -688,12 +610,12 @@ static void get_core_online(unsigned int cpu) * without thermal sensors will be filtered out. */ if (!cpu_has(c, X86_FEATURE_DTHERM)) - return; + return -ENODEV; if (!pdev) { /* Check the microcode version of the CPU */ if (chk_ucode_version(cpu)) - return; + return -EINVAL; /* * Alright, we have DTS support. @@ -701,101 +623,100 @@ static void get_core_online(unsigned int cpu) * online. So, initialize per-pkg data structures and * then bring this core online. */ - err = coretemp_device_add(cpu); - if (err) - return; + pdev = coretemp_device_add(cpu); + if (IS_ERR(pdev)) + return PTR_ERR(pdev); + /* * Check whether pkgtemp support is available. * If so, add interfaces for pkgtemp. */ if (cpu_has(c, X86_FEATURE_PTS)) - coretemp_add_core(cpu, 1); + coretemp_add_core(pdev, cpu, 1); } + + pdata = platform_get_drvdata(pdev); /* - * Physical CPU device already exists. - * So, just add interfaces for this core. + * Check whether a thread sibling is already online. If not add the + * interface for this CPU core. */ - coretemp_add_core(cpu, 0); + if (!cpumask_intersects(&pdata->cpumask, topology_sibling_cpumask(cpu))) + coretemp_add_core(pdev, cpu, 0); + + cpumask_set_cpu(cpu, &pdata->cpumask); + return 0; } -static void put_core_offline(unsigned int cpu) +static int coretemp_cpu_offline(unsigned int cpu) { - int i, indx; - struct platform_data *pdata; struct platform_device *pdev = coretemp_get_pdev(cpu); + struct platform_data *pd; + struct temp_data *tdata; + int indx, target; /* If the physical CPU device does not exist, just return */ if (!pdev) - return; - - pdata = platform_get_drvdata(pdev); - - indx = TO_ATTR_NO(cpu); + return 0; /* The core id is too big, just return */ + indx = TO_ATTR_NO(cpu); if (indx > MAX_CORE_DATA - 1) - return; + return 0; - if (pdata->core_data[indx] && pdata->core_data[indx]->cpu == cpu) - coretemp_remove_core(pdata, indx); + pd = platform_get_drvdata(pdev); + tdata = pd->core_data[indx]; + + cpumask_clear_cpu(cpu, &pd->cpumask); /* - * If a HT sibling of a core is taken offline, but another HT sibling - * of the same core is still online, register the alternate sibling. - * This ensures that exactly one set of attributes is provided as long - * as at least one HT sibling of a core is online. + * If this is the last thread sibling, remove the CPU core + * interface, If there is still a sibling online, transfer the + * target cpu of that core interface to it. */ - for_each_sibling(i, cpu) { - if (i != cpu) { - get_core_online(i); - /* - * Display temperature sensor data for one HT sibling - * per core only, so abort the loop after one such - * sibling has been found. - */ - break; - } + target = cpumask_any_and(&pd->cpumask, topology_sibling_cpumask(cpu)); + if (target >= nr_cpu_ids) { + coretemp_remove_core(pd, indx); + } else if (tdata && tdata->cpu == cpu) { + mutex_lock(&tdata->update_lock); + tdata->cpu = target; + mutex_unlock(&tdata->update_lock); } + /* - * If all cores in this pkg are offline, remove the device. - * coretemp_device_remove calls unregister_platform_device, - * which in turn calls coretemp_remove. This removes the - * pkgtemp entry and does other clean ups. + * If all cores in this pkg are offline, remove the device. This + * will invoke the platform driver remove function, which cleans up + * the rest. */ - if (!is_any_core_online(pdata)) - coretemp_device_remove(cpu); -} + if (cpumask_empty(&pd->cpumask)) { + pkg_devices[topology_logical_package_id(cpu)] = NULL; + platform_device_unregister(pdev); + return 0; + } -static int coretemp_cpu_callback(struct notifier_block *nfb, - unsigned long action, void *hcpu) -{ - unsigned int cpu = (unsigned long) hcpu; - - switch (action) { - case CPU_ONLINE: - case CPU_DOWN_FAILED: - get_core_online(cpu); - break; - case CPU_DOWN_PREPARE: - put_core_offline(cpu); - break; + /* + * Check whether this core is the target for the package + * interface. We need to assign it to some other cpu. + */ + tdata = pd->core_data[PKG_SYSFS_ATTR_NO]; + if (tdata && tdata->cpu == cpu) { + target = cpumask_first(&pd->cpumask); + mutex_lock(&tdata->update_lock); + tdata->cpu = target; + mutex_unlock(&tdata->update_lock); } - return NOTIFY_OK; + return 0; } - -static struct notifier_block coretemp_cpu_notifier __refdata = { - .notifier_call = coretemp_cpu_callback, -}; - static const struct x86_cpu_id __initconst coretemp_ids[] = { { X86_VENDOR_INTEL, X86_FAMILY_ANY, X86_MODEL_ANY, X86_FEATURE_DTHERM }, {} }; MODULE_DEVICE_TABLE(x86cpu, coretemp_ids); +static enum cpuhp_state coretemp_hp_online; + static int __init coretemp_init(void) { - int i, err; + int err; /* * CPUID.06H.EAX[0] indicates whether the CPU has thermal @@ -805,54 +726,38 @@ static int __init coretemp_init(void) if (!x86_match_cpu(coretemp_ids)) return -ENODEV; + max_packages = topology_max_packages(); + pkg_devices = kzalloc(max_packages * sizeof(struct platform_device *), + GFP_KERNEL); + if (!pkg_devices) + return -ENOMEM; + err = platform_driver_register(&coretemp_driver); if (err) - goto exit; + return err; - cpu_notifier_register_begin(); - for_each_online_cpu(i) - get_core_online(i); - -#ifndef CONFIG_HOTPLUG_CPU - if (list_empty(&pdev_list)) { - cpu_notifier_register_done(); - err = -ENODEV; - goto exit_driver_unreg; - } -#endif - - __register_hotcpu_notifier(&coretemp_cpu_notifier); - cpu_notifier_register_done(); + err = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "hwmon/coretemp:online", + coretemp_cpu_online, coretemp_cpu_offline); + if (err < 0) + goto outdrv; + coretemp_hp_online = err; return 0; -#ifndef CONFIG_HOTPLUG_CPU -exit_driver_unreg: +outdrv: platform_driver_unregister(&coretemp_driver); -#endif -exit: + kfree(pkg_devices); return err; } +module_init(coretemp_init) static void __exit coretemp_exit(void) { - struct pdev_entry *p, *n; - - cpu_notifier_register_begin(); - __unregister_hotcpu_notifier(&coretemp_cpu_notifier); - mutex_lock(&pdev_list_mutex); - list_for_each_entry_safe(p, n, &pdev_list, list) { - platform_device_unregister(p->pdev); - list_del(&p->list); - kfree(p); - } - mutex_unlock(&pdev_list_mutex); - cpu_notifier_register_done(); + cpuhp_remove_state(coretemp_hp_online); platform_driver_unregister(&coretemp_driver); + kfree(pkg_devices); } +module_exit(coretemp_exit) MODULE_AUTHOR("Rudolf Marek <r.marek@assembler.cz>"); MODULE_DESCRIPTION("Intel Core temperature monitor"); MODULE_LICENSE("GPL"); - -module_init(coretemp_init) -module_exit(coretemp_exit) diff --git a/drivers/hwmon/ds620.c b/drivers/hwmon/ds620.c index edf550f..0043a4c 100644 --- a/drivers/hwmon/ds620.c +++ b/drivers/hwmon/ds620.c @@ -166,7 +166,7 @@ static ssize_t set_temp(struct device *dev, struct device_attribute *da, if (res) return res; - val = (val * 10 / 625) * 8; + val = (clamp_val(val, -128000, 128000) * 10 / 625) * 8; mutex_lock(&data->update_lock); data->temp[attr->index] = val; diff --git a/drivers/hwmon/emc2103.c b/drivers/hwmon/emc2103.c index 24e395c..4b870ee 100644 --- a/drivers/hwmon/emc2103.c +++ b/drivers/hwmon/emc2103.c @@ -251,7 +251,7 @@ static ssize_t set_temp_min(struct device *dev, struct device_attribute *da, if (result < 0) return result; - val = clamp_val(DIV_ROUND_CLOSEST(val, 1000), -63, 127); + val = DIV_ROUND_CLOSEST(clamp_val(val, -63000, 127000), 1000); mutex_lock(&data->update_lock); data->temp_min[nr] = val; @@ -273,7 +273,7 @@ static ssize_t set_temp_max(struct device *dev, struct device_attribute *da, if (result < 0) return result; - val = clamp_val(DIV_ROUND_CLOSEST(val, 1000), -63, 127); + val = DIV_ROUND_CLOSEST(clamp_val(val, -63000, 127000), 1000); mutex_lock(&data->update_lock); data->temp_max[nr] = val; diff --git a/drivers/hwmon/emc6w201.c b/drivers/hwmon/emc6w201.c index f37fe201..4aee5ad 100644 --- a/drivers/hwmon/emc6w201.c +++ b/drivers/hwmon/emc6w201.c @@ -215,12 +215,13 @@ static ssize_t set_in(struct device *dev, struct device_attribute *devattr, if (err < 0) return err; - val = DIV_ROUND_CLOSEST(val * 0xC0, nominal_mv[nr]); + val = clamp_val(val, 0, 255 * nominal_mv[nr] / 192); + val = DIV_ROUND_CLOSEST(val * 192, nominal_mv[nr]); reg = (sf == min) ? EMC6W201_REG_IN_LOW(nr) : EMC6W201_REG_IN_HIGH(nr); mutex_lock(&data->update_lock); - data->in[sf][nr] = clamp_val(val, 0, 255); + data->in[sf][nr] = val; err = emc6w201_write8(client, reg, data->in[sf][nr]); mutex_unlock(&data->update_lock); @@ -252,12 +253,13 @@ static ssize_t set_temp(struct device *dev, struct device_attribute *devattr, if (err < 0) return err; + val = clamp_val(val, -127000, 127000); val = DIV_ROUND_CLOSEST(val, 1000); reg = (sf == min) ? EMC6W201_REG_TEMP_LOW(nr) : EMC6W201_REG_TEMP_HIGH(nr); mutex_lock(&data->update_lock); - data->temp[sf][nr] = clamp_val(val, -127, 127); + data->temp[sf][nr] = val; err = emc6w201_write8(client, reg, data->temp[sf][nr]); mutex_unlock(&data->update_lock); diff --git a/drivers/hwmon/g762.c b/drivers/hwmon/g762.c index b96a2a9..628be9c 100644 --- a/drivers/hwmon/g762.c +++ b/drivers/hwmon/g762.c @@ -193,14 +193,17 @@ static inline unsigned int rpm_from_cnt(u8 cnt, u32 clk_freq, u16 p, * Convert fan RPM value from sysfs into count value for fan controller * register (FAN_SET_CNT). */ -static inline unsigned char cnt_from_rpm(u32 rpm, u32 clk_freq, u16 p, +static inline unsigned char cnt_from_rpm(unsigned long rpm, u32 clk_freq, u16 p, u8 clk_div, u8 gear_mult) { - if (!rpm) /* to stop the fan, set cnt to 255 */ + unsigned long f1 = clk_freq * 30 * gear_mult; + unsigned long f2 = p * clk_div; + + if (!rpm) /* to stop the fan, set cnt to 255 */ return 0xff; - return clamp_val(((clk_freq * 30 * gear_mult) / (rpm * p * clk_div)), - 0, 255); + rpm = clamp_val(rpm, f1 / (255 * f2), ULONG_MAX / f2); + return DIV_ROUND_CLOSEST(f1, rpm * f2); } /* helper to grab and cache data, at most one time per second */ diff --git a/drivers/hwmon/hwmon.c b/drivers/hwmon/hwmon.c index a74c075..3932f92 100644 --- a/drivers/hwmon/hwmon.c +++ b/drivers/hwmon/hwmon.c @@ -38,12 +38,15 @@ struct hwmon_device { #define to_hwmon_device(d) container_of(d, struct hwmon_device, dev) +#define MAX_SYSFS_ATTR_NAME_LENGTH 32 + struct hwmon_device_attribute { struct device_attribute dev_attr; const struct hwmon_ops *ops; enum hwmon_sensor_types type; u32 attr; int index; + char name[MAX_SYSFS_ATTR_NAME_LENGTH]; }; #define to_hwmon_attr(d) \ @@ -178,6 +181,22 @@ static ssize_t hwmon_attr_show(struct device *dev, return sprintf(buf, "%ld\n", val); } +static ssize_t hwmon_attr_show_string(struct device *dev, + struct device_attribute *devattr, + char *buf) +{ + struct hwmon_device_attribute *hattr = to_hwmon_attr(devattr); + char *s; + int ret; + + ret = hattr->ops->read_string(dev, hattr->type, hattr->attr, + hattr->index, &s); + if (ret < 0) + return ret; + + return sprintf(buf, "%s\n", s); +} + static ssize_t hwmon_attr_store(struct device *dev, struct device_attribute *devattr, const char *buf, size_t count) @@ -205,6 +224,17 @@ static int hwmon_attr_base(enum hwmon_sensor_types type) return 1; } +static bool is_string_attr(enum hwmon_sensor_types type, u32 attr) +{ + return (type == hwmon_temp && attr == hwmon_temp_label) || + (type == hwmon_in && attr == hwmon_in_label) || + (type == hwmon_curr && attr == hwmon_curr_label) || + (type == hwmon_power && attr == hwmon_power_label) || + (type == hwmon_energy && attr == hwmon_energy_label) || + (type == hwmon_humidity && attr == hwmon_humidity_label) || + (type == hwmon_fan && attr == hwmon_fan_label); +} + static struct attribute *hwmon_genattr(struct device *dev, const void *drvdata, enum hwmon_sensor_types type, @@ -218,6 +248,7 @@ static struct attribute *hwmon_genattr(struct device *dev, struct attribute *a; umode_t mode; char *name; + bool is_string = is_string_attr(type, attr); /* The attribute is invisible if there is no template string */ if (!template) @@ -227,32 +258,31 @@ static struct attribute *hwmon_genattr(struct device *dev, if (!mode) return ERR_PTR(-ENOENT); - if ((mode & S_IRUGO) && !ops->read) + if ((mode & S_IRUGO) && ((is_string && !ops->read_string) || + (!is_string && !ops->read))) return ERR_PTR(-EINVAL); if ((mode & S_IWUGO) && !ops->write) return ERR_PTR(-EINVAL); + hattr = devm_kzalloc(dev, sizeof(*hattr), GFP_KERNEL); + if (!hattr) + return ERR_PTR(-ENOMEM); + if (type == hwmon_chip) { name = (char *)template; } else { - name = devm_kzalloc(dev, strlen(template) + 16, GFP_KERNEL); - if (!name) - return ERR_PTR(-ENOMEM); - scnprintf(name, strlen(template) + 16, template, + scnprintf(hattr->name, sizeof(hattr->name), template, index + hwmon_attr_base(type)); + name = hattr->name; } - hattr = devm_kzalloc(dev, sizeof(*hattr), GFP_KERNEL); - if (!hattr) - return ERR_PTR(-ENOMEM); - hattr->type = type; hattr->attr = attr; hattr->index = index; hattr->ops = ops; dattr = &hattr->dev_attr; - dattr->show = hwmon_attr_show; + dattr->show = is_string ? hwmon_attr_show_string : hwmon_attr_show; dattr->store = hwmon_attr_store; a = &dattr->attr; @@ -263,7 +293,11 @@ static struct attribute *hwmon_genattr(struct device *dev, return a; } -static const char * const hwmon_chip_attr_templates[] = { +/* + * Chip attributes are not attribute templates but actual sysfs attributes. + * See hwmon_genattr() for special handling. + */ +static const char * const hwmon_chip_attrs[] = { [hwmon_chip_temp_reset_history] = "temp_reset_history", [hwmon_chip_in_reset_history] = "in_reset_history", [hwmon_chip_curr_reset_history] = "curr_reset_history", @@ -400,7 +434,7 @@ static const char * const hwmon_pwm_attr_templates[] = { }; static const char * const *__templates[] = { - [hwmon_chip] = hwmon_chip_attr_templates, + [hwmon_chip] = hwmon_chip_attrs, [hwmon_temp] = hwmon_temp_attr_templates, [hwmon_in] = hwmon_in_attr_templates, [hwmon_curr] = hwmon_curr_attr_templates, @@ -412,7 +446,7 @@ static const char * const *__templates[] = { }; static const int __templates_size[] = { - [hwmon_chip] = ARRAY_SIZE(hwmon_chip_attr_templates), + [hwmon_chip] = ARRAY_SIZE(hwmon_chip_attrs), [hwmon_temp] = ARRAY_SIZE(hwmon_temp_attr_templates), [hwmon_in] = ARRAY_SIZE(hwmon_in_attr_templates), [hwmon_curr] = ARRAY_SIZE(hwmon_curr_attr_templates), @@ -526,9 +560,9 @@ __hwmon_device_register(struct device *dev, const char *name, void *drvdata, hdev = &hwdev->dev; - if (chip && chip->ops->is_visible) { + if (chip) { struct attribute **attrs; - int ngroups = 2; + int ngroups = 2; /* terminating NULL plus &hwdev->groups */ if (groups) for (i = 0; groups[i]; i++) @@ -572,7 +606,7 @@ __hwmon_device_register(struct device *dev, const char *name, void *drvdata, if (err) goto free_hwmon; - if (chip && chip->ops->is_visible && chip->ops->read && + if (chip && chip->ops->read && chip->info[0]->type == hwmon_chip && (chip->info[0]->config[0] & HWMON_C_REGISTER_TZ)) { const struct hwmon_channel_info **info = chip->info; @@ -626,8 +660,8 @@ EXPORT_SYMBOL_GPL(hwmon_device_register_with_groups); * @dev: the parent device * @name: hwmon name attribute * @drvdata: driver data to attach to created device - * @info: Pointer to hwmon chip information - * @groups - pointer to list of driver specific attribute groups + * @info: pointer to hwmon chip information + * @extra_groups: pointer to list of additional non-standard attribute groups * * hwmon_device_unregister() must be called when the device is no * longer needed. @@ -638,12 +672,12 @@ struct device * hwmon_device_register_with_info(struct device *dev, const char *name, void *drvdata, const struct hwmon_chip_info *chip, - const struct attribute_group **groups) + const struct attribute_group **extra_groups) { - if (chip && (!chip->ops || !chip->info)) + if (chip && (!chip->ops || !chip->ops->is_visible || !chip->info)) return ERR_PTR(-EINVAL); - return __hwmon_device_register(dev, name, drvdata, chip, groups); + return __hwmon_device_register(dev, name, drvdata, chip, extra_groups); } EXPORT_SYMBOL_GPL(hwmon_device_register_with_info); @@ -658,6 +692,9 @@ EXPORT_SYMBOL_GPL(hwmon_device_register_with_info); */ struct device *hwmon_device_register(struct device *dev) { + dev_warn(dev, + "hwmon_device_register() is deprecated. Please convert the driver to use hwmon_device_register_with_info().\n"); + return hwmon_device_register_with_groups(dev, NULL, NULL, NULL); } EXPORT_SYMBOL_GPL(hwmon_device_register); diff --git a/drivers/hwmon/lm85.c b/drivers/hwmon/lm85.c index 6ff773f..29c8136 100644 --- a/drivers/hwmon/lm85.c +++ b/drivers/hwmon/lm85.c @@ -136,7 +136,8 @@ static const int lm85_scaling[] = { /* .001 Volts */ #define SCALE(val, from, to) (((val) * (to) + ((from) / 2)) / (from)) #define INS_TO_REG(n, val) \ - clamp_val(SCALE(val, lm85_scaling[n], 192), 0, 255) + SCALE(clamp_val(val, 0, 255 * lm85_scaling[n] / 192), \ + lm85_scaling[n], 192) #define INSEXT_FROM_REG(n, val, ext) \ SCALE(((val) << 4) + (ext), 192 << 4, lm85_scaling[n]) diff --git a/drivers/hwmon/lm87.c b/drivers/hwmon/lm87.c index a5e2958..13cca36 100644 --- a/drivers/hwmon/lm87.c +++ b/drivers/hwmon/lm87.c @@ -121,7 +121,7 @@ static u8 LM87_REG_TEMP_LOW[3] = { 0x3A, 0x38, 0x2C }; #define IN_FROM_REG(reg, scale) (((reg) * (scale) + 96) / 192) #define IN_TO_REG(val, scale) ((val) <= 0 ? 0 : \ - (val) * 192 >= (scale) * 255 ? 255 : \ + (val) >= (scale) * 255 / 192 ? 255 : \ ((val) * 192 + (scale) / 2) / (scale)) #define TEMP_FROM_REG(reg) ((reg) * 1000) @@ -154,7 +154,6 @@ static u8 LM87_REG_TEMP_LOW[3] = { 0x3A, 0x38, 0x2C }; */ struct lm87_data { - struct device *hwmon_dev; struct mutex update_lock; char valid; /* zero until following fields are valid */ unsigned long last_updated; /* In jiffies */ @@ -181,6 +180,8 @@ struct lm87_data { u16 alarms; /* register values, combined */ u8 vid; /* register values, combined */ u8 vrm; + + const struct attribute_group *attr_groups[6]; }; static inline int lm87_read_value(struct i2c_client *client, u8 reg) @@ -195,7 +196,7 @@ static inline int lm87_write_value(struct i2c_client *client, u8 reg, u8 value) static struct lm87_data *lm87_update_device(struct device *dev) { - struct i2c_client *client = to_i2c_client(dev); + struct i2c_client *client = dev_get_drvdata(dev); struct lm87_data *data = i2c_get_clientdata(client); mutex_lock(&data->update_lock); @@ -309,7 +310,7 @@ static ssize_t show_in_max(struct device *dev, static ssize_t set_in_min(struct device *dev, struct device_attribute *attr, const char *buf, size_t count) { - struct i2c_client *client = to_i2c_client(dev); + struct i2c_client *client = dev_get_drvdata(dev); struct lm87_data *data = i2c_get_clientdata(client); int nr = to_sensor_dev_attr(attr)->index; long val; @@ -330,7 +331,7 @@ static ssize_t set_in_min(struct device *dev, struct device_attribute *attr, static ssize_t set_in_max(struct device *dev, struct device_attribute *attr, const char *buf, size_t count) { - struct i2c_client *client = to_i2c_client(dev); + struct i2c_client *client = dev_get_drvdata(dev); struct lm87_data *data = i2c_get_clientdata(client); int nr = to_sensor_dev_attr(attr)->index; long val; @@ -396,7 +397,7 @@ static ssize_t show_temp_high(struct device *dev, static ssize_t set_temp_low(struct device *dev, struct device_attribute *attr, const char *buf, size_t count) { - struct i2c_client *client = to_i2c_client(dev); + struct i2c_client *client = dev_get_drvdata(dev); struct lm87_data *data = i2c_get_clientdata(client); int nr = to_sensor_dev_attr(attr)->index; long val; @@ -416,7 +417,7 @@ static ssize_t set_temp_low(struct device *dev, struct device_attribute *attr, static ssize_t set_temp_high(struct device *dev, struct device_attribute *attr, const char *buf, size_t count) { - struct i2c_client *client = to_i2c_client(dev); + struct i2c_client *client = dev_get_drvdata(dev); struct lm87_data *data = i2c_get_clientdata(client); int nr = to_sensor_dev_attr(attr)->index; long val; @@ -495,7 +496,7 @@ static ssize_t show_fan_div(struct device *dev, static ssize_t set_fan_min(struct device *dev, struct device_attribute *attr, const char *buf, size_t count) { - struct i2c_client *client = to_i2c_client(dev); + struct i2c_client *client = dev_get_drvdata(dev); struct lm87_data *data = i2c_get_clientdata(client); int nr = to_sensor_dev_attr(attr)->index; long val; @@ -522,7 +523,7 @@ static ssize_t set_fan_min(struct device *dev, struct device_attribute *attr, static ssize_t set_fan_div(struct device *dev, struct device_attribute *attr, const char *buf, size_t count) { - struct i2c_client *client = to_i2c_client(dev); + struct i2c_client *client = dev_get_drvdata(dev); struct lm87_data *data = i2c_get_clientdata(client); int nr = to_sensor_dev_attr(attr)->index; long val; @@ -635,7 +636,7 @@ static ssize_t show_aout(struct device *dev, struct device_attribute *attr, static ssize_t set_aout(struct device *dev, struct device_attribute *attr, const char *buf, size_t count) { - struct i2c_client *client = to_i2c_client(dev); + struct i2c_client *client = dev_get_drvdata(dev); struct lm87_data *data = i2c_get_clientdata(client); long val; int err; @@ -841,23 +842,18 @@ static int lm87_detect(struct i2c_client *client, struct i2c_board_info *info) return 0; } -static void lm87_remove_files(struct i2c_client *client) +static void lm87_restore_config(void *arg) { - struct device *dev = &client->dev; - - sysfs_remove_group(&dev->kobj, &lm87_group); - sysfs_remove_group(&dev->kobj, &lm87_group_in6); - sysfs_remove_group(&dev->kobj, &lm87_group_fan1); - sysfs_remove_group(&dev->kobj, &lm87_group_in7); - sysfs_remove_group(&dev->kobj, &lm87_group_fan2); - sysfs_remove_group(&dev->kobj, &lm87_group_temp3); - sysfs_remove_group(&dev->kobj, &lm87_group_in0_5); - sysfs_remove_group(&dev->kobj, &lm87_group_vid); + struct i2c_client *client = arg; + struct lm87_data *data = i2c_get_clientdata(client); + + lm87_write_value(client, LM87_REG_CONFIG, data->config); } -static void lm87_init_client(struct i2c_client *client) +static int lm87_init_client(struct i2c_client *client) { struct lm87_data *data = i2c_get_clientdata(client); + int rc; if (dev_get_platdata(&client->dev)) { data->channel = *(u8 *)dev_get_platdata(&client->dev); @@ -868,6 +864,10 @@ static void lm87_init_client(struct i2c_client *client) } data->config = lm87_read_value(client, LM87_REG_CONFIG) & 0x6F; + rc = devm_add_action(&client->dev, lm87_restore_config, client); + if (rc) + return rc; + if (!(data->config & 0x01)) { int i; @@ -895,12 +895,15 @@ static void lm87_init_client(struct i2c_client *client) if ((data->config & 0x09) != 0x01) lm87_write_value(client, LM87_REG_CONFIG, (data->config & 0x77) | 0x01); + return 0; } static int lm87_probe(struct i2c_client *client, const struct i2c_device_id *id) { struct lm87_data *data; + struct device *hwmon_dev; int err; + unsigned int group_tail = 0; data = devm_kzalloc(&client->dev, sizeof(struct lm87_data), GFP_KERNEL); if (!data) @@ -910,7 +913,9 @@ static int lm87_probe(struct i2c_client *client, const struct i2c_device_id *id) mutex_init(&data->update_lock); /* Initialize the LM87 chip */ - lm87_init_client(client); + err = lm87_init_client(client); + if (err) + return err; data->in_scale[0] = 2500; data->in_scale[1] = 2700; @@ -921,72 +926,34 @@ static int lm87_probe(struct i2c_client *client, const struct i2c_device_id *id) data->in_scale[6] = 1875; data->in_scale[7] = 1875; - /* Register sysfs hooks */ - err = sysfs_create_group(&client->dev.kobj, &lm87_group); - if (err) - goto exit_stop; - - if (data->channel & CHAN_NO_FAN(0)) { - err = sysfs_create_group(&client->dev.kobj, &lm87_group_in6); - if (err) - goto exit_remove; - } else { - err = sysfs_create_group(&client->dev.kobj, &lm87_group_fan1); - if (err) - goto exit_remove; - } - - if (data->channel & CHAN_NO_FAN(1)) { - err = sysfs_create_group(&client->dev.kobj, &lm87_group_in7); - if (err) - goto exit_remove; - } else { - err = sysfs_create_group(&client->dev.kobj, &lm87_group_fan2); - if (err) - goto exit_remove; - } - - if (data->channel & CHAN_TEMP3) { - err = sysfs_create_group(&client->dev.kobj, &lm87_group_temp3); - if (err) - goto exit_remove; - } else { - err = sysfs_create_group(&client->dev.kobj, &lm87_group_in0_5); - if (err) - goto exit_remove; - } + /* + * Construct the list of attributes, the list depends on the + * configuration of the chip + */ + data->attr_groups[group_tail++] = &lm87_group; + if (data->channel & CHAN_NO_FAN(0)) + data->attr_groups[group_tail++] = &lm87_group_in6; + else + data->attr_groups[group_tail++] = &lm87_group_fan1; + + if (data->channel & CHAN_NO_FAN(1)) + data->attr_groups[group_tail++] = &lm87_group_in7; + else + data->attr_groups[group_tail++] = &lm87_group_fan2; + + if (data->channel & CHAN_TEMP3) + data->attr_groups[group_tail++] = &lm87_group_temp3; + else + data->attr_groups[group_tail++] = &lm87_group_in0_5; if (!(data->channel & CHAN_NO_VID)) { data->vrm = vid_which_vrm(); - err = sysfs_create_group(&client->dev.kobj, &lm87_group_vid); - if (err) - goto exit_remove; - } - - data->hwmon_dev = hwmon_device_register(&client->dev); - if (IS_ERR(data->hwmon_dev)) { - err = PTR_ERR(data->hwmon_dev); - goto exit_remove; + data->attr_groups[group_tail++] = &lm87_group_vid; } - return 0; - -exit_remove: - lm87_remove_files(client); -exit_stop: - lm87_write_value(client, LM87_REG_CONFIG, data->config); - return err; -} - -static int lm87_remove(struct i2c_client *client) -{ - struct lm87_data *data = i2c_get_clientdata(client); - - hwmon_device_unregister(data->hwmon_dev); - lm87_remove_files(client); - - lm87_write_value(client, LM87_REG_CONFIG, data->config); - return 0; + hwmon_dev = devm_hwmon_device_register_with_groups( + &client->dev, client->name, client, data->attr_groups); + return PTR_ERR_OR_ZERO(hwmon_dev); } /* @@ -1006,7 +973,6 @@ static struct i2c_driver lm87_driver = { .name = "lm87", }, .probe = lm87_probe, - .remove = lm87_remove, .id_table = lm87_id, .detect = lm87_detect, .address_list = normal_i2c, diff --git a/drivers/hwmon/mcp3021.c b/drivers/hwmon/mcp3021.c index 972444a..1929734 100644 --- a/drivers/hwmon/mcp3021.c +++ b/drivers/hwmon/mcp3021.c @@ -4,6 +4,7 @@ * Copyright (C) 2008-2009, 2012 Freescale Semiconductor, Inc. * Author: Mingkai Hu <Mingkai.hu@freescale.com> * Reworked by Sven Schuchmann <schuchmann@schleissheimer.de> + * DT support added by Clemens Gruber <clemens.gruber@pqgruber.com> * * This driver export the value of analog input voltage to sysfs, the * voltage unit is mV. Through the sysfs interface, lm-sensors tool @@ -22,11 +23,13 @@ #include <linux/i2c.h> #include <linux/err.h> #include <linux/device.h> +#include <linux/of.h> +#include <linux/of_device.h> -/* Vdd info */ -#define MCP3021_VDD_MAX 5500 -#define MCP3021_VDD_MIN 2700 -#define MCP3021_VDD_REF 3300 +/* Vdd / reference voltage in millivolt */ +#define MCP3021_VDD_REF_MAX 5500 +#define MCP3021_VDD_REF_MIN 2700 +#define MCP3021_VDD_REF_DEFAULT 3300 /* output format */ #define MCP3021_SAR_SHIFT 2 @@ -47,7 +50,7 @@ enum chips { */ struct mcp3021_data { struct device *hwmon_dev; - u32 vdd; /* device power supply */ + u32 vdd; /* supply and reference voltage in millivolt */ u16 sar_shift; u16 sar_mask; u8 output_res; @@ -99,13 +102,14 @@ static ssize_t show_in_input(struct device *dev, struct device_attribute *attr, return sprintf(buf, "%d\n", in_input); } -static DEVICE_ATTR(in0_input, S_IRUGO, show_in_input, NULL); +static DEVICE_ATTR(in0_input, 0444, show_in_input, NULL); static int mcp3021_probe(struct i2c_client *client, const struct i2c_device_id *id) { int err; struct mcp3021_data *data = NULL; + struct device_node *np = client->dev.of_node; if (!i2c_check_functionality(client->adapter, I2C_FUNC_I2C)) return -ENODEV; @@ -117,6 +121,21 @@ static int mcp3021_probe(struct i2c_client *client, i2c_set_clientdata(client, data); + if (np) { + if (!of_property_read_u32(np, "reference-voltage-microvolt", + &data->vdd)) + data->vdd /= 1000; + else + data->vdd = MCP3021_VDD_REF_DEFAULT; + } else { + u32 *pdata = dev_get_platdata(&client->dev); + + if (pdata) + data->vdd = *pdata; + else + data->vdd = MCP3021_VDD_REF_DEFAULT; + } + switch (id->driver_data) { case mcp3021: data->sar_shift = MCP3021_SAR_SHIFT; @@ -131,13 +150,8 @@ static int mcp3021_probe(struct i2c_client *client, break; } - if (dev_get_platdata(&client->dev)) { - data->vdd = *(u32 *)dev_get_platdata(&client->dev); - if (data->vdd > MCP3021_VDD_MAX || data->vdd < MCP3021_VDD_MIN) - return -EINVAL; - } else { - data->vdd = MCP3021_VDD_REF; - } + if (data->vdd > MCP3021_VDD_REF_MAX || data->vdd < MCP3021_VDD_REF_MIN) + return -EINVAL; err = sysfs_create_file(&client->dev.kobj, &dev_attr_in0_input.attr); if (err) @@ -173,9 +187,19 @@ static const struct i2c_device_id mcp3021_id[] = { }; MODULE_DEVICE_TABLE(i2c, mcp3021_id); +#ifdef CONFIG_OF +static const struct of_device_id of_mcp3021_match[] = { + { .compatible = "microchip,mcp3021", .data = (void *)mcp3021 }, + { .compatible = "microchip,mcp3221", .data = (void *)mcp3221 }, + { } +}; +MODULE_DEVICE_TABLE(of, of_mcp3021_match); +#endif + static struct i2c_driver mcp3021_driver = { .driver = { .name = "mcp3021", + .of_match_table = of_match_ptr(of_mcp3021_match), }, .probe = mcp3021_probe, .remove = mcp3021_remove, diff --git a/drivers/hwmon/nct7802.c b/drivers/hwmon/nct7802.c index 3ce33d2..12b94b0 100644 --- a/drivers/hwmon/nct7802.c +++ b/drivers/hwmon/nct7802.c @@ -259,13 +259,15 @@ static int nct7802_read_fan_min(struct nct7802_data *data, u8 reg_fan_low, ret = 0; else if (ret) ret = DIV_ROUND_CLOSEST(1350000U, ret); + else + ret = 1350000U; abort: mutex_unlock(&data->access_lock); return ret; } static int nct7802_write_fan_min(struct nct7802_data *data, u8 reg_fan_low, - u8 reg_fan_high, unsigned int limit) + u8 reg_fan_high, unsigned long limit) { int err; @@ -326,8 +328,8 @@ static int nct7802_write_voltage(struct nct7802_data *data, int nr, int index, int shift = 8 - REG_VOLTAGE_LIMIT_MSB_SHIFT[index - 1][nr]; int err; + voltage = clamp_val(voltage, 0, 0x3ff * nct7802_vmul[nr]); voltage = DIV_ROUND_CLOSEST(voltage, nct7802_vmul[nr]); - voltage = clamp_val(voltage, 0, 0x3ff); mutex_lock(&data->access_lock); err = regmap_write(data->regmap, @@ -402,7 +404,7 @@ static ssize_t store_temp(struct device *dev, struct device_attribute *attr, if (err < 0) return err; - val = clamp_val(DIV_ROUND_CLOSEST(val, 1000), -128, 127); + val = DIV_ROUND_CLOSEST(clamp_val(val, -128000, 127000), 1000); err = regmap_write(data->regmap, nr, val & 0xff); return err ? : count; diff --git a/drivers/hwmon/pmbus/adm1275.c b/drivers/hwmon/pmbus/adm1275.c index 3baa4f4a..4ab5293 100644 --- a/drivers/hwmon/pmbus/adm1275.c +++ b/drivers/hwmon/pmbus/adm1275.c @@ -499,15 +499,27 @@ static int adm1275_probe(struct i2c_client *client, pindex = 2; tindex = 3; - info->func[0] |= PMBUS_HAVE_PIN | PMBUS_HAVE_STATUS_INPUT; + info->func[0] |= PMBUS_HAVE_PIN | PMBUS_HAVE_STATUS_INPUT | + PMBUS_HAVE_VOUT | PMBUS_HAVE_STATUS_VOUT; + + /* Enable VOUT if not enabled (it is disabled by default) */ + if (!(config & ADM1278_VOUT_EN)) { + config |= ADM1278_VOUT_EN; + ret = i2c_smbus_write_byte_data(client, + ADM1275_PMON_CONFIG, + config); + if (ret < 0) { + dev_err(&client->dev, + "Failed to enable VOUT monitoring\n"); + return -ENODEV; + } + } + if (config & ADM1278_TEMP1_EN) info->func[0] |= PMBUS_HAVE_TEMP | PMBUS_HAVE_STATUS_TEMP; if (config & ADM1278_VIN_EN) info->func[0] |= PMBUS_HAVE_VIN; - if (config & ADM1278_VOUT_EN) - info->func[0] |= - PMBUS_HAVE_VOUT | PMBUS_HAVE_STATUS_VOUT; break; case adm1293: case adm1294: diff --git a/drivers/hwmon/scpi-hwmon.c b/drivers/hwmon/scpi-hwmon.c index 559a3dc..094f948 100644 --- a/drivers/hwmon/scpi-hwmon.c +++ b/drivers/hwmon/scpi-hwmon.c @@ -251,6 +251,7 @@ static const struct of_device_id scpi_of_match[] = { {.compatible = "arm,scpi-sensors"}, {}, }; +MODULE_DEVICE_TABLE(of, scpi_of_match); static struct platform_driver scpi_hwmon_platdrv = { .driver = { diff --git a/drivers/hwmon/smsc47m192.c b/drivers/hwmon/smsc47m192.c index 6ac7cda..15650f2 100644 --- a/drivers/hwmon/smsc47m192.c +++ b/drivers/hwmon/smsc47m192.c @@ -77,14 +77,15 @@ static inline unsigned int IN_FROM_REG(u8 reg, int n) static inline u8 IN_TO_REG(unsigned long val, int n) { - return clamp_val(SCALE(val, 192, nom_mv[n]), 0, 255); + val = clamp_val(val, 0, nom_mv[n] * 255 / 192); + return SCALE(val, 192, nom_mv[n]); } /* * TEMP: 0.001 degC units (-128C to +127C) * REG: 1C/bit, two's complement */ -static inline s8 TEMP_TO_REG(int val) +static inline s8 TEMP_TO_REG(long val) { return SCALE(clamp_val(val, -128000, 127000), 1, 1000); } diff --git a/drivers/hwmon/tc654.c b/drivers/hwmon/tc654.c new file mode 100644 index 0000000..18136e1 --- /dev/null +++ b/drivers/hwmon/tc654.c @@ -0,0 +1,514 @@ +/* + * tc654.c - Linux kernel modules for fan speed controller + * + * Copyright (C) 2016 Allied Telesis Labs NZ + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ + +#include <linux/bitops.h> +#include <linux/err.h> +#include <linux/hwmon.h> +#include <linux/hwmon-sysfs.h> +#include <linux/i2c.h> +#include <linux/init.h> +#include <linux/jiffies.h> +#include <linux/module.h> +#include <linux/mutex.h> +#include <linux/slab.h> +#include <linux/util_macros.h> + +enum tc654_regs { + TC654_REG_RPM1 = 0x00, /* RPM Output 1 */ + TC654_REG_RPM2 = 0x01, /* RPM Output 2 */ + TC654_REG_FAN_FAULT1 = 0x02, /* Fan Fault 1 Threshold */ + TC654_REG_FAN_FAULT2 = 0x03, /* Fan Fault 2 Threshold */ + TC654_REG_CONFIG = 0x04, /* Configuration */ + TC654_REG_STATUS = 0x05, /* Status */ + TC654_REG_DUTY_CYCLE = 0x06, /* Fan Speed Duty Cycle */ + TC654_REG_MFR_ID = 0x07, /* Manufacturer Identification */ + TC654_REG_VER_ID = 0x08, /* Version Identification */ +}; + +/* Macros to easily index the registers */ +#define TC654_REG_RPM(idx) (TC654_REG_RPM1 + (idx)) +#define TC654_REG_FAN_FAULT(idx) (TC654_REG_FAN_FAULT1 + (idx)) + +/* Config register bits */ +#define TC654_REG_CONFIG_RES BIT(6) /* Resolution Selection */ +#define TC654_REG_CONFIG_DUTYC BIT(5) /* Duty Cycle Control */ +#define TC654_REG_CONFIG_SDM BIT(0) /* Shutdown Mode */ + +/* Status register bits */ +#define TC654_REG_STATUS_F2F BIT(1) /* Fan 2 Fault */ +#define TC654_REG_STATUS_F1F BIT(0) /* Fan 1 Fault */ + +/* RPM resolution for RPM Output registers */ +#define TC654_HIGH_RPM_RESOLUTION 25 /* 25 RPM resolution */ +#define TC654_LOW_RPM_RESOLUTION 50 /* 50 RPM resolution */ + +/* Convert to the fan fault RPM threshold from register value */ +#define TC654_FAN_FAULT_FROM_REG(val) ((val) * 50) /* 50 RPM resolution */ + +/* Convert to register value from the fan fault RPM threshold */ +#define TC654_FAN_FAULT_TO_REG(val) (((val) / 50) & 0xff) + +/* Register data is read (and cached) at most once per second. */ +#define TC654_UPDATE_INTERVAL HZ + +struct tc654_data { + struct i2c_client *client; + + /* update mutex */ + struct mutex update_lock; + + /* tc654 register cache */ + bool valid; + unsigned long last_updated; /* in jiffies */ + + u8 rpm_output[2]; /* The fan RPM data for fans 1 and 2 is then + * written to registers RPM1 and RPM2 + */ + u8 fan_fault[2]; /* The Fan Fault Threshold Registers are used to + * set the fan fault threshold levels for fan 1 + * and fan 2 + */ + u8 config; /* The Configuration Register is an 8-bit read/ + * writable multi-function control register + * 7: Fan Fault Clear + * 1 = Clear Fan Fault + * 0 = Normal Operation (default) + * 6: Resolution Selection for RPM Output Registers + * RPM Output Registers (RPM1 and RPM2) will be + * set for + * 1 = 25 RPM (9-bit) resolution + * 0 = 50 RPM (8-bit) resolution (default) + * 5: Duty Cycle Control Method + * The V OUT duty cycle will be controlled via + * 1 = the SMBus interface. + * 0 = via the V IN analog input pin. (default) + * 4,3: Fan 2 Pulses Per Rotation + * 00 = 1 + * 01 = 2 (default) + * 10 = 4 + * 11 = 8 + * 2,1: Fan 1 Pulses Per Rotation + * 00 = 1 + * 01 = 2 (default) + * 10 = 4 + * 11 = 8 + * 0: Shutdown Mode + * 1 = Shutdown mode. + * 0 = Normal operation. (default) + */ + u8 status; /* The Status register provides all the information + * about what is going on within the TC654/TC655 + * devices. + * 7,6: Unimplemented, Read as '0' + * 5: Over-Temperature Fault Condition + * 1 = Over-Temperature condition has occurred + * 0 = Normal operation. V IN is less than 2.6V + * 4: RPM2 Counter Overflow + * 1 = Fault condition + * 0 = Normal operation + * 3: RPM1 Counter Overflow + * 1 = Fault condition + * 0 = Normal operation + * 2: V IN Input Status + * 1 = V IN is open + * 0 = Normal operation. voltage present at V IN + * 1: Fan 2 Fault + * 1 = Fault condition + * 0 = Normal operation + * 0: Fan 1 Fault + * 1 = Fault condition + * 0 = Normal operation + */ + u8 duty_cycle; /* The DUTY_CYCLE register is a 4-bit read/ + * writable register used to control the duty + * cycle of the V OUT output. + */ +}; + +/* helper to grab and cache data, at most one time per second */ +static struct tc654_data *tc654_update_client(struct device *dev) +{ + struct tc654_data *data = dev_get_drvdata(dev); + struct i2c_client *client = data->client; + int ret = 0; + + mutex_lock(&data->update_lock); + if (time_before(jiffies, data->last_updated + TC654_UPDATE_INTERVAL) && + likely(data->valid)) + goto out; + + ret = i2c_smbus_read_byte_data(client, TC654_REG_RPM(0)); + if (ret < 0) + goto out; + data->rpm_output[0] = ret; + + ret = i2c_smbus_read_byte_data(client, TC654_REG_RPM(1)); + if (ret < 0) + goto out; + data->rpm_output[1] = ret; + + ret = i2c_smbus_read_byte_data(client, TC654_REG_FAN_FAULT(0)); + if (ret < 0) + goto out; + data->fan_fault[0] = ret; + + ret = i2c_smbus_read_byte_data(client, TC654_REG_FAN_FAULT(1)); + if (ret < 0) + goto out; + data->fan_fault[1] = ret; + + ret = i2c_smbus_read_byte_data(client, TC654_REG_CONFIG); + if (ret < 0) + goto out; + data->config = ret; + + ret = i2c_smbus_read_byte_data(client, TC654_REG_STATUS); + if (ret < 0) + goto out; + data->status = ret; + + ret = i2c_smbus_read_byte_data(client, TC654_REG_DUTY_CYCLE); + if (ret < 0) + goto out; + data->duty_cycle = ret & 0x0f; + + data->last_updated = jiffies; + data->valid = true; +out: + mutex_unlock(&data->update_lock); + + if (ret < 0) /* upon error, encode it in return value */ + data = ERR_PTR(ret); + + return data; +} + +/* + * sysfs attributes + */ + +static ssize_t show_fan(struct device *dev, struct device_attribute *da, + char *buf) +{ + int nr = to_sensor_dev_attr(da)->index; + struct tc654_data *data = tc654_update_client(dev); + int val; + + if (IS_ERR(data)) + return PTR_ERR(data); + + if (data->config & TC654_REG_CONFIG_RES) + val = data->rpm_output[nr] * TC654_HIGH_RPM_RESOLUTION; + else + val = data->rpm_output[nr] * TC654_LOW_RPM_RESOLUTION; + + return sprintf(buf, "%d\n", val); +} + +static ssize_t show_fan_min(struct device *dev, struct device_attribute *da, + char *buf) +{ + int nr = to_sensor_dev_attr(da)->index; + struct tc654_data *data = tc654_update_client(dev); + + if (IS_ERR(data)) + return PTR_ERR(data); + + return sprintf(buf, "%d\n", + TC654_FAN_FAULT_FROM_REG(data->fan_fault[nr])); +} + +static ssize_t set_fan_min(struct device *dev, struct device_attribute *da, + const char *buf, size_t count) +{ + int nr = to_sensor_dev_attr(da)->index; + struct tc654_data *data = dev_get_drvdata(dev); + struct i2c_client *client = data->client; + unsigned long val; + int ret; + + if (kstrtoul(buf, 10, &val)) + return -EINVAL; + + val = clamp_val(val, 0, 12750); + + mutex_lock(&data->update_lock); + + data->fan_fault[nr] = TC654_FAN_FAULT_TO_REG(val); + ret = i2c_smbus_write_byte_data(client, TC654_REG_FAN_FAULT(nr), + data->fan_fault[nr]); + + mutex_unlock(&data->update_lock); + return ret < 0 ? ret : count; +} + +static ssize_t show_fan_alarm(struct device *dev, struct device_attribute *da, + char *buf) +{ + int nr = to_sensor_dev_attr(da)->index; + struct tc654_data *data = tc654_update_client(dev); + int val; + + if (IS_ERR(data)) + return PTR_ERR(data); + + if (nr == 0) + val = !!(data->status & TC654_REG_STATUS_F1F); + else + val = !!(data->status & TC654_REG_STATUS_F2F); + + return sprintf(buf, "%d\n", val); +} + +static const u8 TC654_FAN_PULSE_SHIFT[] = { 1, 3 }; + +static ssize_t show_fan_pulses(struct device *dev, struct device_attribute *da, + char *buf) +{ + int nr = to_sensor_dev_attr(da)->index; + struct tc654_data *data = tc654_update_client(dev); + u8 val; + + if (IS_ERR(data)) + return PTR_ERR(data); + + val = BIT((data->config >> TC654_FAN_PULSE_SHIFT[nr]) & 0x03); + return sprintf(buf, "%d\n", val); +} + +static ssize_t set_fan_pulses(struct device *dev, struct device_attribute *da, + const char *buf, size_t count) +{ + int nr = to_sensor_dev_attr(da)->index; + struct tc654_data *data = dev_get_drvdata(dev); + struct i2c_client *client = data->client; + u8 config; + unsigned long val; + int ret; + + if (kstrtoul(buf, 10, &val)) + return -EINVAL; + + switch (val) { + case 1: + config = 0; + break; + case 2: + config = 1; + break; + case 4: + config = 2; + break; + case 8: + config = 3; + break; + default: + return -EINVAL; + } + + mutex_lock(&data->update_lock); + + data->config &= ~(0x03 << TC654_FAN_PULSE_SHIFT[nr]); + data->config |= (config << TC654_FAN_PULSE_SHIFT[nr]); + ret = i2c_smbus_write_byte_data(client, TC654_REG_CONFIG, data->config); + + mutex_unlock(&data->update_lock); + return ret < 0 ? ret : count; +} + +static ssize_t show_pwm_mode(struct device *dev, + struct device_attribute *da, char *buf) +{ + struct tc654_data *data = tc654_update_client(dev); + + if (IS_ERR(data)) + return PTR_ERR(data); + + return sprintf(buf, "%d\n", !!(data->config & TC654_REG_CONFIG_DUTYC)); +} + +static ssize_t set_pwm_mode(struct device *dev, + struct device_attribute *da, + const char *buf, size_t count) +{ + struct tc654_data *data = dev_get_drvdata(dev); + struct i2c_client *client = data->client; + unsigned long val; + int ret; + + if (kstrtoul(buf, 10, &val)) + return -EINVAL; + + if (val != 0 && val != 1) + return -EINVAL; + + mutex_lock(&data->update_lock); + + if (val) + data->config |= TC654_REG_CONFIG_DUTYC; + else + data->config &= ~TC654_REG_CONFIG_DUTYC; + + ret = i2c_smbus_write_byte_data(client, TC654_REG_CONFIG, data->config); + + mutex_unlock(&data->update_lock); + return ret < 0 ? ret : count; +} + +static const int tc654_pwm_map[16] = { 77, 88, 102, 112, 124, 136, 148, 160, + 172, 184, 196, 207, 219, 231, 243, 255}; + +static ssize_t show_pwm(struct device *dev, struct device_attribute *da, + char *buf) +{ + struct tc654_data *data = tc654_update_client(dev); + int pwm; + + if (IS_ERR(data)) + return PTR_ERR(data); + + if (data->config & TC654_REG_CONFIG_SDM) + pwm = 0; + else + pwm = tc654_pwm_map[data->duty_cycle]; + + return sprintf(buf, "%d\n", pwm); +} + +static ssize_t set_pwm(struct device *dev, struct device_attribute *da, + const char *buf, size_t count) +{ + struct tc654_data *data = dev_get_drvdata(dev); + struct i2c_client *client = data->client; + unsigned long val; + int ret; + + if (kstrtoul(buf, 10, &val)) + return -EINVAL; + if (val > 255) + return -EINVAL; + + mutex_lock(&data->update_lock); + + if (val == 0) + data->config |= TC654_REG_CONFIG_SDM; + else + data->config &= ~TC654_REG_CONFIG_SDM; + + data->duty_cycle = find_closest(val, tc654_pwm_map, + ARRAY_SIZE(tc654_pwm_map)); + + ret = i2c_smbus_write_byte_data(client, TC654_REG_CONFIG, data->config); + if (ret < 0) + goto out; + + ret = i2c_smbus_write_byte_data(client, TC654_REG_DUTY_CYCLE, + data->duty_cycle); + +out: + mutex_unlock(&data->update_lock); + return ret < 0 ? ret : count; +} + +static SENSOR_DEVICE_ATTR(fan1_input, S_IRUGO, show_fan, NULL, 0); +static SENSOR_DEVICE_ATTR(fan2_input, S_IRUGO, show_fan, NULL, 1); +static SENSOR_DEVICE_ATTR(fan1_min, S_IWUSR | S_IRUGO, show_fan_min, + set_fan_min, 0); +static SENSOR_DEVICE_ATTR(fan2_min, S_IWUSR | S_IRUGO, show_fan_min, + set_fan_min, 1); +static SENSOR_DEVICE_ATTR(fan1_alarm, S_IRUGO, show_fan_alarm, NULL, 0); +static SENSOR_DEVICE_ATTR(fan2_alarm, S_IRUGO, show_fan_alarm, NULL, 1); +static SENSOR_DEVICE_ATTR(fan1_pulses, S_IWUSR | S_IRUGO, show_fan_pulses, + set_fan_pulses, 0); +static SENSOR_DEVICE_ATTR(fan2_pulses, S_IWUSR | S_IRUGO, show_fan_pulses, + set_fan_pulses, 1); +static SENSOR_DEVICE_ATTR(pwm1_mode, S_IWUSR | S_IRUGO, + show_pwm_mode, set_pwm_mode, 0); +static SENSOR_DEVICE_ATTR(pwm1, S_IWUSR | S_IRUGO, show_pwm, + set_pwm, 0); + +/* Driver data */ +static struct attribute *tc654_attrs[] = { + &sensor_dev_attr_fan1_input.dev_attr.attr, + &sensor_dev_attr_fan2_input.dev_attr.attr, + &sensor_dev_attr_fan1_min.dev_attr.attr, + &sensor_dev_attr_fan2_min.dev_attr.attr, + &sensor_dev_attr_fan1_alarm.dev_attr.attr, + &sensor_dev_attr_fan2_alarm.dev_attr.attr, + &sensor_dev_attr_fan1_pulses.dev_attr.attr, + &sensor_dev_attr_fan2_pulses.dev_attr.attr, + &sensor_dev_attr_pwm1_mode.dev_attr.attr, + &sensor_dev_attr_pwm1.dev_attr.attr, + NULL +}; + +ATTRIBUTE_GROUPS(tc654); + +/* + * device probe and removal + */ + +static int tc654_probe(struct i2c_client *client, + const struct i2c_device_id *id) +{ + struct device *dev = &client->dev; + struct tc654_data *data; + struct device *hwmon_dev; + int ret; + + if (!i2c_check_functionality(client->adapter, I2C_FUNC_SMBUS_BYTE_DATA)) + return -ENODEV; + + data = devm_kzalloc(dev, sizeof(struct tc654_data), GFP_KERNEL); + if (!data) + return -ENOMEM; + + data->client = client; + mutex_init(&data->update_lock); + + ret = i2c_smbus_read_byte_data(client, TC654_REG_CONFIG); + if (ret < 0) + return ret; + + data->config = ret; + + hwmon_dev = + devm_hwmon_device_register_with_groups(dev, client->name, data, + tc654_groups); + return PTR_ERR_OR_ZERO(hwmon_dev); +} + +static const struct i2c_device_id tc654_id[] = { + {"tc654", 0}, + {"tc655", 0}, + {} +}; + +MODULE_DEVICE_TABLE(i2c, tc654_id); + +static struct i2c_driver tc654_driver = { + .driver = { + .name = "tc654", + }, + .probe = tc654_probe, + .id_table = tc654_id, +}; + +module_i2c_driver(tc654_driver); + +MODULE_AUTHOR("Allied Telesis Labs"); +MODULE_DESCRIPTION("Microchip TC654/TC655 driver"); +MODULE_LICENSE("GPL"); diff --git a/drivers/hwmon/tmp108.c b/drivers/hwmon/tmp108.c new file mode 100644 index 0000000..91bb9463 --- /dev/null +++ b/drivers/hwmon/tmp108.c @@ -0,0 +1,469 @@ +/* Texas Instruments TMP108 SMBus temperature sensor driver + * + * Copyright (C) 2016 John Muir <john@jmuir.com> + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ + +#include <linux/delay.h> +#include <linux/device.h> +#include <linux/err.h> +#include <linux/hwmon.h> +#include <linux/hwmon-sysfs.h> +#include <linux/module.h> +#include <linux/mutex.h> +#include <linux/of.h> +#include <linux/i2c.h> +#include <linux/init.h> +#include <linux/jiffies.h> +#include <linux/regmap.h> +#include <linux/slab.h> + +#define DRIVER_NAME "tmp108" + +#define TMP108_REG_TEMP 0x00 +#define TMP108_REG_CONF 0x01 +#define TMP108_REG_TLOW 0x02 +#define TMP108_REG_THIGH 0x03 + +#define TMP108_TEMP_MIN_MC -50000 /* Minimum millicelcius. */ +#define TMP108_TEMP_MAX_MC 127937 /* Maximum millicelcius. */ + +/* Configuration register bits. + * Note: these bit definitions are byte swapped. + */ +#define TMP108_CONF_M0 0x0100 /* Sensor mode. */ +#define TMP108_CONF_M1 0x0200 +#define TMP108_CONF_TM 0x0400 /* Thermostat mode. */ +#define TMP108_CONF_FL 0x0800 /* Watchdog flag - TLOW */ +#define TMP108_CONF_FH 0x1000 /* Watchdog flag - THIGH */ +#define TMP108_CONF_CR0 0x2000 /* Conversion rate. */ +#define TMP108_CONF_CR1 0x4000 +#define TMP108_CONF_ID 0x8000 +#define TMP108_CONF_HYS0 0x0010 /* Hysteresis. */ +#define TMP108_CONF_HYS1 0x0020 +#define TMP108_CONF_POL 0x0080 /* Polarity of alert. */ + +/* Defaults set by the hardware upon reset. */ +#define TMP108_CONF_DEFAULTS (TMP108_CONF_CR0 | TMP108_CONF_TM |\ + TMP108_CONF_HYS0 | TMP108_CONF_M1) +/* These bits are read-only. */ +#define TMP108_CONF_READ_ONLY (TMP108_CONF_FL | TMP108_CONF_FH |\ + TMP108_CONF_ID) + +#define TMP108_CONF_MODE_MASK (TMP108_CONF_M0|TMP108_CONF_M1) +#define TMP108_MODE_SHUTDOWN 0x0000 +#define TMP108_MODE_ONE_SHOT TMP108_CONF_M0 +#define TMP108_MODE_CONTINUOUS TMP108_CONF_M1 /* Default */ + /* When M1 is set, M0 is ignored. */ + +#define TMP108_CONF_CONVRATE_MASK (TMP108_CONF_CR0|TMP108_CONF_CR1) +#define TMP108_CONVRATE_0P25HZ 0x0000 +#define TMP108_CONVRATE_1HZ TMP108_CONF_CR0 /* Default */ +#define TMP108_CONVRATE_4HZ TMP108_CONF_CR1 +#define TMP108_CONVRATE_16HZ (TMP108_CONF_CR0|TMP108_CONF_CR1) + +#define TMP108_CONF_HYSTERESIS_MASK (TMP108_CONF_HYS0|TMP108_CONF_HYS1) +#define TMP108_HYSTERESIS_0C 0x0000 +#define TMP108_HYSTERESIS_1C TMP108_CONF_HYS0 /* Default */ +#define TMP108_HYSTERESIS_2C TMP108_CONF_HYS1 +#define TMP108_HYSTERESIS_4C (TMP108_CONF_HYS0|TMP108_CONF_HYS1) + +#define TMP108_CONVERSION_TIME_MS 30 /* in milli-seconds */ + +struct tmp108 { + struct regmap *regmap; + u16 orig_config; + unsigned long ready_time; +}; + +/* convert 12-bit TMP108 register value to milliCelsius */ +static inline int tmp108_temp_reg_to_mC(s16 val) +{ + return (val & ~0x0f) * 1000 / 256; +} + +/* convert milliCelsius to left adjusted 12-bit TMP108 register value */ +static inline u16 tmp108_mC_to_temp_reg(int val) +{ + return (val * 256) / 1000; +} + +static int tmp108_read(struct device *dev, enum hwmon_sensor_types type, + u32 attr, int channel, long *temp) +{ + struct tmp108 *tmp108 = dev_get_drvdata(dev); + unsigned int regval; + int err, hyst; + + if (type == hwmon_chip) { + if (attr == hwmon_chip_update_interval) { + err = regmap_read(tmp108->regmap, TMP108_REG_CONF, + ®val); + if (err < 0) + return err; + switch (regval & TMP108_CONF_CONVRATE_MASK) { + case TMP108_CONVRATE_0P25HZ: + default: + *temp = 4000; + break; + case TMP108_CONVRATE_1HZ: + *temp = 1000; + break; + case TMP108_CONVRATE_4HZ: + *temp = 250; + break; + case TMP108_CONVRATE_16HZ: + *temp = 63; + break; + } + return 0; + } + return -EOPNOTSUPP; + } + + switch (attr) { + case hwmon_temp_input: + /* Is it too early to return a conversion ? */ + if (time_before(jiffies, tmp108->ready_time)) { + dev_dbg(dev, "%s: Conversion not ready yet..\n", + __func__); + return -EAGAIN; + } + err = regmap_read(tmp108->regmap, TMP108_REG_TEMP, ®val); + if (err < 0) + return err; + *temp = tmp108_temp_reg_to_mC(regval); + break; + case hwmon_temp_min: + case hwmon_temp_max: + err = regmap_read(tmp108->regmap, attr == hwmon_temp_min ? + TMP108_REG_TLOW : TMP108_REG_THIGH, ®val); + if (err < 0) + return err; + *temp = tmp108_temp_reg_to_mC(regval); + break; + case hwmon_temp_min_alarm: + case hwmon_temp_max_alarm: + err = regmap_read(tmp108->regmap, TMP108_REG_CONF, ®val); + if (err < 0) + return err; + *temp = !!(regval & (attr == hwmon_temp_min_alarm ? + TMP108_CONF_FL : TMP108_CONF_FH)); + break; + case hwmon_temp_min_hyst: + case hwmon_temp_max_hyst: + err = regmap_read(tmp108->regmap, TMP108_REG_CONF, ®val); + if (err < 0) + return err; + switch (regval & TMP108_CONF_HYSTERESIS_MASK) { + case TMP108_HYSTERESIS_0C: + default: + hyst = 0; + break; + case TMP108_HYSTERESIS_1C: + hyst = 1000; + break; + case TMP108_HYSTERESIS_2C: + hyst = 2000; + break; + case TMP108_HYSTERESIS_4C: + hyst = 4000; + break; + } + err = regmap_read(tmp108->regmap, attr == hwmon_temp_min_hyst ? + TMP108_REG_TLOW : TMP108_REG_THIGH, ®val); + if (err < 0) + return err; + *temp = tmp108_temp_reg_to_mC(regval); + if (attr == hwmon_temp_min_hyst) + *temp += hyst; + else + *temp -= hyst; + break; + default: + return -EOPNOTSUPP; + } + + return 0; +} + +static int tmp108_write(struct device *dev, enum hwmon_sensor_types type, + u32 attr, int channel, long temp) +{ + struct tmp108 *tmp108 = dev_get_drvdata(dev); + u32 regval, mask; + int err; + + if (type == hwmon_chip) { + if (attr == hwmon_chip_update_interval) { + if (temp < 156) + mask = TMP108_CONVRATE_16HZ; + else if (temp < 625) + mask = TMP108_CONVRATE_4HZ; + else if (temp < 2500) + mask = TMP108_CONVRATE_1HZ; + else + mask = TMP108_CONVRATE_0P25HZ; + return regmap_update_bits(tmp108->regmap, + TMP108_REG_CONF, + TMP108_CONF_CONVRATE_MASK, + mask); + } + return -EOPNOTSUPP; + } + + switch (attr) { + case hwmon_temp_min: + case hwmon_temp_max: + temp = clamp_val(temp, TMP108_TEMP_MIN_MC, TMP108_TEMP_MAX_MC); + return regmap_write(tmp108->regmap, + attr == hwmon_temp_min ? + TMP108_REG_TLOW : TMP108_REG_THIGH, + tmp108_mC_to_temp_reg(temp)); + case hwmon_temp_min_hyst: + case hwmon_temp_max_hyst: + temp = clamp_val(temp, TMP108_TEMP_MIN_MC, TMP108_TEMP_MAX_MC); + err = regmap_read(tmp108->regmap, + attr == hwmon_temp_min_hyst ? + TMP108_REG_TLOW : TMP108_REG_THIGH, + ®val); + if (err < 0) + return err; + if (attr == hwmon_temp_min_hyst) + temp -= tmp108_temp_reg_to_mC(regval); + else + temp = tmp108_temp_reg_to_mC(regval) - temp; + if (temp < 500) + mask = TMP108_HYSTERESIS_0C; + else if (temp < 1500) + mask = TMP108_HYSTERESIS_1C; + else if (temp < 3000) + mask = TMP108_HYSTERESIS_2C; + else + mask = TMP108_HYSTERESIS_4C; + return regmap_update_bits(tmp108->regmap, TMP108_REG_CONF, + TMP108_CONF_HYSTERESIS_MASK, mask); + default: + return -EOPNOTSUPP; + } +} + +static umode_t tmp108_is_visible(const void *data, enum hwmon_sensor_types type, + u32 attr, int channel) +{ + if (type == hwmon_chip && attr == hwmon_chip_update_interval) + return 0644; + + if (type != hwmon_temp) + return 0; + + switch (attr) { + case hwmon_temp_input: + case hwmon_temp_min_alarm: + case hwmon_temp_max_alarm: + return 0444; + case hwmon_temp_min: + case hwmon_temp_max: + case hwmon_temp_min_hyst: + case hwmon_temp_max_hyst: + return 0644; + default: + return 0; + } +} + +static u32 tmp108_chip_config[] = { + HWMON_C_REGISTER_TZ | HWMON_C_UPDATE_INTERVAL, + 0 +}; + +static const struct hwmon_channel_info tmp108_chip = { + .type = hwmon_chip, + .config = tmp108_chip_config, +}; + +static u32 tmp108_temp_config[] = { + HWMON_T_INPUT | HWMON_T_MAX | HWMON_T_MIN | HWMON_T_MIN_HYST + | HWMON_T_MAX_HYST | HWMON_T_MIN_ALARM | HWMON_T_MAX_ALARM, + 0 +}; + +static const struct hwmon_channel_info tmp108_temp = { + .type = hwmon_temp, + .config = tmp108_temp_config, +}; + +static const struct hwmon_channel_info *tmp108_info[] = { + &tmp108_chip, + &tmp108_temp, + NULL +}; + +static const struct hwmon_ops tmp108_hwmon_ops = { + .is_visible = tmp108_is_visible, + .read = tmp108_read, + .write = tmp108_write, +}; + +static const struct hwmon_chip_info tmp108_chip_info = { + .ops = &tmp108_hwmon_ops, + .info = tmp108_info, +}; + +static void tmp108_restore_config(void *data) +{ + struct tmp108 *tmp108 = data; + + regmap_write(tmp108->regmap, TMP108_REG_CONF, tmp108->orig_config); +} + +static bool tmp108_is_writeable_reg(struct device *dev, unsigned int reg) +{ + return reg != TMP108_REG_TEMP; +} + +static bool tmp108_is_volatile_reg(struct device *dev, unsigned int reg) +{ + /* Configuration register must be volatile to enable FL and FH. */ + return reg == TMP108_REG_TEMP || reg == TMP108_REG_CONF; +} + +static const struct regmap_config tmp108_regmap_config = { + .reg_bits = 8, + .val_bits = 16, + .max_register = TMP108_REG_THIGH, + .writeable_reg = tmp108_is_writeable_reg, + .volatile_reg = tmp108_is_volatile_reg, + .val_format_endian = REGMAP_ENDIAN_BIG, + .cache_type = REGCACHE_RBTREE, + .use_single_rw = true, +}; + +static int tmp108_probe(struct i2c_client *client, + const struct i2c_device_id *id) +{ + struct device *dev = &client->dev; + struct device *hwmon_dev; + struct tmp108 *tmp108; + int err; + u32 config; + + if (!i2c_check_functionality(client->adapter, + I2C_FUNC_SMBUS_WORD_DATA)) { + dev_err(dev, + "adapter doesn't support SMBus word transactions\n"); + return -ENODEV; + } + + tmp108 = devm_kzalloc(dev, sizeof(*tmp108), GFP_KERNEL); + if (!tmp108) + return -ENOMEM; + + dev_set_drvdata(dev, tmp108); + + tmp108->regmap = devm_regmap_init_i2c(client, &tmp108_regmap_config); + if (IS_ERR(tmp108->regmap)) { + err = PTR_ERR(tmp108->regmap); + dev_err(dev, "regmap init failed: %d", err); + return err; + } + + err = regmap_read(tmp108->regmap, TMP108_REG_CONF, &config); + if (err < 0) { + dev_err(dev, "error reading config register: %d", err); + return err; + } + tmp108->orig_config = config; + + /* Only continuous mode is supported. */ + config &= ~TMP108_CONF_MODE_MASK; + config |= TMP108_MODE_CONTINUOUS; + + /* Only comparator mode is supported. */ + config &= ~TMP108_CONF_TM; + + err = regmap_write(tmp108->regmap, TMP108_REG_CONF, config); + if (err < 0) { + dev_err(dev, "error writing config register: %d", err); + return err; + } + + tmp108->ready_time = jiffies; + if ((tmp108->orig_config & TMP108_CONF_MODE_MASK) == + TMP108_MODE_SHUTDOWN) + tmp108->ready_time += + msecs_to_jiffies(TMP108_CONVERSION_TIME_MS); + + err = devm_add_action_or_reset(dev, tmp108_restore_config, tmp108); + if (err) { + dev_err(dev, "add action or reset failed: %d", err); + return err; + } + + hwmon_dev = devm_hwmon_device_register_with_info(dev, client->name, + tmp108, + &tmp108_chip_info, + NULL); + return PTR_ERR_OR_ZERO(hwmon_dev); +} + +static int __maybe_unused tmp108_suspend(struct device *dev) +{ + struct tmp108 *tmp108 = dev_get_drvdata(dev); + + return regmap_update_bits(tmp108->regmap, TMP108_REG_CONF, + TMP108_CONF_MODE_MASK, TMP108_MODE_SHUTDOWN); +} + +static int __maybe_unused tmp108_resume(struct device *dev) +{ + struct tmp108 *tmp108 = dev_get_drvdata(dev); + int err; + + err = regmap_update_bits(tmp108->regmap, TMP108_REG_CONF, + TMP108_CONF_MODE_MASK, TMP108_MODE_CONTINUOUS); + tmp108->ready_time = jiffies + + msecs_to_jiffies(TMP108_CONVERSION_TIME_MS); + return err; +} + +static SIMPLE_DEV_PM_OPS(tmp108_dev_pm_ops, tmp108_suspend, tmp108_resume); + +static const struct i2c_device_id tmp108_i2c_ids[] = { + { "tmp108", 0 }, + { } +}; +MODULE_DEVICE_TABLE(i2c, tmp108_i2c_ids); + +#ifdef CONFIG_OF +static const struct of_device_id tmp108_of_ids[] = { + { .compatible = "ti,tmp108", }, + {} +}; +MODULE_DEVICE_TABLE(of, tmp108_of_ids); +#endif + +static struct i2c_driver tmp108_driver = { + .driver = { + .name = DRIVER_NAME, + .pm = &tmp108_dev_pm_ops, + .of_match_table = of_match_ptr(tmp108_of_ids), + }, + .probe = tmp108_probe, + .id_table = tmp108_i2c_ids, +}; + +module_i2c_driver(tmp108_driver); + +MODULE_AUTHOR("John Muir <john@jmuir.com>"); +MODULE_DESCRIPTION("Texas Instruments TMP108 temperature sensor driver"); +MODULE_LICENSE("GPL"); diff --git a/drivers/hwmon/via-cputemp.c b/drivers/hwmon/via-cputemp.c index ac91c07..d1f209a 100644 --- a/drivers/hwmon/via-cputemp.c +++ b/drivers/hwmon/via-cputemp.c @@ -220,7 +220,7 @@ struct pdev_entry { static LIST_HEAD(pdev_list); static DEFINE_MUTEX(pdev_list_mutex); -static int via_cputemp_device_add(unsigned int cpu) +static int via_cputemp_online(unsigned int cpu) { int err; struct platform_device *pdev; @@ -261,7 +261,7 @@ exit: return err; } -static void via_cputemp_device_remove(unsigned int cpu) +static int via_cputemp_down_prep(unsigned int cpu) { struct pdev_entry *p; @@ -272,33 +272,13 @@ static void via_cputemp_device_remove(unsigned int cpu) list_del(&p->list); mutex_unlock(&pdev_list_mutex); kfree(p); - return; + return 0; } } mutex_unlock(&pdev_list_mutex); + return 0; } -static int via_cputemp_cpu_callback(struct notifier_block *nfb, - unsigned long action, void *hcpu) -{ - unsigned int cpu = (unsigned long) hcpu; - - switch (action) { - case CPU_ONLINE: - case CPU_DOWN_FAILED: - via_cputemp_device_add(cpu); - break; - case CPU_DOWN_PREPARE: - via_cputemp_device_remove(cpu); - break; - } - return NOTIFY_OK; -} - -static struct notifier_block via_cputemp_cpu_notifier __refdata = { - .notifier_call = via_cputemp_cpu_callback, -}; - static const struct x86_cpu_id __initconst cputemp_ids[] = { { X86_VENDOR_CENTAUR, 6, 0xa, }, /* C7 A */ { X86_VENDOR_CENTAUR, 6, 0xd, }, /* C7 D */ @@ -307,9 +287,11 @@ static const struct x86_cpu_id __initconst cputemp_ids[] = { }; MODULE_DEVICE_TABLE(x86cpu, cputemp_ids); +static enum cpuhp_state via_temp_online; + static int __init via_cputemp_init(void) { - int i, err; + int err; if (!x86_match_cpu(cputemp_ids)) return -ENODEV; @@ -318,58 +300,33 @@ static int __init via_cputemp_init(void) if (err) goto exit; - cpu_notifier_register_begin(); - for_each_online_cpu(i) { - struct cpuinfo_x86 *c = &cpu_data(i); - - if (c->x86 != 6) - continue; - - if (c->x86_model < 0x0a) - continue; - - if (c->x86_model > 0x0f) { - pr_warn("Unknown CPU model 0x%x\n", c->x86_model); - continue; - } - - via_cputemp_device_add(i); - } + err = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "hwmon/via:online", + via_cputemp_online, via_cputemp_down_prep); + if (err < 0) + goto exit_driver_unreg; + via_temp_online = err; #ifndef CONFIG_HOTPLUG_CPU if (list_empty(&pdev_list)) { - cpu_notifier_register_done(); err = -ENODEV; - goto exit_driver_unreg; + goto exit_hp_unreg; } #endif - - __register_hotcpu_notifier(&via_cputemp_cpu_notifier); - cpu_notifier_register_done(); return 0; #ifndef CONFIG_HOTPLUG_CPU +exit_hp_unreg: + cpuhp_remove_state_nocalls(via_temp_online); +#endif exit_driver_unreg: platform_driver_unregister(&via_cputemp_driver); -#endif exit: return err; } static void __exit via_cputemp_exit(void) { - struct pdev_entry *p, *n; - - cpu_notifier_register_begin(); - __unregister_hotcpu_notifier(&via_cputemp_cpu_notifier); - mutex_lock(&pdev_list_mutex); - list_for_each_entry_safe(p, n, &pdev_list, list) { - platform_device_unregister(p->pdev); - list_del(&p->list); - kfree(p); - } - mutex_unlock(&pdev_list_mutex); - cpu_notifier_register_done(); + cpuhp_remove_state(via_temp_online); platform_driver_unregister(&via_cputemp_driver); } diff --git a/drivers/input/misc/xen-kbdfront.c b/drivers/input/misc/xen-kbdfront.c index 227fbd2..3900875 100644 --- a/drivers/input/misc/xen-kbdfront.c +++ b/drivers/input/misc/xen-kbdfront.c @@ -108,7 +108,8 @@ static irqreturn_t input_handler(int rq, void *dev_id) static int xenkbd_probe(struct xenbus_device *dev, const struct xenbus_device_id *id) { - int ret, i, abs; + int ret, i; + unsigned int abs; struct xenkbd_info *info; struct input_dev *kbd, *ptr; @@ -127,8 +128,7 @@ static int xenkbd_probe(struct xenbus_device *dev, if (!info->page) goto error_nomem; - if (xenbus_scanf(XBT_NIL, dev->otherend, "feature-abs-pointer", "%d", &abs) < 0) - abs = 0; + abs = xenbus_read_unsigned(dev->otherend, "feature-abs-pointer", 0); if (abs) { ret = xenbus_write(XBT_NIL, dev->nodename, "request-abs-pointer", "1"); @@ -322,11 +322,8 @@ static void xenkbd_backend_changed(struct xenbus_device *dev, case XenbusStateInitWait: InitWait: - ret = xenbus_scanf(XBT_NIL, info->xbdev->otherend, - "feature-abs-pointer", "%d", &val); - if (ret < 0) - val = 0; - if (val) { + if (xenbus_read_unsigned(info->xbdev->otherend, + "feature-abs-pointer", 0)) { ret = xenbus_write(XBT_NIL, info->xbdev->nodename, "request-abs-pointer", "1"); if (ret) diff --git a/drivers/irqchip/irq-gic-v3.c b/drivers/irqchip/irq-gic-v3.c index 19d642e..26e1d7f 100644 --- a/drivers/irqchip/irq-gic-v3.c +++ b/drivers/irqchip/irq-gic-v3.c @@ -120,11 +120,10 @@ static void gic_redist_wait_for_rwp(void) } #ifdef CONFIG_ARM64 -static DEFINE_STATIC_KEY_FALSE(is_cavium_thunderx); static u64 __maybe_unused gic_read_iar(void) { - if (static_branch_unlikely(&is_cavium_thunderx)) + if (cpus_have_const_cap(ARM64_WORKAROUND_CAVIUM_23154)) return gic_read_iar_cavium_thunderx(); else return gic_read_iar_common(); @@ -905,14 +904,6 @@ static const struct irq_domain_ops partition_domain_ops = { .select = gic_irq_domain_select, }; -static void gicv3_enable_quirks(void) -{ -#ifdef CONFIG_ARM64 - if (cpus_have_cap(ARM64_WORKAROUND_CAVIUM_23154)) - static_branch_enable(&is_cavium_thunderx); -#endif -} - static int __init gic_init_bases(void __iomem *dist_base, struct redist_region *rdist_regs, u32 nr_redist_regions, @@ -935,8 +926,6 @@ static int __init gic_init_bases(void __iomem *dist_base, gic_data.nr_redist_regions = nr_redist_regions; gic_data.redist_stride = redist_stride; - gicv3_enable_quirks(); - /* * Find out how many interrupts are supported. * The GIC only supports up to 1020 interrupt sources (SGI+PPI+SPI) diff --git a/drivers/message/fusion/mptbase.c b/drivers/message/fusion/mptbase.c index 89c7ed1..1e73064 100644 --- a/drivers/message/fusion/mptbase.c +++ b/drivers/message/fusion/mptbase.c @@ -2585,10 +2585,7 @@ mpt_do_ioc_recovery(MPT_ADAPTER *ioc, u32 reason, int sleepFlag) (void) GetLanConfigPages(ioc); a = (u8*)&ioc->lan_cnfg_page1.HardwareAddressLow; dprintk(ioc, printk(MYIOC_s_DEBUG_FMT - "LanAddr = %02X:%02X:%02X" - ":%02X:%02X:%02X\n", - ioc->name, a[5], a[4], - a[3], a[2], a[1], a[0])); + "LanAddr = %pMR\n", ioc->name, a)); } break; @@ -2868,21 +2865,21 @@ MptDisplayIocCapabilities(MPT_ADAPTER *ioc) printk(KERN_INFO "%s: ", ioc->name); if (ioc->prod_name) - printk("%s: ", ioc->prod_name); - printk("Capabilities={"); + pr_cont("%s: ", ioc->prod_name); + pr_cont("Capabilities={"); if (ioc->pfacts[0].ProtocolFlags & MPI_PORTFACTS_PROTOCOL_INITIATOR) { - printk("Initiator"); + pr_cont("Initiator"); i++; } if (ioc->pfacts[0].ProtocolFlags & MPI_PORTFACTS_PROTOCOL_TARGET) { - printk("%sTarget", i ? "," : ""); + pr_cont("%sTarget", i ? "," : ""); i++; } if (ioc->pfacts[0].ProtocolFlags & MPI_PORTFACTS_PROTOCOL_LAN) { - printk("%sLAN", i ? "," : ""); + pr_cont("%sLAN", i ? "," : ""); i++; } @@ -2891,12 +2888,12 @@ MptDisplayIocCapabilities(MPT_ADAPTER *ioc) * This would probably evoke more questions than it's worth */ if (ioc->pfacts[0].ProtocolFlags & MPI_PORTFACTS_PROTOCOL_TARGET) { - printk("%sLogBusAddr", i ? "," : ""); + pr_cont("%sLogBusAddr", i ? "," : ""); i++; } #endif - printk("}\n"); + pr_cont("}\n"); } /*=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=*/ @@ -6783,8 +6780,7 @@ static int mpt_iocinfo_proc_show(struct seq_file *m, void *v) if (ioc->bus_type == FC) { if (ioc->pfacts[p].ProtocolFlags & MPI_PORTFACTS_PROTOCOL_LAN) { u8 *a = (u8*)&ioc->lan_cnfg_page1.HardwareAddressLow; - seq_printf(m, " LanAddr = %02X:%02X:%02X:%02X:%02X:%02X\n", - a[5], a[4], a[3], a[2], a[1], a[0]); + seq_printf(m, " LanAddr = %pMR\n", a); } seq_printf(m, " WWN = %08X%08X:%08X%08X\n", ioc->fc_port_page0[p].WWNN.High, @@ -6861,8 +6857,7 @@ mpt_print_ioc_summary(MPT_ADAPTER *ioc, char *buffer, int *size, int len, int sh if (showlan && (ioc->pfacts[0].ProtocolFlags & MPI_PORTFACTS_PROTOCOL_LAN)) { u8 *a = (u8*)&ioc->lan_cnfg_page1.HardwareAddressLow; - y += sprintf(buffer+len+y, ", LanAddr=%02X:%02X:%02X:%02X:%02X:%02X", - a[5], a[4], a[3], a[2], a[1], a[0]); + y += sprintf(buffer+len+y, ", LanAddr=%pMR", a); } y += sprintf(buffer+len+y, ", IRQ=%d", ioc->pci_irq); @@ -6896,8 +6891,7 @@ static void seq_mpt_print_ioc_summary(MPT_ADAPTER *ioc, struct seq_file *m, int if (showlan && (ioc->pfacts[0].ProtocolFlags & MPI_PORTFACTS_PROTOCOL_LAN)) { u8 *a = (u8*)&ioc->lan_cnfg_page1.HardwareAddressLow; - seq_printf(m, ", LanAddr=%02X:%02X:%02X:%02X:%02X:%02X", - a[5], a[4], a[3], a[2], a[1], a[0]); + seq_printf(m, ", LanAddr=%pMR", a); } seq_printf(m, ", IRQ=%d", ioc->pci_irq); diff --git a/drivers/message/fusion/mptscsih.c b/drivers/message/fusion/mptscsih.c index 6c9fc11..08a807d 100644 --- a/drivers/message/fusion/mptscsih.c +++ b/drivers/message/fusion/mptscsih.c @@ -1366,15 +1366,10 @@ mptscsih_qcmd(struct scsi_cmnd *SCpnt) /* Default to untagged. Once a target structure has been allocated, * use the Inquiry data to determine if device supports tagged. */ - if ((vdevice->vtarget->tflags & MPT_TARGET_FLAGS_Q_YES) - && (SCpnt->device->tagged_supported)) { + if ((vdevice->vtarget->tflags & MPT_TARGET_FLAGS_Q_YES) && + SCpnt->device->tagged_supported) scsictl = scsidir | MPI_SCSIIO_CONTROL_SIMPLEQ; - if (SCpnt->request && SCpnt->request->ioprio) { - if (((SCpnt->request->ioprio & 0x7) == 1) || - !(SCpnt->request->ioprio & 0x7)) - scsictl |= MPI_SCSIIO_CONTROL_HEADOFQ; - } - } else + else scsictl = scsidir | MPI_SCSIIO_CONTROL_UNTAGGED; diff --git a/drivers/mmc/Kconfig b/drivers/mmc/Kconfig index f2eeb38..7e803fc4 100644 --- a/drivers/mmc/Kconfig +++ b/drivers/mmc/Kconfig @@ -23,8 +23,6 @@ if MMC source "drivers/mmc/core/Kconfig" -source "drivers/mmc/card/Kconfig" - source "drivers/mmc/host/Kconfig" endif # MMC diff --git a/drivers/mmc/Makefile b/drivers/mmc/Makefile index 400756e..416b6d1 100644 --- a/drivers/mmc/Makefile +++ b/drivers/mmc/Makefile @@ -5,5 +5,4 @@ subdir-ccflags-$(CONFIG_MMC_DEBUG) := -DDEBUG obj-$(CONFIG_MMC) += core/ -obj-$(CONFIG_MMC) += card/ obj-$(subst m,y,$(CONFIG_MMC)) += host/ diff --git a/drivers/mmc/card/Kconfig b/drivers/mmc/card/Kconfig deleted file mode 100644 index 5562308..0000000 --- a/drivers/mmc/card/Kconfig +++ /dev/null @@ -1,70 +0,0 @@ -# -# MMC/SD card drivers -# - -comment "MMC/SD/SDIO Card Drivers" - -config MMC_BLOCK - tristate "MMC block device driver" - depends on BLOCK - default y - help - Say Y here to enable the MMC block device driver support. - This provides a block device driver, which you can use to - mount the filesystem. Almost everyone wishing MMC support - should say Y or M here. - -config MMC_BLOCK_MINORS - int "Number of minors per block device" - depends on MMC_BLOCK - range 4 256 - default 8 - help - Number of minors per block device. One is needed for every - partition on the disk (plus one for the whole disk). - - Number of total MMC minors available is 256, so your number - of supported block devices will be limited to 256 divided - by this number. - - Default is 8 to be backwards compatible with previous - hardwired device numbering. - - If unsure, say 8 here. - -config MMC_BLOCK_BOUNCE - bool "Use bounce buffer for simple hosts" - depends on MMC_BLOCK - default y - help - SD/MMC is a high latency protocol where it is crucial to - send large requests in order to get high performance. Many - controllers, however, are restricted to continuous memory - (i.e. they can't do scatter-gather), something the kernel - rarely can provide. - - Say Y here to help these restricted hosts by bouncing - requests back and forth from a large buffer. You will get - a big performance gain at the cost of up to 64 KiB of - physical memory. - - If unsure, say Y here. - -config SDIO_UART - tristate "SDIO UART/GPS class support" - depends on TTY - help - SDIO function driver for SDIO cards that implements the UART - class, as well as the GPS class which appears like a UART. - -config MMC_TEST - tristate "MMC host test driver" - help - Development driver that performs a series of reads and writes - to a memory card in order to expose certain well known bugs - in host controllers. The tests are executed by writing to the - "test" file in debugfs under each card. Note that whatever is - on your card will be overwritten by these tests. - - This driver is only of interest to those developing or - testing a host driver. Most people should say N here. diff --git a/drivers/mmc/card/Makefile b/drivers/mmc/card/Makefile deleted file mode 100644 index c73b406..0000000 --- a/drivers/mmc/card/Makefile +++ /dev/null @@ -1,10 +0,0 @@ -# -# Makefile for MMC/SD card drivers -# - -obj-$(CONFIG_MMC_BLOCK) += mmc_block.o -mmc_block-objs := block.o queue.o -obj-$(CONFIG_MMC_TEST) += mmc_test.o - -obj-$(CONFIG_SDIO_UART) += sdio_uart.o - diff --git a/drivers/mmc/core/Kconfig b/drivers/mmc/core/Kconfig index 250f223..cdfa852 100644 --- a/drivers/mmc/core/Kconfig +++ b/drivers/mmc/core/Kconfig @@ -22,3 +22,69 @@ config PWRSEQ_SIMPLE This driver can also be built as a module. If so, the module will be called pwrseq_simple. + +config MMC_BLOCK + tristate "MMC block device driver" + depends on BLOCK + default y + help + Say Y here to enable the MMC block device driver support. + This provides a block device driver, which you can use to + mount the filesystem. Almost everyone wishing MMC support + should say Y or M here. + +config MMC_BLOCK_MINORS + int "Number of minors per block device" + depends on MMC_BLOCK + range 4 256 + default 8 + help + Number of minors per block device. One is needed for every + partition on the disk (plus one for the whole disk). + + Number of total MMC minors available is 256, so your number + of supported block devices will be limited to 256 divided + by this number. + + Default is 8 to be backwards compatible with previous + hardwired device numbering. + + If unsure, say 8 here. + +config MMC_BLOCK_BOUNCE + bool "Use bounce buffer for simple hosts" + depends on MMC_BLOCK + default y + help + SD/MMC is a high latency protocol where it is crucial to + send large requests in order to get high performance. Many + controllers, however, are restricted to continuous memory + (i.e. they can't do scatter-gather), something the kernel + rarely can provide. + + Say Y here to help these restricted hosts by bouncing + requests back and forth from a large buffer. You will get + a big performance gain at the cost of up to 64 KiB of + physical memory. + + If unsure, say Y here. + +config SDIO_UART + tristate "SDIO UART/GPS class support" + depends on TTY + help + SDIO function driver for SDIO cards that implements the UART + class, as well as the GPS class which appears like a UART. + +config MMC_TEST + tristate "MMC host test driver" + help + Development driver that performs a series of reads and writes + to a memory card in order to expose certain well known bugs + in host controllers. The tests are executed by writing to the + "test" file in debugfs under each card. Note that whatever is + on your card will be overwritten by these tests. + + This driver is only of interest to those developing or + testing a host driver. Most people should say N here. + diff --git a/drivers/mmc/core/Makefile b/drivers/mmc/core/Makefile index f007151..b2a257d 100644 --- a/drivers/mmc/core/Makefile +++ b/drivers/mmc/core/Makefile @@ -12,3 +12,7 @@ mmc_core-$(CONFIG_OF) += pwrseq.o obj-$(CONFIG_PWRSEQ_SIMPLE) += pwrseq_simple.o obj-$(CONFIG_PWRSEQ_EMMC) += pwrseq_emmc.o mmc_core-$(CONFIG_DEBUG_FS) += debugfs.o +obj-$(CONFIG_MMC_BLOCK) += mmc_block.o +mmc_block-objs := block.o queue.o +obj-$(CONFIG_MMC_TEST) += mmc_test.o +obj-$(CONFIG_SDIO_UART) += sdio_uart.o diff --git a/drivers/mmc/card/block.c b/drivers/mmc/core/block.c index bab3f07..bab3f07 100644 --- a/drivers/mmc/card/block.c +++ b/drivers/mmc/core/block.c diff --git a/drivers/mmc/card/block.h b/drivers/mmc/core/block.h index cdabb2e..cdabb2e 100644 --- a/drivers/mmc/card/block.h +++ b/drivers/mmc/core/block.h diff --git a/drivers/mmc/card/mmc_test.c b/drivers/mmc/core/mmc_test.c index ec1d1c4..3ab6e52 100644 --- a/drivers/mmc/card/mmc_test.c +++ b/drivers/mmc/core/mmc_test.c @@ -1,6 +1,4 @@ /* - * linux/drivers/mmc/card/mmc_test.c - * * Copyright 2007-2008 Pierre Ossman * * This program is free software; you can redistribute it and/or modify diff --git a/drivers/mmc/card/queue.c b/drivers/mmc/core/queue.c index 6ae6bfb..a6496d8 100644 --- a/drivers/mmc/card/queue.c +++ b/drivers/mmc/core/queue.c @@ -1,6 +1,4 @@ /* - * linux/drivers/mmc/card/queue.c - * * Copyright (C) 2003 Russell King, All Rights Reserved. * Copyright 2006-2007 Pierre Ossman * diff --git a/drivers/mmc/card/queue.h b/drivers/mmc/core/queue.h index dac8c3d..dac8c3d 100644 --- a/drivers/mmc/card/queue.h +++ b/drivers/mmc/core/queue.h diff --git a/drivers/mmc/card/sdio_uart.c b/drivers/mmc/core/sdio_uart.c index 491c187..d3c91f4 100644 --- a/drivers/mmc/card/sdio_uart.c +++ b/drivers/mmc/core/sdio_uart.c @@ -1,5 +1,5 @@ /* - * linux/drivers/mmc/card/sdio_uart.c - SDIO UART/GPS driver + * SDIO UART/GPS driver * * Based on drivers/serial/8250.c and drivers/serial/serial_core.c * by Russell King. diff --git a/drivers/net/xen-netback/xenbus.c b/drivers/net/xen-netback/xenbus.c index 55a4488..3124eae 100644 --- a/drivers/net/xen-netback/xenbus.c +++ b/drivers/net/xen-netback/xenbus.c @@ -785,12 +785,9 @@ static void xen_mcast_ctrl_changed(struct xenbus_watch *watch, struct xenvif *vif = container_of(watch, struct xenvif, mcast_ctrl_watch); struct xenbus_device *dev = xenvif_to_xenbus_device(vif); - int val; - if (xenbus_scanf(XBT_NIL, dev->otherend, - "request-multicast-control", "%d", &val) < 0) - val = 0; - vif->multicast_control = !!val; + vif->multicast_control = !!xenbus_read_unsigned(dev->otherend, + "request-multicast-control", 0); } static int xen_register_mcast_ctrl_watch(struct xenbus_device *dev, @@ -934,14 +931,11 @@ static void connect(struct backend_info *be) /* Check whether the frontend requested multiple queues * and read the number requested. */ - err = xenbus_scanf(XBT_NIL, dev->otherend, - "multi-queue-num-queues", - "%u", &requested_num_queues); - if (err < 0) { - requested_num_queues = 1; /* Fall back to single queue */ - } else if (requested_num_queues > xenvif_max_queues) { + requested_num_queues = xenbus_read_unsigned(dev->otherend, + "multi-queue-num-queues", 1); + if (requested_num_queues > xenvif_max_queues) { /* buggy or malicious guest */ - xenbus_dev_fatal(dev, err, + xenbus_dev_fatal(dev, -EINVAL, "guest requested %u queues, exceeding the maximum of %u.", requested_num_queues, xenvif_max_queues); return; @@ -1134,7 +1128,7 @@ static int read_xenbus_vif_flags(struct backend_info *be) struct xenvif *vif = be->vif; struct xenbus_device *dev = be->dev; unsigned int rx_copy; - int err, val; + int err; err = xenbus_scanf(XBT_NIL, dev->otherend, "request-rx-copy", "%u", &rx_copy); @@ -1150,10 +1144,7 @@ static int read_xenbus_vif_flags(struct backend_info *be) if (!rx_copy) return -EOPNOTSUPP; - if (xenbus_scanf(XBT_NIL, dev->otherend, - "feature-rx-notify", "%d", &val) < 0) - val = 0; - if (!val) { + if (!xenbus_read_unsigned(dev->otherend, "feature-rx-notify", 0)) { /* - Reduce drain timeout to poll more frequently for * Rx requests. * - Disable Rx stall detection. @@ -1162,34 +1153,21 @@ static int read_xenbus_vif_flags(struct backend_info *be) be->vif->stall_timeout = 0; } - if (xenbus_scanf(XBT_NIL, dev->otherend, "feature-sg", - "%d", &val) < 0) - val = 0; - vif->can_sg = !!val; + vif->can_sg = !!xenbus_read_unsigned(dev->otherend, "feature-sg", 0); vif->gso_mask = 0; - if (xenbus_scanf(XBT_NIL, dev->otherend, "feature-gso-tcpv4", - "%d", &val) < 0) - val = 0; - if (val) + if (xenbus_read_unsigned(dev->otherend, "feature-gso-tcpv4", 0)) vif->gso_mask |= GSO_BIT(TCPV4); - if (xenbus_scanf(XBT_NIL, dev->otherend, "feature-gso-tcpv6", - "%d", &val) < 0) - val = 0; - if (val) + if (xenbus_read_unsigned(dev->otherend, "feature-gso-tcpv6", 0)) vif->gso_mask |= GSO_BIT(TCPV6); - if (xenbus_scanf(XBT_NIL, dev->otherend, "feature-no-csum-offload", - "%d", &val) < 0) - val = 0; - vif->ip_csum = !val; + vif->ip_csum = !xenbus_read_unsigned(dev->otherend, + "feature-no-csum-offload", 0); - if (xenbus_scanf(XBT_NIL, dev->otherend, "feature-ipv6-csum-offload", - "%d", &val) < 0) - val = 0; - vif->ipv6_csum = !!val; + vif->ipv6_csum = !!xenbus_read_unsigned(dev->otherend, + "feature-ipv6-csum-offload", 0); return 0; } diff --git a/drivers/net/xen-netfront.c b/drivers/net/xen-netfront.c index e085c8c..a479cd9 100644 --- a/drivers/net/xen-netfront.c +++ b/drivers/net/xen-netfront.c @@ -1169,43 +1169,23 @@ static netdev_features_t xennet_fix_features(struct net_device *dev, netdev_features_t features) { struct netfront_info *np = netdev_priv(dev); - int val; - if (features & NETIF_F_SG) { - if (xenbus_scanf(XBT_NIL, np->xbdev->otherend, "feature-sg", - "%d", &val) < 0) - val = 0; + if (features & NETIF_F_SG && + !xenbus_read_unsigned(np->xbdev->otherend, "feature-sg", 0)) + features &= ~NETIF_F_SG; - if (!val) - features &= ~NETIF_F_SG; - } - - if (features & NETIF_F_IPV6_CSUM) { - if (xenbus_scanf(XBT_NIL, np->xbdev->otherend, - "feature-ipv6-csum-offload", "%d", &val) < 0) - val = 0; - - if (!val) - features &= ~NETIF_F_IPV6_CSUM; - } - - if (features & NETIF_F_TSO) { - if (xenbus_scanf(XBT_NIL, np->xbdev->otherend, - "feature-gso-tcpv4", "%d", &val) < 0) - val = 0; + if (features & NETIF_F_IPV6_CSUM && + !xenbus_read_unsigned(np->xbdev->otherend, + "feature-ipv6-csum-offload", 0)) + features &= ~NETIF_F_IPV6_CSUM; - if (!val) - features &= ~NETIF_F_TSO; - } + if (features & NETIF_F_TSO && + !xenbus_read_unsigned(np->xbdev->otherend, "feature-gso-tcpv4", 0)) + features &= ~NETIF_F_TSO; - if (features & NETIF_F_TSO6) { - if (xenbus_scanf(XBT_NIL, np->xbdev->otherend, - "feature-gso-tcpv6", "%d", &val) < 0) - val = 0; - - if (!val) - features &= ~NETIF_F_TSO6; - } + if (features & NETIF_F_TSO6 && + !xenbus_read_unsigned(np->xbdev->otherend, "feature-gso-tcpv6", 0)) + features &= ~NETIF_F_TSO6; return features; } @@ -1823,18 +1803,13 @@ static int talk_to_netback(struct xenbus_device *dev, info->netdev->irq = 0; /* Check if backend supports multiple queues */ - err = xenbus_scanf(XBT_NIL, info->xbdev->otherend, - "multi-queue-max-queues", "%u", &max_queues); - if (err < 0) - max_queues = 1; + max_queues = xenbus_read_unsigned(info->xbdev->otherend, + "multi-queue-max-queues", 1); num_queues = min(max_queues, xennet_max_queues); /* Check feature-split-event-channels */ - err = xenbus_scanf(XBT_NIL, info->xbdev->otherend, - "feature-split-event-channels", "%u", - &feature_split_evtchn); - if (err < 0) - feature_split_evtchn = 0; + feature_split_evtchn = xenbus_read_unsigned(info->xbdev->otherend, + "feature-split-event-channels", 0); /* Read mac addr. */ err = xen_net_read_mac(dev, info->netdev->dev_addr); @@ -1968,16 +1943,10 @@ static int xennet_connect(struct net_device *dev) struct netfront_info *np = netdev_priv(dev); unsigned int num_queues = 0; int err; - unsigned int feature_rx_copy; unsigned int j = 0; struct netfront_queue *queue = NULL; - err = xenbus_scanf(XBT_NIL, np->xbdev->otherend, - "feature-rx-copy", "%u", &feature_rx_copy); - if (err != 1) - feature_rx_copy = 0; - - if (!feature_rx_copy) { + if (!xenbus_read_unsigned(np->xbdev->otherend, "feature-rx-copy", 0)) { dev_info(&dev->dev, "backend does not support copying receive path\n"); return -ENODEV; diff --git a/drivers/nvme/host/core.c b/drivers/nvme/host/core.c index 35b3fee..b40cfb0 100644 --- a/drivers/nvme/host/core.c +++ b/drivers/nvme/host/core.c @@ -263,21 +263,6 @@ static inline int nvme_setup_discard(struct nvme_ns *ns, struct request *req, return BLK_MQ_RQ_QUEUE_OK; } -static inline void nvme_setup_write_zeroes(struct nvme_ns *ns, - struct request *req, struct nvme_command *cmnd) -{ - struct nvme_write_zeroes_cmd *write_zeroes = &cmnd->write_zeroes; - - memset(cmnd, 0, sizeof(*cmnd)); - write_zeroes->opcode = nvme_cmd_write_zeroes; - write_zeroes->nsid = cpu_to_le32(ns->ns_id); - write_zeroes->slba = - cpu_to_le64(nvme_block_nr(ns, blk_rq_pos(req))); - write_zeroes->length = - cpu_to_le16((blk_rq_bytes(req) >> ns->lba_shift) - 1); - write_zeroes->control = 0; -} - static inline void nvme_setup_rw(struct nvme_ns *ns, struct request *req, struct nvme_command *cmnd) { @@ -330,8 +315,6 @@ int nvme_setup_cmd(struct nvme_ns *ns, struct request *req, nvme_setup_flush(ns, cmd); else if (req_op(req) == REQ_OP_DISCARD) ret = nvme_setup_discard(ns, req, cmd); - else if (req_op(req) == REQ_OP_WRITE_ZEROES) - nvme_setup_write_zeroes(ns, req, cmd); else nvme_setup_rw(ns, req, cmd); @@ -952,10 +935,6 @@ static void __nvme_revalidate_disk(struct gendisk *disk, struct nvme_id_ns *id) if (ns->ctrl->oncs & NVME_CTRL_ONCS_DSM) nvme_config_discard(ns); - if (ns->ctrl->oncs & NVME_CTRL_ONCS_WRITE_ZEROES) - blk_queue_max_write_zeroes_sectors(ns->queue, - ((u32)(USHRT_MAX + 1) * bs) >> 9); - blk_mq_unfreeze_queue(disk->queue); } diff --git a/drivers/nvme/host/pci.c b/drivers/nvme/host/pci.c index 3ba38f4..d6e6bce 100644 --- a/drivers/nvme/host/pci.c +++ b/drivers/nvme/host/pci.c @@ -2086,9 +2086,6 @@ static const struct pci_error_handlers nvme_err_handler = { .reset_notify = nvme_reset_notify, }; -/* Move to pci_ids.h later */ -#define PCI_CLASS_STORAGE_EXPRESS 0x010802 - static const struct pci_device_id nvme_id_table[] = { { PCI_VDEVICE(INTEL, 0x0953), .driver_data = NVME_QUIRK_STRIPE_SIZE | diff --git a/drivers/nvme/target/configfs.c b/drivers/nvme/target/configfs.c index d0f60c3..6f50741 100644 --- a/drivers/nvme/target/configfs.c +++ b/drivers/nvme/target/configfs.c @@ -480,7 +480,7 @@ out_free_link: return ret; } -static int nvmet_port_subsys_drop_link(struct config_item *parent, +static void nvmet_port_subsys_drop_link(struct config_item *parent, struct config_item *target) { struct nvmet_port *port = to_nvmet_port(parent->ci_parent); @@ -493,7 +493,7 @@ static int nvmet_port_subsys_drop_link(struct config_item *parent, goto found; } up_write(&nvmet_config_sem); - return -EINVAL; + return; found: list_del(&p->entry); @@ -502,7 +502,6 @@ found: nvmet_disable_port(port); up_write(&nvmet_config_sem); kfree(p); - return 0; } static struct configfs_item_operations nvmet_port_subsys_item_ops = { @@ -556,7 +555,7 @@ out_free_link: return ret; } -static int nvmet_allowed_hosts_drop_link(struct config_item *parent, +static void nvmet_allowed_hosts_drop_link(struct config_item *parent, struct config_item *target) { struct nvmet_subsys *subsys = to_subsys(parent->ci_parent); @@ -569,14 +568,13 @@ static int nvmet_allowed_hosts_drop_link(struct config_item *parent, goto found; } up_write(&nvmet_config_sem); - return -EINVAL; + return; found: list_del(&p->entry); nvmet_genctr++; up_write(&nvmet_config_sem); kfree(p); - return 0; } static struct configfs_item_operations nvmet_allowed_hosts_item_ops = { diff --git a/drivers/pci/hotplug/s390_pci_hpc.c b/drivers/pci/hotplug/s390_pci_hpc.c index 50b8b7d..530d0e4 100644 --- a/drivers/pci/hotplug/s390_pci_hpc.c +++ b/drivers/pci/hotplug/s390_pci_hpc.c @@ -5,12 +5,13 @@ * * Author(s): * Jan Glauber <jang@linux.vnet.ibm.com> + * + * License: GPL */ #define KMSG_COMPONENT "zpci" #define pr_fmt(fmt) KMSG_COMPONENT ": " fmt -#include <linux/module.h> #include <linux/kernel.h> #include <linux/slab.h> #include <linux/pci.h> @@ -21,10 +22,6 @@ #define SLOT_NAME_SIZE 10 static LIST_HEAD(s390_hotplug_slot_list); -MODULE_AUTHOR("Jan Glauber <jang@linux.vnet.ibm.com"); -MODULE_DESCRIPTION("Hot Plug PCI Controller for System z"); -MODULE_LICENSE("GPL"); - static int zpci_fn_configured(enum zpci_state state) { return state == ZPCI_FN_STATE_CONFIGURED || diff --git a/drivers/pci/xen-pcifront.c b/drivers/pci/xen-pcifront.c index d6ff5e8..8fc2e95 100644 --- a/drivers/pci/xen-pcifront.c +++ b/drivers/pci/xen-pcifront.c @@ -1038,10 +1038,8 @@ static int pcifront_detach_devices(struct pcifront_device *pdev) err = -ENOMEM; goto out; } - err = xenbus_scanf(XBT_NIL, pdev->xdev->otherend, str, "%d", - &state); - if (err != 1) - state = XenbusStateUnknown; + state = xenbus_read_unsigned(pdev->xdev->otherend, str, + XenbusStateUnknown); if (state != XenbusStateClosing) continue; diff --git a/drivers/phy/phy-qcom-ufs-i.h b/drivers/phy/phy-qcom-ufs-i.h index 2bd5ce4..d505d98 100644 --- a/drivers/phy/phy-qcom-ufs-i.h +++ b/drivers/phy/phy-qcom-ufs-i.h @@ -141,11 +141,8 @@ struct ufs_qcom_phy_specific_ops { struct ufs_qcom_phy *get_ufs_qcom_phy(struct phy *generic_phy); int ufs_qcom_phy_power_on(struct phy *generic_phy); int ufs_qcom_phy_power_off(struct phy *generic_phy); -int ufs_qcom_phy_exit(struct phy *generic_phy); -int ufs_qcom_phy_init_clks(struct phy *generic_phy, - struct ufs_qcom_phy *phy_common); -int ufs_qcom_phy_init_vregulators(struct phy *generic_phy, - struct ufs_qcom_phy *phy_common); +int ufs_qcom_phy_init_clks(struct ufs_qcom_phy *phy_common); +int ufs_qcom_phy_init_vregulators(struct ufs_qcom_phy *phy_common); int ufs_qcom_phy_remove(struct phy *generic_phy, struct ufs_qcom_phy *ufs_qcom_phy); struct phy *ufs_qcom_phy_generic_probe(struct platform_device *pdev, diff --git a/drivers/phy/phy-qcom-ufs-qmp-14nm.c b/drivers/phy/phy-qcom-ufs-qmp-14nm.c index 6ee5149..c71c847 100644 --- a/drivers/phy/phy-qcom-ufs-qmp-14nm.c +++ b/drivers/phy/phy-qcom-ufs-qmp-14nm.c @@ -44,30 +44,12 @@ void ufs_qcom_phy_qmp_14nm_advertise_quirks(struct ufs_qcom_phy *phy_common) static int ufs_qcom_phy_qmp_14nm_init(struct phy *generic_phy) { - struct ufs_qcom_phy_qmp_14nm *phy = phy_get_drvdata(generic_phy); - struct ufs_qcom_phy *phy_common = &phy->common_cfg; - int err; - - err = ufs_qcom_phy_init_clks(generic_phy, phy_common); - if (err) { - dev_err(phy_common->dev, "%s: ufs_qcom_phy_init_clks() failed %d\n", - __func__, err); - goto out; - } - - err = ufs_qcom_phy_init_vregulators(generic_phy, phy_common); - if (err) { - dev_err(phy_common->dev, "%s: ufs_qcom_phy_init_vregulators() failed %d\n", - __func__, err); - goto out; - } - phy_common->vdda_phy.max_uV = UFS_PHY_VDDA_PHY_UV; - phy_common->vdda_phy.min_uV = UFS_PHY_VDDA_PHY_UV; - - ufs_qcom_phy_qmp_14nm_advertise_quirks(phy_common); + return 0; +} -out: - return err; +static int ufs_qcom_phy_qmp_14nm_exit(struct phy *generic_phy) +{ + return 0; } static @@ -117,7 +99,7 @@ static int ufs_qcom_phy_qmp_14nm_is_pcs_ready(struct ufs_qcom_phy *phy_common) static const struct phy_ops ufs_qcom_phy_qmp_14nm_phy_ops = { .init = ufs_qcom_phy_qmp_14nm_init, - .exit = ufs_qcom_phy_exit, + .exit = ufs_qcom_phy_qmp_14nm_exit, .power_on = ufs_qcom_phy_power_on, .power_off = ufs_qcom_phy_power_off, .owner = THIS_MODULE, @@ -136,6 +118,7 @@ static int ufs_qcom_phy_qmp_14nm_probe(struct platform_device *pdev) struct device *dev = &pdev->dev; struct phy *generic_phy; struct ufs_qcom_phy_qmp_14nm *phy; + struct ufs_qcom_phy *phy_common; int err = 0; phy = devm_kzalloc(dev, sizeof(*phy), GFP_KERNEL); @@ -143,8 +126,9 @@ static int ufs_qcom_phy_qmp_14nm_probe(struct platform_device *pdev) err = -ENOMEM; goto out; } + phy_common = &phy->common_cfg; - generic_phy = ufs_qcom_phy_generic_probe(pdev, &phy->common_cfg, + generic_phy = ufs_qcom_phy_generic_probe(pdev, phy_common, &ufs_qcom_phy_qmp_14nm_phy_ops, &phy_14nm_ops); if (!generic_phy) { @@ -154,39 +138,43 @@ static int ufs_qcom_phy_qmp_14nm_probe(struct platform_device *pdev) goto out; } - phy_set_drvdata(generic_phy, phy); + err = ufs_qcom_phy_init_clks(phy_common); + if (err) { + dev_err(phy_common->dev, + "%s: ufs_qcom_phy_init_clks() failed %d\n", + __func__, err); + goto out; + } - strlcpy(phy->common_cfg.name, UFS_PHY_NAME, - sizeof(phy->common_cfg.name)); + err = ufs_qcom_phy_init_vregulators(phy_common); + if (err) { + dev_err(phy_common->dev, + "%s: ufs_qcom_phy_init_vregulators() failed %d\n", + __func__, err); + goto out; + } + phy_common->vdda_phy.max_uV = UFS_PHY_VDDA_PHY_UV; + phy_common->vdda_phy.min_uV = UFS_PHY_VDDA_PHY_UV; -out: - return err; -} + ufs_qcom_phy_qmp_14nm_advertise_quirks(phy_common); -static int ufs_qcom_phy_qmp_14nm_remove(struct platform_device *pdev) -{ - struct device *dev = &pdev->dev; - struct phy *generic_phy = to_phy(dev); - struct ufs_qcom_phy *ufs_qcom_phy = get_ufs_qcom_phy(generic_phy); - int err = 0; + phy_set_drvdata(generic_phy, phy); - err = ufs_qcom_phy_remove(generic_phy, ufs_qcom_phy); - if (err) - dev_err(dev, "%s: ufs_qcom_phy_remove failed = %d\n", - __func__, err); + strlcpy(phy_common->name, UFS_PHY_NAME, sizeof(phy_common->name)); +out: return err; } static const struct of_device_id ufs_qcom_phy_qmp_14nm_of_match[] = { {.compatible = "qcom,ufs-phy-qmp-14nm"}, + {.compatible = "qcom,msm8996-ufs-phy-qmp-14nm"}, {}, }; MODULE_DEVICE_TABLE(of, ufs_qcom_phy_qmp_14nm_of_match); static struct platform_driver ufs_qcom_phy_qmp_14nm_driver = { .probe = ufs_qcom_phy_qmp_14nm_probe, - .remove = ufs_qcom_phy_qmp_14nm_remove, .driver = { .of_match_table = ufs_qcom_phy_qmp_14nm_of_match, .name = "ufs_qcom_phy_qmp_14nm", diff --git a/drivers/phy/phy-qcom-ufs-qmp-20nm.c b/drivers/phy/phy-qcom-ufs-qmp-20nm.c index 770087a..1a26a64 100644 --- a/drivers/phy/phy-qcom-ufs-qmp-20nm.c +++ b/drivers/phy/phy-qcom-ufs-qmp-20nm.c @@ -63,28 +63,12 @@ void ufs_qcom_phy_qmp_20nm_advertise_quirks(struct ufs_qcom_phy *phy_common) static int ufs_qcom_phy_qmp_20nm_init(struct phy *generic_phy) { - struct ufs_qcom_phy_qmp_20nm *phy = phy_get_drvdata(generic_phy); - struct ufs_qcom_phy *phy_common = &phy->common_cfg; - int err = 0; - - err = ufs_qcom_phy_init_clks(generic_phy, phy_common); - if (err) { - dev_err(phy_common->dev, "%s: ufs_qcom_phy_init_clks() failed %d\n", - __func__, err); - goto out; - } - - err = ufs_qcom_phy_init_vregulators(generic_phy, phy_common); - if (err) { - dev_err(phy_common->dev, "%s: ufs_qcom_phy_init_vregulators() failed %d\n", - __func__, err); - goto out; - } - - ufs_qcom_phy_qmp_20nm_advertise_quirks(phy_common); + return 0; +} -out: - return err; +static int ufs_qcom_phy_qmp_20nm_exit(struct phy *generic_phy) +{ + return 0; } static @@ -173,7 +157,7 @@ static int ufs_qcom_phy_qmp_20nm_is_pcs_ready(struct ufs_qcom_phy *phy_common) static const struct phy_ops ufs_qcom_phy_qmp_20nm_phy_ops = { .init = ufs_qcom_phy_qmp_20nm_init, - .exit = ufs_qcom_phy_exit, + .exit = ufs_qcom_phy_qmp_20nm_exit, .power_on = ufs_qcom_phy_power_on, .power_off = ufs_qcom_phy_power_off, .owner = THIS_MODULE, @@ -192,6 +176,7 @@ static int ufs_qcom_phy_qmp_20nm_probe(struct platform_device *pdev) struct device *dev = &pdev->dev; struct phy *generic_phy; struct ufs_qcom_phy_qmp_20nm *phy; + struct ufs_qcom_phy *phy_common; int err = 0; phy = devm_kzalloc(dev, sizeof(*phy), GFP_KERNEL); @@ -199,8 +184,9 @@ static int ufs_qcom_phy_qmp_20nm_probe(struct platform_device *pdev) err = -ENOMEM; goto out; } + phy_common = &phy->common_cfg; - generic_phy = ufs_qcom_phy_generic_probe(pdev, &phy->common_cfg, + generic_phy = ufs_qcom_phy_generic_probe(pdev, phy_common, &ufs_qcom_phy_qmp_20nm_phy_ops, &phy_20nm_ops); if (!generic_phy) { @@ -210,27 +196,27 @@ static int ufs_qcom_phy_qmp_20nm_probe(struct platform_device *pdev) goto out; } - phy_set_drvdata(generic_phy, phy); + err = ufs_qcom_phy_init_clks(phy_common); + if (err) { + dev_err(phy_common->dev, "%s: ufs_qcom_phy_init_clks() failed %d\n", + __func__, err); + goto out; + } - strlcpy(phy->common_cfg.name, UFS_PHY_NAME, - sizeof(phy->common_cfg.name)); + err = ufs_qcom_phy_init_vregulators(phy_common); + if (err) { + dev_err(phy_common->dev, "%s: ufs_qcom_phy_init_vregulators() failed %d\n", + __func__, err); + goto out; + } -out: - return err; -} + ufs_qcom_phy_qmp_20nm_advertise_quirks(phy_common); -static int ufs_qcom_phy_qmp_20nm_remove(struct platform_device *pdev) -{ - struct device *dev = &pdev->dev; - struct phy *generic_phy = to_phy(dev); - struct ufs_qcom_phy *ufs_qcom_phy = get_ufs_qcom_phy(generic_phy); - int err = 0; + phy_set_drvdata(generic_phy, phy); - err = ufs_qcom_phy_remove(generic_phy, ufs_qcom_phy); - if (err) - dev_err(dev, "%s: ufs_qcom_phy_remove failed = %d\n", - __func__, err); + strlcpy(phy_common->name, UFS_PHY_NAME, sizeof(phy_common->name)); +out: return err; } @@ -242,7 +228,6 @@ MODULE_DEVICE_TABLE(of, ufs_qcom_phy_qmp_20nm_of_match); static struct platform_driver ufs_qcom_phy_qmp_20nm_driver = { .probe = ufs_qcom_phy_qmp_20nm_probe, - .remove = ufs_qcom_phy_qmp_20nm_remove, .driver = { .of_match_table = ufs_qcom_phy_qmp_20nm_of_match, .name = "ufs_qcom_phy_qmp_20nm", diff --git a/drivers/phy/phy-qcom-ufs.c b/drivers/phy/phy-qcom-ufs.c index 18a5b49..c69568b 100644 --- a/drivers/phy/phy-qcom-ufs.c +++ b/drivers/phy/phy-qcom-ufs.c @@ -22,13 +22,6 @@ #define VDDP_REF_CLK_MIN_UV 1200000 #define VDDP_REF_CLK_MAX_UV 1200000 -static int __ufs_qcom_phy_init_vreg(struct phy *, struct ufs_qcom_phy_vreg *, - const char *, bool); -static int ufs_qcom_phy_init_vreg(struct phy *, struct ufs_qcom_phy_vreg *, - const char *); -static int ufs_qcom_phy_base_init(struct platform_device *pdev, - struct ufs_qcom_phy *phy_common); - int ufs_qcom_phy_calibrate(struct ufs_qcom_phy *ufs_qcom_phy, struct ufs_qcom_phy_calibration *tbl_A, int tbl_size_A, @@ -75,45 +68,6 @@ out: } EXPORT_SYMBOL_GPL(ufs_qcom_phy_calibrate); -struct phy *ufs_qcom_phy_generic_probe(struct platform_device *pdev, - struct ufs_qcom_phy *common_cfg, - const struct phy_ops *ufs_qcom_phy_gen_ops, - struct ufs_qcom_phy_specific_ops *phy_spec_ops) -{ - int err; - struct device *dev = &pdev->dev; - struct phy *generic_phy = NULL; - struct phy_provider *phy_provider; - - err = ufs_qcom_phy_base_init(pdev, common_cfg); - if (err) { - dev_err(dev, "%s: phy base init failed %d\n", __func__, err); - goto out; - } - - phy_provider = devm_of_phy_provider_register(dev, of_phy_simple_xlate); - if (IS_ERR(phy_provider)) { - err = PTR_ERR(phy_provider); - dev_err(dev, "%s: failed to register phy %d\n", __func__, err); - goto out; - } - - generic_phy = devm_phy_create(dev, NULL, ufs_qcom_phy_gen_ops); - if (IS_ERR(generic_phy)) { - err = PTR_ERR(generic_phy); - dev_err(dev, "%s: failed to create phy %d\n", __func__, err); - generic_phy = NULL; - goto out; - } - - common_cfg->phy_spec_ops = phy_spec_ops; - common_cfg->dev = dev; - -out: - return generic_phy; -} -EXPORT_SYMBOL_GPL(ufs_qcom_phy_generic_probe); - /* * This assumes the embedded phy structure inside generic_phy is of type * struct ufs_qcom_phy. In order to function properly it's crucial @@ -154,13 +108,50 @@ int ufs_qcom_phy_base_init(struct platform_device *pdev, return 0; } -static int __ufs_qcom_phy_clk_get(struct phy *phy, +struct phy *ufs_qcom_phy_generic_probe(struct platform_device *pdev, + struct ufs_qcom_phy *common_cfg, + const struct phy_ops *ufs_qcom_phy_gen_ops, + struct ufs_qcom_phy_specific_ops *phy_spec_ops) +{ + int err; + struct device *dev = &pdev->dev; + struct phy *generic_phy = NULL; + struct phy_provider *phy_provider; + + err = ufs_qcom_phy_base_init(pdev, common_cfg); + if (err) { + dev_err(dev, "%s: phy base init failed %d\n", __func__, err); + goto out; + } + + phy_provider = devm_of_phy_provider_register(dev, of_phy_simple_xlate); + if (IS_ERR(phy_provider)) { + err = PTR_ERR(phy_provider); + dev_err(dev, "%s: failed to register phy %d\n", __func__, err); + goto out; + } + + generic_phy = devm_phy_create(dev, NULL, ufs_qcom_phy_gen_ops); + if (IS_ERR(generic_phy)) { + err = PTR_ERR(generic_phy); + dev_err(dev, "%s: failed to create phy %d\n", __func__, err); + generic_phy = NULL; + goto out; + } + + common_cfg->phy_spec_ops = phy_spec_ops; + common_cfg->dev = dev; + +out: + return generic_phy; +} +EXPORT_SYMBOL_GPL(ufs_qcom_phy_generic_probe); + +static int __ufs_qcom_phy_clk_get(struct device *dev, const char *name, struct clk **clk_out, bool err_print) { struct clk *clk; int err = 0; - struct ufs_qcom_phy *ufs_qcom_phy = get_ufs_qcom_phy(phy); - struct device *dev = ufs_qcom_phy->dev; clk = devm_clk_get(dev, name); if (IS_ERR(clk)) { @@ -174,42 +165,44 @@ static int __ufs_qcom_phy_clk_get(struct phy *phy, return err; } -static -int ufs_qcom_phy_clk_get(struct phy *phy, +static int ufs_qcom_phy_clk_get(struct device *dev, const char *name, struct clk **clk_out) { - return __ufs_qcom_phy_clk_get(phy, name, clk_out, true); + return __ufs_qcom_phy_clk_get(dev, name, clk_out, true); } -int -ufs_qcom_phy_init_clks(struct phy *generic_phy, - struct ufs_qcom_phy *phy_common) +int ufs_qcom_phy_init_clks(struct ufs_qcom_phy *phy_common) { int err; - err = ufs_qcom_phy_clk_get(generic_phy, "tx_iface_clk", + if (of_device_is_compatible(phy_common->dev->of_node, + "qcom,msm8996-ufs-phy-qmp-14nm")) + goto skip_txrx_clk; + + err = ufs_qcom_phy_clk_get(phy_common->dev, "tx_iface_clk", &phy_common->tx_iface_clk); if (err) goto out; - err = ufs_qcom_phy_clk_get(generic_phy, "rx_iface_clk", + err = ufs_qcom_phy_clk_get(phy_common->dev, "rx_iface_clk", &phy_common->rx_iface_clk); if (err) goto out; - err = ufs_qcom_phy_clk_get(generic_phy, "ref_clk_src", + err = ufs_qcom_phy_clk_get(phy_common->dev, "ref_clk_src", &phy_common->ref_clk_src); if (err) goto out; +skip_txrx_clk: /* * "ref_clk_parent" is optional hence don't abort init if it's not * found. */ - __ufs_qcom_phy_clk_get(generic_phy, "ref_clk_parent", + __ufs_qcom_phy_clk_get(phy_common->dev, "ref_clk_parent", &phy_common->ref_clk_parent, false); - err = ufs_qcom_phy_clk_get(generic_phy, "ref_clk", + err = ufs_qcom_phy_clk_get(phy_common->dev, "ref_clk", &phy_common->ref_clk); out: @@ -217,41 +210,14 @@ out: } EXPORT_SYMBOL_GPL(ufs_qcom_phy_init_clks); -int -ufs_qcom_phy_init_vregulators(struct phy *generic_phy, - struct ufs_qcom_phy *phy_common) -{ - int err; - - err = ufs_qcom_phy_init_vreg(generic_phy, &phy_common->vdda_pll, - "vdda-pll"); - if (err) - goto out; - - err = ufs_qcom_phy_init_vreg(generic_phy, &phy_common->vdda_phy, - "vdda-phy"); - - if (err) - goto out; - - /* vddp-ref-clk-* properties are optional */ - __ufs_qcom_phy_init_vreg(generic_phy, &phy_common->vddp_ref_clk, - "vddp-ref-clk", true); -out: - return err; -} -EXPORT_SYMBOL_GPL(ufs_qcom_phy_init_vregulators); - -static int __ufs_qcom_phy_init_vreg(struct phy *phy, +static int __ufs_qcom_phy_init_vreg(struct device *dev, struct ufs_qcom_phy_vreg *vreg, const char *name, bool optional) { int err = 0; - struct ufs_qcom_phy *ufs_qcom_phy = get_ufs_qcom_phy(phy); - struct device *dev = ufs_qcom_phy->dev; char prop_name[MAX_PROP_NAME]; - vreg->name = kstrdup(name, GFP_KERNEL); + vreg->name = devm_kstrdup(dev, name, GFP_KERNEL); if (!vreg->name) { err = -ENOMEM; goto out; @@ -304,14 +270,36 @@ out: return err; } -static int ufs_qcom_phy_init_vreg(struct phy *phy, +static int ufs_qcom_phy_init_vreg(struct device *dev, struct ufs_qcom_phy_vreg *vreg, const char *name) { - return __ufs_qcom_phy_init_vreg(phy, vreg, name, false); + return __ufs_qcom_phy_init_vreg(dev, vreg, name, false); } -static -int ufs_qcom_phy_cfg_vreg(struct phy *phy, +int ufs_qcom_phy_init_vregulators(struct ufs_qcom_phy *phy_common) +{ + int err; + + err = ufs_qcom_phy_init_vreg(phy_common->dev, &phy_common->vdda_pll, + "vdda-pll"); + if (err) + goto out; + + err = ufs_qcom_phy_init_vreg(phy_common->dev, &phy_common->vdda_phy, + "vdda-phy"); + + if (err) + goto out; + + /* vddp-ref-clk-* properties are optional */ + __ufs_qcom_phy_init_vreg(phy_common->dev, &phy_common->vddp_ref_clk, + "vddp-ref-clk", true); +out: + return err; +} +EXPORT_SYMBOL_GPL(ufs_qcom_phy_init_vregulators); + +static int ufs_qcom_phy_cfg_vreg(struct device *dev, struct ufs_qcom_phy_vreg *vreg, bool on) { int ret = 0; @@ -319,10 +307,6 @@ int ufs_qcom_phy_cfg_vreg(struct phy *phy, const char *name = vreg->name; int min_uV; int uA_load; - struct ufs_qcom_phy *ufs_qcom_phy = get_ufs_qcom_phy(phy); - struct device *dev = ufs_qcom_phy->dev; - - BUG_ON(!vreg); if (regulator_count_voltages(reg) > 0) { min_uV = on ? vreg->min_uV : 0; @@ -350,18 +334,15 @@ out: return ret; } -static -int ufs_qcom_phy_enable_vreg(struct phy *phy, +static int ufs_qcom_phy_enable_vreg(struct device *dev, struct ufs_qcom_phy_vreg *vreg) { - struct ufs_qcom_phy *ufs_qcom_phy = get_ufs_qcom_phy(phy); - struct device *dev = ufs_qcom_phy->dev; int ret = 0; if (!vreg || vreg->enabled) goto out; - ret = ufs_qcom_phy_cfg_vreg(phy, vreg, true); + ret = ufs_qcom_phy_cfg_vreg(dev, vreg, true); if (ret) { dev_err(dev, "%s: ufs_qcom_phy_cfg_vreg() failed, err=%d\n", __func__, ret); @@ -380,10 +361,9 @@ out: return ret; } -int ufs_qcom_phy_enable_ref_clk(struct phy *generic_phy) +static int ufs_qcom_phy_enable_ref_clk(struct ufs_qcom_phy *phy) { int ret = 0; - struct ufs_qcom_phy *phy = get_ufs_qcom_phy(generic_phy); if (phy->is_ref_clk_enabled) goto out; @@ -430,14 +410,10 @@ out_disable_src: out: return ret; } -EXPORT_SYMBOL_GPL(ufs_qcom_phy_enable_ref_clk); -static -int ufs_qcom_phy_disable_vreg(struct phy *phy, +static int ufs_qcom_phy_disable_vreg(struct device *dev, struct ufs_qcom_phy_vreg *vreg) { - struct ufs_qcom_phy *ufs_qcom_phy = get_ufs_qcom_phy(phy); - struct device *dev = ufs_qcom_phy->dev; int ret = 0; if (!vreg || !vreg->enabled || vreg->is_always_on) @@ -447,7 +423,7 @@ int ufs_qcom_phy_disable_vreg(struct phy *phy, if (!ret) { /* ignore errors on applying disable config */ - ufs_qcom_phy_cfg_vreg(phy, vreg, false); + ufs_qcom_phy_cfg_vreg(dev, vreg, false); vreg->enabled = false; } else { dev_err(dev, "%s: %s disable failed, err=%d\n", @@ -457,10 +433,8 @@ out: return ret; } -void ufs_qcom_phy_disable_ref_clk(struct phy *generic_phy) +static void ufs_qcom_phy_disable_ref_clk(struct ufs_qcom_phy *phy) { - struct ufs_qcom_phy *phy = get_ufs_qcom_phy(generic_phy); - if (phy->is_ref_clk_enabled) { clk_disable_unprepare(phy->ref_clk); /* @@ -473,7 +447,6 @@ void ufs_qcom_phy_disable_ref_clk(struct phy *generic_phy) phy->is_ref_clk_enabled = false; } } -EXPORT_SYMBOL_GPL(ufs_qcom_phy_disable_ref_clk); #define UFS_REF_CLK_EN (1 << 5) @@ -526,9 +499,8 @@ void ufs_qcom_phy_disable_dev_ref_clk(struct phy *generic_phy) EXPORT_SYMBOL_GPL(ufs_qcom_phy_disable_dev_ref_clk); /* Turn ON M-PHY RMMI interface clocks */ -int ufs_qcom_phy_enable_iface_clk(struct phy *generic_phy) +static int ufs_qcom_phy_enable_iface_clk(struct ufs_qcom_phy *phy) { - struct ufs_qcom_phy *phy = get_ufs_qcom_phy(generic_phy); int ret = 0; if (phy->is_iface_clk_enabled) @@ -552,20 +524,16 @@ int ufs_qcom_phy_enable_iface_clk(struct phy *generic_phy) out: return ret; } -EXPORT_SYMBOL_GPL(ufs_qcom_phy_enable_iface_clk); /* Turn OFF M-PHY RMMI interface clocks */ -void ufs_qcom_phy_disable_iface_clk(struct phy *generic_phy) +void ufs_qcom_phy_disable_iface_clk(struct ufs_qcom_phy *phy) { - struct ufs_qcom_phy *phy = get_ufs_qcom_phy(generic_phy); - if (phy->is_iface_clk_enabled) { clk_disable_unprepare(phy->tx_iface_clk); clk_disable_unprepare(phy->rx_iface_clk); phy->is_iface_clk_enabled = false; } } -EXPORT_SYMBOL_GPL(ufs_qcom_phy_disable_iface_clk); int ufs_qcom_phy_start_serdes(struct phy *generic_phy) { @@ -634,29 +602,6 @@ int ufs_qcom_phy_calibrate_phy(struct phy *generic_phy, bool is_rate_B) } EXPORT_SYMBOL_GPL(ufs_qcom_phy_calibrate_phy); -int ufs_qcom_phy_remove(struct phy *generic_phy, - struct ufs_qcom_phy *ufs_qcom_phy) -{ - phy_power_off(generic_phy); - - kfree(ufs_qcom_phy->vdda_pll.name); - kfree(ufs_qcom_phy->vdda_phy.name); - - return 0; -} -EXPORT_SYMBOL_GPL(ufs_qcom_phy_remove); - -int ufs_qcom_phy_exit(struct phy *generic_phy) -{ - struct ufs_qcom_phy *ufs_qcom_phy = get_ufs_qcom_phy(generic_phy); - - if (ufs_qcom_phy->is_powered_on) - phy_power_off(generic_phy); - - return 0; -} -EXPORT_SYMBOL_GPL(ufs_qcom_phy_exit); - int ufs_qcom_phy_is_pcs_ready(struct phy *generic_phy) { struct ufs_qcom_phy *ufs_qcom_phy = get_ufs_qcom_phy(generic_phy); @@ -678,7 +623,10 @@ int ufs_qcom_phy_power_on(struct phy *generic_phy) struct device *dev = phy_common->dev; int err; - err = ufs_qcom_phy_enable_vreg(generic_phy, &phy_common->vdda_phy); + if (phy_common->is_powered_on) + return 0; + + err = ufs_qcom_phy_enable_vreg(dev, &phy_common->vdda_phy); if (err) { dev_err(dev, "%s enable vdda_phy failed, err=%d\n", __func__, err); @@ -688,23 +636,30 @@ int ufs_qcom_phy_power_on(struct phy *generic_phy) phy_common->phy_spec_ops->power_control(phy_common, true); /* vdda_pll also enables ref clock LDOs so enable it first */ - err = ufs_qcom_phy_enable_vreg(generic_phy, &phy_common->vdda_pll); + err = ufs_qcom_phy_enable_vreg(dev, &phy_common->vdda_pll); if (err) { dev_err(dev, "%s enable vdda_pll failed, err=%d\n", __func__, err); goto out_disable_phy; } - err = ufs_qcom_phy_enable_ref_clk(generic_phy); + err = ufs_qcom_phy_enable_iface_clk(phy_common); if (err) { - dev_err(dev, "%s enable phy ref clock failed, err=%d\n", + dev_err(dev, "%s enable phy iface clock failed, err=%d\n", __func__, err); goto out_disable_pll; } + err = ufs_qcom_phy_enable_ref_clk(phy_common); + if (err) { + dev_err(dev, "%s enable phy ref clock failed, err=%d\n", + __func__, err); + goto out_disable_iface_clk; + } + /* enable device PHY ref_clk pad rail */ if (phy_common->vddp_ref_clk.reg) { - err = ufs_qcom_phy_enable_vreg(generic_phy, + err = ufs_qcom_phy_enable_vreg(dev, &phy_common->vddp_ref_clk); if (err) { dev_err(dev, "%s enable vddp_ref_clk failed, err=%d\n", @@ -717,11 +672,13 @@ int ufs_qcom_phy_power_on(struct phy *generic_phy) goto out; out_disable_ref_clk: - ufs_qcom_phy_disable_ref_clk(generic_phy); + ufs_qcom_phy_disable_ref_clk(phy_common); +out_disable_iface_clk: + ufs_qcom_phy_disable_iface_clk(phy_common); out_disable_pll: - ufs_qcom_phy_disable_vreg(generic_phy, &phy_common->vdda_pll); + ufs_qcom_phy_disable_vreg(dev, &phy_common->vdda_pll); out_disable_phy: - ufs_qcom_phy_disable_vreg(generic_phy, &phy_common->vdda_phy); + ufs_qcom_phy_disable_vreg(dev, &phy_common->vdda_phy); out: return err; } @@ -731,15 +688,19 @@ int ufs_qcom_phy_power_off(struct phy *generic_phy) { struct ufs_qcom_phy *phy_common = get_ufs_qcom_phy(generic_phy); + if (!phy_common->is_powered_on) + return 0; + phy_common->phy_spec_ops->power_control(phy_common, false); if (phy_common->vddp_ref_clk.reg) - ufs_qcom_phy_disable_vreg(generic_phy, + ufs_qcom_phy_disable_vreg(phy_common->dev, &phy_common->vddp_ref_clk); - ufs_qcom_phy_disable_ref_clk(generic_phy); + ufs_qcom_phy_disable_ref_clk(phy_common); + ufs_qcom_phy_disable_iface_clk(phy_common); - ufs_qcom_phy_disable_vreg(generic_phy, &phy_common->vdda_pll); - ufs_qcom_phy_disable_vreg(generic_phy, &phy_common->vdda_phy); + ufs_qcom_phy_disable_vreg(phy_common->dev, &phy_common->vdda_pll); + ufs_qcom_phy_disable_vreg(phy_common->dev, &phy_common->vdda_phy); phy_common->is_powered_on = false; return 0; diff --git a/drivers/s390/block/dasd.c b/drivers/s390/block/dasd.c index 1de0890..0e3fdfd 100644 --- a/drivers/s390/block/dasd.c +++ b/drivers/s390/block/dasd.c @@ -69,6 +69,7 @@ static void dasd_block_tasklet(struct dasd_block *); static void do_kick_device(struct work_struct *); static void do_restore_device(struct work_struct *); static void do_reload_device(struct work_struct *); +static void do_requeue_requests(struct work_struct *); static void dasd_return_cqr_cb(struct dasd_ccw_req *, void *); static void dasd_device_timeout(unsigned long); static void dasd_block_timeout(unsigned long); @@ -125,6 +126,7 @@ struct dasd_device *dasd_alloc_device(void) INIT_WORK(&device->kick_work, do_kick_device); INIT_WORK(&device->restore_device, do_restore_device); INIT_WORK(&device->reload_device, do_reload_device); + INIT_WORK(&device->requeue_requests, do_requeue_requests); device->state = DASD_STATE_NEW; device->target = DASD_STATE_NEW; mutex_init(&device->state_mutex); @@ -1448,9 +1450,9 @@ int dasd_start_IO(struct dasd_ccw_req *cqr) cqr->starttime = jiffies; cqr->retries--; if (!test_bit(DASD_CQR_VERIFY_PATH, &cqr->flags)) { - cqr->lpm &= device->path_data.opm; + cqr->lpm &= dasd_path_get_opm(device); if (!cqr->lpm) - cqr->lpm = device->path_data.opm; + cqr->lpm = dasd_path_get_opm(device); } if (cqr->cpmode == 1) { rc = ccw_device_tm_start(device->cdev, cqr->cpaddr, @@ -1483,8 +1485,8 @@ int dasd_start_IO(struct dasd_ccw_req *cqr) DBF_DEV_EVENT(DBF_WARNING, device, "start_IO: selected paths gone (%x)", cqr->lpm); - } else if (cqr->lpm != device->path_data.opm) { - cqr->lpm = device->path_data.opm; + } else if (cqr->lpm != dasd_path_get_opm(device)) { + cqr->lpm = dasd_path_get_opm(device); DBF_DEV_EVENT(DBF_DEBUG, device, "%s", "start_IO: selected paths gone," " retry on all paths"); @@ -1493,11 +1495,10 @@ int dasd_start_IO(struct dasd_ccw_req *cqr) "start_IO: all paths in opm gone," " do path verification"); dasd_generic_last_path_gone(device); - device->path_data.opm = 0; - device->path_data.ppm = 0; - device->path_data.npm = 0; - device->path_data.tbvpm = - ccw_device_get_path_mask(device->cdev); + dasd_path_no_path(device); + dasd_path_set_tbvpm(device, + ccw_device_get_path_mask( + device->cdev)); } break; case -ENODEV: @@ -1623,6 +1624,13 @@ void dasd_generic_handle_state_change(struct dasd_device *device) } EXPORT_SYMBOL_GPL(dasd_generic_handle_state_change); +static int dasd_check_hpf_error(struct irb *irb) +{ + return (scsw_tm_is_valid_schxs(&irb->scsw) && + (irb->scsw.tm.sesq == SCSW_SESQ_DEV_NOFCX || + irb->scsw.tm.sesq == SCSW_SESQ_PATH_NOFCX)); +} + /* * Interrupt handler for "normal" ssch-io based dasd devices. */ @@ -1642,7 +1650,7 @@ void dasd_int_handler(struct ccw_device *cdev, unsigned long intparm, switch (PTR_ERR(irb)) { case -EIO: if (cqr && cqr->status == DASD_CQR_CLEAR_PENDING) { - device = (struct dasd_device *) cqr->startdev; + device = cqr->startdev; cqr->status = DASD_CQR_CLEARED; dasd_device_clear_timer(device); wake_up(&dasd_flush_wq); @@ -1749,19 +1757,26 @@ void dasd_int_handler(struct ccw_device *cdev, unsigned long intparm, struct dasd_ccw_req, devlist); } } else { /* error */ + /* check for HPF error + * call discipline function to requeue all requests + * and disable HPF accordingly + */ + if (cqr->cpmode && dasd_check_hpf_error(irb) && + device->discipline->handle_hpf_error) + device->discipline->handle_hpf_error(device, irb); /* * If we don't want complex ERP for this request, then just * reset this and retry it in the fastpath */ if (!test_bit(DASD_CQR_FLAGS_USE_ERP, &cqr->flags) && cqr->retries > 0) { - if (cqr->lpm == device->path_data.opm) + if (cqr->lpm == dasd_path_get_opm(device)) DBF_DEV_EVENT(DBF_DEBUG, device, "default ERP in fastpath " "(%i retries left)", cqr->retries); if (!test_bit(DASD_CQR_VERIFY_PATH, &cqr->flags)) - cqr->lpm = device->path_data.opm; + cqr->lpm = dasd_path_get_opm(device); cqr->status = DASD_CQR_QUEUED; next = cqr; } else @@ -2002,17 +2017,18 @@ static void __dasd_device_check_path_events(struct dasd_device *device) { int rc; - if (device->path_data.tbvpm) { - if (device->stopped & ~(DASD_STOPPED_DC_WAIT | - DASD_UNRESUMED_PM)) - return; - rc = device->discipline->verify_path( - device, device->path_data.tbvpm); - if (rc) - dasd_device_set_timer(device, 50); - else - device->path_data.tbvpm = 0; - } + if (!dasd_path_get_tbvpm(device)) + return; + + if (device->stopped & + ~(DASD_STOPPED_DC_WAIT | DASD_UNRESUMED_PM)) + return; + rc = device->discipline->verify_path(device, + dasd_path_get_tbvpm(device)); + if (rc) + dasd_device_set_timer(device, 50); + else + dasd_path_clear_all_verify(device); }; /* @@ -2924,10 +2940,10 @@ static int _dasd_requeue_request(struct dasd_ccw_req *cqr) if (!block) return -EINVAL; - spin_lock_irqsave(&block->queue_lock, flags); + spin_lock_irqsave(&block->request_queue_lock, flags); req = (struct request *) cqr->callback_data; blk_requeue_request(block->request_queue, req); - spin_unlock_irqrestore(&block->queue_lock, flags); + spin_unlock_irqrestore(&block->request_queue_lock, flags); return 0; } @@ -3121,6 +3137,7 @@ static int dasd_alloc_queue(struct dasd_block *block) */ static void dasd_setup_queue(struct dasd_block *block) { + struct request_queue *q = block->request_queue; int max; if (block->base->features & DASD_FEATURE_USERAW) { @@ -3135,17 +3152,16 @@ static void dasd_setup_queue(struct dasd_block *block) } else { max = block->base->discipline->max_blocks << block->s2b_shift; } - queue_flag_set_unlocked(QUEUE_FLAG_NONROT, block->request_queue); - block->request_queue->limits.max_dev_sectors = max; - blk_queue_logical_block_size(block->request_queue, - block->bp_block); - blk_queue_max_hw_sectors(block->request_queue, max); - blk_queue_max_segments(block->request_queue, -1L); + queue_flag_set_unlocked(QUEUE_FLAG_NONROT, q); + q->limits.max_dev_sectors = max; + blk_queue_logical_block_size(q, block->bp_block); + blk_queue_max_hw_sectors(q, max); + blk_queue_max_segments(q, USHRT_MAX); /* with page sized segments we can translate each segement into * one idaw/tidaw */ - blk_queue_max_segment_size(block->request_queue, PAGE_SIZE); - blk_queue_segment_boundary(block->request_queue, PAGE_SIZE - 1); + blk_queue_max_segment_size(q, PAGE_SIZE); + blk_queue_segment_boundary(q, PAGE_SIZE - 1); } /* @@ -3517,11 +3533,15 @@ int dasd_generic_set_offline(struct ccw_device *cdev) struct dasd_device *device; struct dasd_block *block; int max_count, open_count, rc; + unsigned long flags; rc = 0; - device = dasd_device_from_cdev(cdev); - if (IS_ERR(device)) + spin_lock_irqsave(get_ccwdev_lock(cdev), flags); + device = dasd_device_from_cdev_locked(cdev); + if (IS_ERR(device)) { + spin_unlock_irqrestore(get_ccwdev_lock(cdev), flags); return PTR_ERR(device); + } /* * We must make sure that this device is currently not in use. @@ -3540,8 +3560,7 @@ int dasd_generic_set_offline(struct ccw_device *cdev) pr_warn("%s: The DASD cannot be set offline while it is in use\n", dev_name(&cdev->dev)); clear_bit(DASD_FLAG_OFFLINE, &device->flags); - dasd_put_device(device); - return -EBUSY; + goto out_busy; } } @@ -3551,19 +3570,19 @@ int dasd_generic_set_offline(struct ccw_device *cdev) * could only be called by normal offline so safe_offline flag * needs to be removed to run normal offline and kill all I/O */ - if (test_and_set_bit(DASD_FLAG_OFFLINE, &device->flags)) { + if (test_and_set_bit(DASD_FLAG_OFFLINE, &device->flags)) /* Already doing normal offline processing */ - dasd_put_device(device); - return -EBUSY; - } else + goto out_busy; + else clear_bit(DASD_FLAG_SAFE_OFFLINE, &device->flags); - - } else - if (test_bit(DASD_FLAG_OFFLINE, &device->flags)) { + } else { + if (test_bit(DASD_FLAG_OFFLINE, &device->flags)) /* Already doing offline processing */ - dasd_put_device(device); - return -EBUSY; - } + goto out_busy; + } + + set_bit(DASD_FLAG_OFFLINE, &device->flags); + spin_unlock_irqrestore(get_ccwdev_lock(cdev), flags); /* * if safe_offline called set safe_offline_running flag and @@ -3591,7 +3610,6 @@ int dasd_generic_set_offline(struct ccw_device *cdev) goto interrupted; } - set_bit(DASD_FLAG_OFFLINE, &device->flags); dasd_set_target_state(device, DASD_STATE_NEW); /* dasd_delete_device destroys the device reference. */ block = device->block; @@ -3610,7 +3628,14 @@ interrupted: clear_bit(DASD_FLAG_SAFE_OFFLINE_RUNNING, &device->flags); clear_bit(DASD_FLAG_OFFLINE, &device->flags); dasd_put_device(device); + return rc; + +out_busy: + dasd_put_device(device); + spin_unlock_irqrestore(get_ccwdev_lock(cdev), flags); + + return -EBUSY; } EXPORT_SYMBOL_GPL(dasd_generic_set_offline); @@ -3675,14 +3700,12 @@ int dasd_generic_notify(struct ccw_device *cdev, int event) case CIO_GONE: case CIO_BOXED: case CIO_NO_PATH: - device->path_data.opm = 0; - device->path_data.ppm = 0; - device->path_data.npm = 0; + dasd_path_no_path(device); ret = dasd_generic_last_path_gone(device); break; case CIO_OPER: ret = 1; - if (device->path_data.opm) + if (dasd_path_get_opm(device)) ret = dasd_generic_path_operational(device); break; } @@ -3693,48 +3716,32 @@ EXPORT_SYMBOL_GPL(dasd_generic_notify); void dasd_generic_path_event(struct ccw_device *cdev, int *path_event) { - int chp; - __u8 oldopm, eventlpm; struct dasd_device *device; + int chp, oldopm, hpfpm, ifccpm; device = dasd_device_from_cdev_locked(cdev); if (IS_ERR(device)) return; + + oldopm = dasd_path_get_opm(device); for (chp = 0; chp < 8; chp++) { - eventlpm = 0x80 >> chp; if (path_event[chp] & PE_PATH_GONE) { - oldopm = device->path_data.opm; - device->path_data.opm &= ~eventlpm; - device->path_data.ppm &= ~eventlpm; - device->path_data.npm &= ~eventlpm; - if (oldopm && !device->path_data.opm) { - dev_warn(&device->cdev->dev, - "No verified channel paths remain " - "for the device\n"); - DBF_DEV_EVENT(DBF_WARNING, device, - "%s", "last verified path gone"); - dasd_eer_write(device, NULL, DASD_EER_NOPATH); - dasd_device_set_stop_bits(device, - DASD_STOPPED_DC_WAIT); - } + dasd_path_notoper(device, chp); } if (path_event[chp] & PE_PATH_AVAILABLE) { - device->path_data.opm &= ~eventlpm; - device->path_data.ppm &= ~eventlpm; - device->path_data.npm &= ~eventlpm; - device->path_data.tbvpm |= eventlpm; + dasd_path_available(device, chp); dasd_schedule_device_bh(device); } if (path_event[chp] & PE_PATHGROUP_ESTABLISHED) { - if (!(device->path_data.opm & eventlpm) && - !(device->path_data.tbvpm & eventlpm)) { + if (!dasd_path_is_operational(device, chp) && + !dasd_path_need_verify(device, chp)) { /* * we can not establish a pathgroup on an * unavailable path, so trigger a path * verification first */ - device->path_data.tbvpm |= eventlpm; - dasd_schedule_device_bh(device); + dasd_path_available(device, chp); + dasd_schedule_device_bh(device); } DBF_DEV_EVENT(DBF_WARNING, device, "%s", "Pathgroup re-established\n"); @@ -3742,45 +3749,65 @@ void dasd_generic_path_event(struct ccw_device *cdev, int *path_event) device->discipline->kick_validate(device); } } + hpfpm = dasd_path_get_hpfpm(device); + ifccpm = dasd_path_get_ifccpm(device); + if (!dasd_path_get_opm(device) && hpfpm) { + /* + * device has no operational paths but at least one path is + * disabled due to HPF errors + * disable HPF at all and use the path(s) again + */ + if (device->discipline->disable_hpf) + device->discipline->disable_hpf(device); + dasd_device_set_stop_bits(device, DASD_STOPPED_NOT_ACC); + dasd_path_set_tbvpm(device, hpfpm); + dasd_schedule_device_bh(device); + dasd_schedule_requeue(device); + } else if (!dasd_path_get_opm(device) && ifccpm) { + /* + * device has no operational paths but at least one path is + * disabled due to IFCC errors + * trigger path verification on paths with IFCC errors + */ + dasd_path_set_tbvpm(device, ifccpm); + dasd_schedule_device_bh(device); + } + if (oldopm && !dasd_path_get_opm(device) && !hpfpm && !ifccpm) { + dev_warn(&device->cdev->dev, + "No verified channel paths remain for the device\n"); + DBF_DEV_EVENT(DBF_WARNING, device, + "%s", "last verified path gone"); + dasd_eer_write(device, NULL, DASD_EER_NOPATH); + dasd_device_set_stop_bits(device, + DASD_STOPPED_DC_WAIT); + } dasd_put_device(device); } EXPORT_SYMBOL_GPL(dasd_generic_path_event); int dasd_generic_verify_path(struct dasd_device *device, __u8 lpm) { - if (!device->path_data.opm && lpm) { - device->path_data.opm = lpm; + if (!dasd_path_get_opm(device) && lpm) { + dasd_path_set_opm(device, lpm); dasd_generic_path_operational(device); } else - device->path_data.opm |= lpm; + dasd_path_add_opm(device, lpm); return 0; } EXPORT_SYMBOL_GPL(dasd_generic_verify_path); - -int dasd_generic_pm_freeze(struct ccw_device *cdev) +/* + * clear active requests and requeue them to block layer if possible + */ +static int dasd_generic_requeue_all_requests(struct dasd_device *device) { - struct dasd_device *device = dasd_device_from_cdev(cdev); - struct list_head freeze_queue; + struct list_head requeue_queue; struct dasd_ccw_req *cqr, *n; struct dasd_ccw_req *refers; int rc; - if (IS_ERR(device)) - return PTR_ERR(device); - - /* mark device as suspended */ - set_bit(DASD_FLAG_SUSPENDED, &device->flags); - - if (device->discipline->freeze) - rc = device->discipline->freeze(device); - - /* disallow new I/O */ - dasd_device_set_stop_bits(device, DASD_STOPPED_PM); - - /* clear active requests and requeue them to block layer if possible */ - INIT_LIST_HEAD(&freeze_queue); - spin_lock_irq(get_ccwdev_lock(cdev)); + INIT_LIST_HEAD(&requeue_queue); + spin_lock_irq(get_ccwdev_lock(device->cdev)); rc = 0; list_for_each_entry_safe(cqr, n, &device->ccw_queue, devlist) { /* Check status and move request to flush_queue */ @@ -3791,25 +3818,22 @@ int dasd_generic_pm_freeze(struct ccw_device *cdev) dev_err(&device->cdev->dev, "Unable to terminate request %p " "on suspend\n", cqr); - spin_unlock_irq(get_ccwdev_lock(cdev)); + spin_unlock_irq(get_ccwdev_lock(device->cdev)); dasd_put_device(device); return rc; } } - list_move_tail(&cqr->devlist, &freeze_queue); + list_move_tail(&cqr->devlist, &requeue_queue); } - spin_unlock_irq(get_ccwdev_lock(cdev)); + spin_unlock_irq(get_ccwdev_lock(device->cdev)); - list_for_each_entry_safe(cqr, n, &freeze_queue, devlist) { + list_for_each_entry_safe(cqr, n, &requeue_queue, devlist) { wait_event(dasd_flush_wq, (cqr->status != DASD_CQR_CLEAR_PENDING)); - if (cqr->status == DASD_CQR_CLEARED) - cqr->status = DASD_CQR_QUEUED; - /* requeue requests to blocklayer will only work for - block device requests */ - if (_dasd_requeue_request(cqr)) - continue; + /* mark sleepon requests as ended */ + if (cqr->callback_data == DASD_SLEEPON_START_TAG) + cqr->callback_data = DASD_SLEEPON_END_TAG; /* remove requests from device and block queue */ list_del_init(&cqr->devlist); @@ -3821,6 +3845,14 @@ int dasd_generic_pm_freeze(struct ccw_device *cdev) dasd_free_erp_request(cqr, cqr->memdev); cqr = refers; } + + /* + * requeue requests to blocklayer will only work + * for block device requests + */ + if (_dasd_requeue_request(cqr)) + continue; + if (cqr->block) list_del_init(&cqr->blocklist); cqr->block->base->discipline->free_cp( @@ -3831,15 +3863,56 @@ int dasd_generic_pm_freeze(struct ccw_device *cdev) * if requests remain then they are internal request * and go back to the device queue */ - if (!list_empty(&freeze_queue)) { + if (!list_empty(&requeue_queue)) { /* move freeze_queue to start of the ccw_queue */ - spin_lock_irq(get_ccwdev_lock(cdev)); - list_splice_tail(&freeze_queue, &device->ccw_queue); - spin_unlock_irq(get_ccwdev_lock(cdev)); + spin_lock_irq(get_ccwdev_lock(device->cdev)); + list_splice_tail(&requeue_queue, &device->ccw_queue); + spin_unlock_irq(get_ccwdev_lock(device->cdev)); } - dasd_put_device(device); + /* wake up generic waitqueue for eventually ended sleepon requests */ + wake_up(&generic_waitq); return rc; } + +static void do_requeue_requests(struct work_struct *work) +{ + struct dasd_device *device = container_of(work, struct dasd_device, + requeue_requests); + dasd_generic_requeue_all_requests(device); + dasd_device_remove_stop_bits(device, DASD_STOPPED_NOT_ACC); + if (device->block) + dasd_schedule_block_bh(device->block); + dasd_put_device(device); +} + +void dasd_schedule_requeue(struct dasd_device *device) +{ + dasd_get_device(device); + /* queue call to dasd_reload_device to the kernel event daemon. */ + if (!schedule_work(&device->requeue_requests)) + dasd_put_device(device); +} +EXPORT_SYMBOL(dasd_schedule_requeue); + +int dasd_generic_pm_freeze(struct ccw_device *cdev) +{ + struct dasd_device *device = dasd_device_from_cdev(cdev); + int rc; + + if (IS_ERR(device)) + return PTR_ERR(device); + + /* mark device as suspended */ + set_bit(DASD_FLAG_SUSPENDED, &device->flags); + + if (device->discipline->freeze) + rc = device->discipline->freeze(device); + + /* disallow new I/O */ + dasd_device_set_stop_bits(device, DASD_STOPPED_PM); + + return dasd_generic_requeue_all_requests(device); +} EXPORT_SYMBOL_GPL(dasd_generic_pm_freeze); int dasd_generic_restore_device(struct ccw_device *cdev) diff --git a/drivers/s390/block/dasd_3990_erp.c b/drivers/s390/block/dasd_3990_erp.c index 8305ab6..95f7645 100644 --- a/drivers/s390/block/dasd_3990_erp.c +++ b/drivers/s390/block/dasd_3990_erp.c @@ -152,7 +152,7 @@ dasd_3990_erp_alternate_path(struct dasd_ccw_req * erp) opm = ccw_device_get_path_mask(device->cdev); spin_unlock_irqrestore(get_ccwdev_lock(device->cdev), flags); if (erp->lpm == 0) - erp->lpm = device->path_data.opm & + erp->lpm = dasd_path_get_opm(device) & ~(erp->irb.esw.esw0.sublog.lpum); else erp->lpm &= ~(erp->irb.esw.esw0.sublog.lpum); @@ -273,7 +273,7 @@ static struct dasd_ccw_req *dasd_3990_erp_action_1(struct dasd_ccw_req *erp) !test_bit(DASD_CQR_VERIFY_PATH, &erp->flags)) { erp->status = DASD_CQR_FILLED; erp->retries = 10; - erp->lpm = erp->startdev->path_data.opm; + erp->lpm = dasd_path_get_opm(erp->startdev); erp->function = dasd_3990_erp_action_1_sec; } return erp; @@ -1926,7 +1926,7 @@ dasd_3990_erp_compound_path(struct dasd_ccw_req * erp, char *sense) !test_bit(DASD_CQR_VERIFY_PATH, &erp->flags)) { /* reset the lpm and the status to be able to * try further actions. */ - erp->lpm = erp->startdev->path_data.opm; + erp->lpm = dasd_path_get_opm(erp->startdev); erp->status = DASD_CQR_NEED_ERP; } } @@ -2208,6 +2208,51 @@ dasd_3990_erp_inspect_32(struct dasd_ccw_req * erp, char *sense) } /* end dasd_3990_erp_inspect_32 */ +static void dasd_3990_erp_disable_path(struct dasd_device *device, __u8 lpum) +{ + int pos = pathmask_to_pos(lpum); + + /* no remaining path, cannot disable */ + if (!(dasd_path_get_opm(device) & ~lpum)) + return; + + dev_err(&device->cdev->dev, + "Path %x.%02x (pathmask %02x) is disabled - IFCC threshold exceeded\n", + device->path[pos].cssid, device->path[pos].chpid, lpum); + dasd_path_remove_opm(device, lpum); + dasd_path_add_ifccpm(device, lpum); + device->path[pos].errorclk = 0; + atomic_set(&device->path[pos].error_count, 0); +} + +static void dasd_3990_erp_account_error(struct dasd_ccw_req *erp) +{ + struct dasd_device *device = erp->startdev; + __u8 lpum = erp->refers->irb.esw.esw1.lpum; + int pos = pathmask_to_pos(lpum); + unsigned long long clk; + + if (!device->path_thrhld) + return; + + clk = get_tod_clock(); + /* + * check if the last error is longer ago than the timeout, + * if so reset error state + */ + if ((tod_to_ns(clk - device->path[pos].errorclk) / NSEC_PER_SEC) + >= device->path_interval) { + atomic_set(&device->path[pos].error_count, 0); + device->path[pos].errorclk = 0; + } + atomic_inc(&device->path[pos].error_count); + device->path[pos].errorclk = clk; + /* threshold exceeded disable path if possible */ + if (atomic_read(&device->path[pos].error_count) >= + device->path_thrhld) + dasd_3990_erp_disable_path(device, lpum); +} + /* ***************************************************************************** * main ERP control functions (24 and 32 byte sense) @@ -2237,6 +2282,7 @@ dasd_3990_erp_control_check(struct dasd_ccw_req *erp) | SCHN_STAT_CHN_CTRL_CHK)) { DBF_DEV_EVENT(DBF_WARNING, device, "%s", "channel or interface control check"); + dasd_3990_erp_account_error(erp); erp = dasd_3990_erp_action_4(erp, NULL); } return erp; diff --git a/drivers/s390/block/dasd_devmap.c b/drivers/s390/block/dasd_devmap.c index 15a1a70..84ca314 100644 --- a/drivers/s390/block/dasd_devmap.c +++ b/drivers/s390/block/dasd_devmap.c @@ -725,27 +725,15 @@ static ssize_t dasd_ff_show(struct device *dev, struct device_attribute *attr, static ssize_t dasd_ff_store(struct device *dev, struct device_attribute *attr, const char *buf, size_t count) { - struct dasd_devmap *devmap; - int val; - char *endp; - - devmap = dasd_devmap_from_cdev(to_ccwdev(dev)); - if (IS_ERR(devmap)) - return PTR_ERR(devmap); + unsigned int val; + int rc; - val = simple_strtoul(buf, &endp, 0); - if (((endp + 1) < (buf + count)) || (val > 1)) + if (kstrtouint(buf, 0, &val) || val > 1) return -EINVAL; - spin_lock(&dasd_devmap_lock); - if (val) - devmap->features |= DASD_FEATURE_FAILFAST; - else - devmap->features &= ~DASD_FEATURE_FAILFAST; - if (devmap->device) - devmap->device->features = devmap->features; - spin_unlock(&dasd_devmap_lock); - return count; + rc = dasd_set_feature(to_ccwdev(dev), DASD_FEATURE_FAILFAST, val); + + return rc ? : count; } static DEVICE_ATTR(failfast, 0644, dasd_ff_show, dasd_ff_store); @@ -771,32 +759,41 @@ static ssize_t dasd_ro_store(struct device *dev, struct device_attribute *attr, const char *buf, size_t count) { - struct dasd_devmap *devmap; + struct ccw_device *cdev = to_ccwdev(dev); struct dasd_device *device; - int val; - char *endp; - - devmap = dasd_devmap_from_cdev(to_ccwdev(dev)); - if (IS_ERR(devmap)) - return PTR_ERR(devmap); + unsigned long flags; + unsigned int val; + int rc; - val = simple_strtoul(buf, &endp, 0); - if (((endp + 1) < (buf + count)) || (val > 1)) + if (kstrtouint(buf, 0, &val) || val > 1) return -EINVAL; - spin_lock(&dasd_devmap_lock); - if (val) - devmap->features |= DASD_FEATURE_READONLY; - else - devmap->features &= ~DASD_FEATURE_READONLY; - device = devmap->device; - if (device) { - device->features = devmap->features; - val = val || test_bit(DASD_FLAG_DEVICE_RO, &device->flags); + rc = dasd_set_feature(cdev, DASD_FEATURE_READONLY, val); + if (rc) + return rc; + + device = dasd_device_from_cdev(cdev); + if (IS_ERR(device)) + return PTR_ERR(device); + + spin_lock_irqsave(get_ccwdev_lock(cdev), flags); + val = val || test_bit(DASD_FLAG_DEVICE_RO, &device->flags); + + if (!device->block || !device->block->gdp || + test_bit(DASD_FLAG_OFFLINE, &device->flags)) { + spin_unlock_irqrestore(get_ccwdev_lock(cdev), flags); + goto out; } - spin_unlock(&dasd_devmap_lock); - if (device && device->block && device->block->gdp) - set_disk_ro(device->block->gdp, val); + /* Increase open_count to avoid losing the block device */ + atomic_inc(&device->block->open_count); + spin_unlock_irqrestore(get_ccwdev_lock(cdev), flags); + + set_disk_ro(device->block->gdp, val); + atomic_dec(&device->block->open_count); + +out: + dasd_put_device(device); + return count; } @@ -823,27 +820,15 @@ static ssize_t dasd_erplog_store(struct device *dev, struct device_attribute *attr, const char *buf, size_t count) { - struct dasd_devmap *devmap; - int val; - char *endp; - - devmap = dasd_devmap_from_cdev(to_ccwdev(dev)); - if (IS_ERR(devmap)) - return PTR_ERR(devmap); + unsigned int val; + int rc; - val = simple_strtoul(buf, &endp, 0); - if (((endp + 1) < (buf + count)) || (val > 1)) + if (kstrtouint(buf, 0, &val) || val > 1) return -EINVAL; - spin_lock(&dasd_devmap_lock); - if (val) - devmap->features |= DASD_FEATURE_ERPLOG; - else - devmap->features &= ~DASD_FEATURE_ERPLOG; - if (devmap->device) - devmap->device->features = devmap->features; - spin_unlock(&dasd_devmap_lock); - return count; + rc = dasd_set_feature(to_ccwdev(dev), DASD_FEATURE_ERPLOG, val); + + return rc ? : count; } static DEVICE_ATTR(erplog, 0644, dasd_erplog_show, dasd_erplog_store); @@ -871,16 +856,14 @@ dasd_use_diag_store(struct device *dev, struct device_attribute *attr, const char *buf, size_t count) { struct dasd_devmap *devmap; + unsigned int val; ssize_t rc; - int val; - char *endp; devmap = dasd_devmap_from_cdev(to_ccwdev(dev)); if (IS_ERR(devmap)) return PTR_ERR(devmap); - val = simple_strtoul(buf, &endp, 0); - if (((endp + 1) < (buf + count)) || (val > 1)) + if (kstrtouint(buf, 0, &val) || val > 1) return -EINVAL; spin_lock(&dasd_devmap_lock); @@ -994,10 +977,12 @@ dasd_access_show(struct device *dev, struct device_attribute *attr, if (IS_ERR(device)) return PTR_ERR(device); - if (device->discipline->host_access_count) - count = device->discipline->host_access_count(device); - else + if (!device->discipline) + count = -ENODEV; + else if (!device->discipline->host_access_count) count = -EOPNOTSUPP; + else + count = device->discipline->host_access_count(device); dasd_put_device(device); if (count < 0) @@ -1197,27 +1182,25 @@ static ssize_t dasd_eer_store(struct device *dev, struct device_attribute *attr, const char *buf, size_t count) { - struct dasd_devmap *devmap; - int val, rc; - char *endp; + struct dasd_device *device; + unsigned int val; + int rc = 0; - devmap = dasd_devmap_from_cdev(to_ccwdev(dev)); - if (IS_ERR(devmap)) - return PTR_ERR(devmap); - if (!devmap->device) - return -ENODEV; + device = dasd_device_from_cdev(to_ccwdev(dev)); + if (IS_ERR(device)) + return PTR_ERR(device); - val = simple_strtoul(buf, &endp, 0); - if (((endp + 1) < (buf + count)) || (val > 1)) + if (kstrtouint(buf, 0, &val) || val > 1) return -EINVAL; - if (val) { - rc = dasd_eer_enable(devmap->device); - if (rc) - return rc; - } else - dasd_eer_disable(devmap->device); - return count; + if (val) + rc = dasd_eer_enable(device); + else + dasd_eer_disable(device); + + dasd_put_device(device); + + return rc ? : count; } static DEVICE_ATTR(eer_enabled, 0644, dasd_eer_show, dasd_eer_store); @@ -1360,6 +1343,50 @@ dasd_timeout_store(struct device *dev, struct device_attribute *attr, static DEVICE_ATTR(timeout, 0644, dasd_timeout_show, dasd_timeout_store); + +static ssize_t +dasd_path_reset_store(struct device *dev, struct device_attribute *attr, + const char *buf, size_t count) +{ + struct dasd_device *device; + unsigned int val; + + device = dasd_device_from_cdev(to_ccwdev(dev)); + if (IS_ERR(device)) + return -ENODEV; + + if ((kstrtouint(buf, 16, &val) != 0) || val > 0xff) + val = 0; + + if (device->discipline && device->discipline->reset_path) + device->discipline->reset_path(device, (__u8) val); + + dasd_put_device(device); + return count; +} + +static DEVICE_ATTR(path_reset, 0200, NULL, dasd_path_reset_store); + +static ssize_t dasd_hpf_show(struct device *dev, struct device_attribute *attr, + char *buf) +{ + struct dasd_device *device; + int hpf; + + device = dasd_device_from_cdev(to_ccwdev(dev)); + if (IS_ERR(device)) + return -ENODEV; + if (!device->discipline || !device->discipline->hpf_enabled) { + dasd_put_device(device); + return snprintf(buf, PAGE_SIZE, "%d\n", dasd_nofcx); + } + hpf = device->discipline->hpf_enabled(device); + dasd_put_device(device); + return snprintf(buf, PAGE_SIZE, "%d\n", hpf); +} + +static DEVICE_ATTR(hpf, 0444, dasd_hpf_show, NULL); + static ssize_t dasd_reservation_policy_show(struct device *dev, struct device_attribute *attr, char *buf) @@ -1385,27 +1412,17 @@ static ssize_t dasd_reservation_policy_store(struct device *dev, struct device_attribute *attr, const char *buf, size_t count) { - struct dasd_devmap *devmap; + struct ccw_device *cdev = to_ccwdev(dev); int rc; - devmap = dasd_devmap_from_cdev(to_ccwdev(dev)); - if (IS_ERR(devmap)) - return PTR_ERR(devmap); - rc = 0; - spin_lock(&dasd_devmap_lock); if (sysfs_streq("ignore", buf)) - devmap->features &= ~DASD_FEATURE_FAILONSLCK; + rc = dasd_set_feature(cdev, DASD_FEATURE_FAILONSLCK, 0); else if (sysfs_streq("fail", buf)) - devmap->features |= DASD_FEATURE_FAILONSLCK; + rc = dasd_set_feature(cdev, DASD_FEATURE_FAILONSLCK, 1); else rc = -EINVAL; - if (devmap->device) - devmap->device->features = devmap->features; - spin_unlock(&dasd_devmap_lock); - if (rc) - return rc; - else - return count; + + return rc ? : count; } static DEVICE_ATTR(reservation_policy, 0644, @@ -1461,25 +1478,120 @@ static ssize_t dasd_pm_show(struct device *dev, struct device_attribute *attr, char *buf) { struct dasd_device *device; - u8 opm, nppm, cablepm, cuirpm, hpfpm; + u8 opm, nppm, cablepm, cuirpm, hpfpm, ifccpm; device = dasd_device_from_cdev(to_ccwdev(dev)); if (IS_ERR(device)) return sprintf(buf, "0\n"); - opm = device->path_data.opm; - nppm = device->path_data.npm; - cablepm = device->path_data.cablepm; - cuirpm = device->path_data.cuirpm; - hpfpm = device->path_data.hpfpm; + opm = dasd_path_get_opm(device); + nppm = dasd_path_get_nppm(device); + cablepm = dasd_path_get_cablepm(device); + cuirpm = dasd_path_get_cuirpm(device); + hpfpm = dasd_path_get_hpfpm(device); + ifccpm = dasd_path_get_ifccpm(device); dasd_put_device(device); - return sprintf(buf, "%02x %02x %02x %02x %02x\n", opm, nppm, - cablepm, cuirpm, hpfpm); + return sprintf(buf, "%02x %02x %02x %02x %02x %02x\n", opm, nppm, + cablepm, cuirpm, hpfpm, ifccpm); } static DEVICE_ATTR(path_masks, 0444, dasd_pm_show, NULL); +/* + * threshold value for IFCC/CCC errors + */ +static ssize_t +dasd_path_threshold_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct dasd_device *device; + int len; + + device = dasd_device_from_cdev(to_ccwdev(dev)); + if (IS_ERR(device)) + return -ENODEV; + len = snprintf(buf, PAGE_SIZE, "%lu\n", device->path_thrhld); + dasd_put_device(device); + return len; +} + +static ssize_t +dasd_path_threshold_store(struct device *dev, struct device_attribute *attr, + const char *buf, size_t count) +{ + struct dasd_device *device; + unsigned long flags; + unsigned long val; + + device = dasd_device_from_cdev(to_ccwdev(dev)); + if (IS_ERR(device)) + return -ENODEV; + + if ((kstrtoul(buf, 10, &val) != 0) || + (val > DASD_THRHLD_MAX) || val == 0) { + dasd_put_device(device); + return -EINVAL; + } + spin_lock_irqsave(get_ccwdev_lock(to_ccwdev(dev)), flags); + if (val) + device->path_thrhld = val; + spin_unlock_irqrestore(get_ccwdev_lock(to_ccwdev(dev)), flags); + dasd_put_device(device); + return count; +} + +static DEVICE_ATTR(path_threshold, 0644, dasd_path_threshold_show, + dasd_path_threshold_store); +/* + * interval for IFCC/CCC checks + * meaning time with no IFCC/CCC error before the error counter + * gets reset + */ +static ssize_t +dasd_path_interval_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct dasd_device *device; + int len; + + device = dasd_device_from_cdev(to_ccwdev(dev)); + if (IS_ERR(device)) + return -ENODEV; + len = snprintf(buf, PAGE_SIZE, "%lu\n", device->path_interval); + dasd_put_device(device); + return len; +} + +static ssize_t +dasd_path_interval_store(struct device *dev, struct device_attribute *attr, + const char *buf, size_t count) +{ + struct dasd_device *device; + unsigned long flags; + unsigned long val; + + device = dasd_device_from_cdev(to_ccwdev(dev)); + if (IS_ERR(device)) + return -ENODEV; + + if ((kstrtoul(buf, 10, &val) != 0) || + (val > DASD_INTERVAL_MAX) || val == 0) { + dasd_put_device(device); + return -EINVAL; + } + spin_lock_irqsave(get_ccwdev_lock(to_ccwdev(dev)), flags); + if (val) + device->path_interval = val; + spin_unlock_irqrestore(get_ccwdev_lock(to_ccwdev(dev)), flags); + dasd_put_device(device); + return count; +} + +static DEVICE_ATTR(path_interval, 0644, dasd_path_interval_show, + dasd_path_interval_store); + + static struct attribute * dasd_attrs[] = { &dev_attr_readonly.attr, &dev_attr_discipline.attr, @@ -1500,6 +1612,10 @@ static struct attribute * dasd_attrs[] = { &dev_attr_safe_offline.attr, &dev_attr_host_access_count.attr, &dev_attr_path_masks.attr, + &dev_attr_path_threshold.attr, + &dev_attr_path_interval.attr, + &dev_attr_path_reset.attr, + &dev_attr_hpf.attr, NULL, }; @@ -1531,7 +1647,7 @@ dasd_set_feature(struct ccw_device *cdev, int feature, int flag) { struct dasd_devmap *devmap; - devmap = dasd_find_busid(dev_name(&cdev->dev)); + devmap = dasd_devmap_from_cdev(cdev); if (IS_ERR(devmap)) return PTR_ERR(devmap); diff --git a/drivers/s390/block/dasd_eckd.c b/drivers/s390/block/dasd_eckd.c index a7a8847..67bf50c 100644 --- a/drivers/s390/block/dasd_eckd.c +++ b/drivers/s390/block/dasd_eckd.c @@ -1042,8 +1042,11 @@ static void dasd_eckd_clear_conf_data(struct dasd_device *device) private->conf_data = NULL; private->conf_len = 0; for (i = 0; i < 8; i++) { - kfree(private->path_conf_data[i]); - private->path_conf_data[i] = NULL; + kfree(device->path[i].conf_data); + device->path[i].conf_data = NULL; + device->path[i].cssid = 0; + device->path[i].ssid = 0; + device->path[i].chpid = 0; } } @@ -1055,13 +1058,14 @@ static int dasd_eckd_read_conf(struct dasd_device *device) int rc, path_err, pos; __u8 lpm, opm; struct dasd_eckd_private *private, path_private; - struct dasd_path *path_data; struct dasd_uid *uid; char print_path_uid[60], print_device_uid[60]; + struct channel_path_desc *chp_desc; + struct subchannel_id sch_id; private = device->private; - path_data = &device->path_data; opm = ccw_device_get_path_mask(device->cdev); + ccw_device_get_schid(device->cdev, &sch_id); conf_data_saved = 0; path_err = 0; /* get configuration data per operational path */ @@ -1081,7 +1085,7 @@ static int dasd_eckd_read_conf(struct dasd_device *device) "No configuration data " "retrieved"); /* no further analysis possible */ - path_data->opm |= lpm; + dasd_path_add_opm(device, opm); continue; /* no error */ } /* save first valid configuration data */ @@ -1098,8 +1102,13 @@ static int dasd_eckd_read_conf(struct dasd_device *device) } pos = pathmask_to_pos(lpm); /* store per path conf_data */ - private->path_conf_data[pos] = - (struct dasd_conf_data *) conf_data; + device->path[pos].conf_data = conf_data; + device->path[pos].cssid = sch_id.cssid; + device->path[pos].ssid = sch_id.ssid; + chp_desc = ccw_device_get_chp_desc(device->cdev, pos); + if (chp_desc) + device->path[pos].chpid = chp_desc->chpid; + kfree(chp_desc); /* * build device UID that other path data * can be compared to it @@ -1154,42 +1163,66 @@ static int dasd_eckd_read_conf(struct dasd_device *device) "device %s instead of %s\n", lpm, print_path_uid, print_device_uid); path_err = -EINVAL; - path_data->cablepm |= lpm; + dasd_path_add_cablepm(device, lpm); continue; } pos = pathmask_to_pos(lpm); /* store per path conf_data */ - private->path_conf_data[pos] = - (struct dasd_conf_data *) conf_data; + device->path[pos].conf_data = conf_data; + device->path[pos].cssid = sch_id.cssid; + device->path[pos].ssid = sch_id.ssid; + chp_desc = ccw_device_get_chp_desc(device->cdev, pos); + if (chp_desc) + device->path[pos].chpid = chp_desc->chpid; + kfree(chp_desc); path_private.conf_data = NULL; path_private.conf_len = 0; } switch (dasd_eckd_path_access(conf_data, conf_len)) { case 0x02: - path_data->npm |= lpm; + dasd_path_add_nppm(device, lpm); break; case 0x03: - path_data->ppm |= lpm; + dasd_path_add_ppm(device, lpm); break; } - if (!path_data->opm) { - path_data->opm = lpm; + if (!dasd_path_get_opm(device)) { + dasd_path_set_opm(device, lpm); dasd_generic_path_operational(device); } else { - path_data->opm |= lpm; + dasd_path_add_opm(device, lpm); } - /* - * if the path is used - * it should not be in one of the negative lists - */ - path_data->cablepm &= ~lpm; - path_data->hpfpm &= ~lpm; - path_data->cuirpm &= ~lpm; } return path_err; } +static u32 get_fcx_max_data(struct dasd_device *device) +{ + struct dasd_eckd_private *private = device->private; + int fcx_in_css, fcx_in_gneq, fcx_in_features; + int tpm, mdc; + + if (dasd_nofcx) + return 0; + /* is transport mode supported? */ + fcx_in_css = css_general_characteristics.fcx; + fcx_in_gneq = private->gneq->reserved2[7] & 0x04; + fcx_in_features = private->features.feature[40] & 0x80; + tpm = fcx_in_css && fcx_in_gneq && fcx_in_features; + + if (!tpm) + return 0; + + mdc = ccw_device_get_mdc(device->cdev, 0); + if (mdc < 0) { + dev_warn(&device->cdev->dev, "Detecting the maximum supported data size for zHPF requests failed\n"); + return 0; + } else { + return (u32)mdc * FCX_MAX_DATA_FACTOR; + } +} + static int verify_fcx_max_data(struct dasd_device *device, __u8 lpm) { struct dasd_eckd_private *private = device->private; @@ -1222,8 +1255,7 @@ static int rebuild_device_uid(struct dasd_device *device, struct path_verification_work_data *data) { struct dasd_eckd_private *private = device->private; - struct dasd_path *path_data = &device->path_data; - __u8 lpm, opm = path_data->opm; + __u8 lpm, opm = dasd_path_get_opm(device); int rc = -ENODEV; for (lpm = 0x80; lpm; lpm >>= 1) { @@ -1356,7 +1388,7 @@ static void do_path_verification_work(struct work_struct *work) * in other case the device UID may have changed and * the first working path UID will be used as device UID */ - if (device->path_data.opm && + if (dasd_path_get_opm(device) && dasd_eckd_compare_path_uid(device, &path_private)) { /* * the comparison was not successful @@ -1406,23 +1438,17 @@ static void do_path_verification_work(struct work_struct *work) * situation in dasd_start_IO. */ spin_lock_irqsave(get_ccwdev_lock(device->cdev), flags); - if (!device->path_data.opm && opm) { - device->path_data.opm = opm; - device->path_data.cablepm &= ~opm; - device->path_data.cuirpm &= ~opm; - device->path_data.hpfpm &= ~opm; + if (!dasd_path_get_opm(device) && opm) { + dasd_path_set_opm(device, opm); dasd_generic_path_operational(device); } else { - device->path_data.opm |= opm; - device->path_data.cablepm &= ~opm; - device->path_data.cuirpm &= ~opm; - device->path_data.hpfpm &= ~opm; + dasd_path_add_opm(device, opm); } - device->path_data.npm |= npm; - device->path_data.ppm |= ppm; - device->path_data.tbvpm |= epm; - device->path_data.cablepm |= cablepm; - device->path_data.hpfpm |= hpfpm; + dasd_path_add_nppm(device, npm); + dasd_path_add_ppm(device, ppm); + dasd_path_add_tbvpm(device, epm); + dasd_path_add_cablepm(device, cablepm); + dasd_path_add_nohpfpm(device, hpfpm); spin_unlock_irqrestore(get_ccwdev_lock(device->cdev), flags); } clear_bit(DASD_FLAG_PATH_VERIFY, &device->flags); @@ -1456,6 +1482,19 @@ static int dasd_eckd_verify_path(struct dasd_device *device, __u8 lpm) return 0; } +static void dasd_eckd_reset_path(struct dasd_device *device, __u8 pm) +{ + struct dasd_eckd_private *private = device->private; + unsigned long flags; + + if (!private->fcx_max_data) + private->fcx_max_data = get_fcx_max_data(device); + spin_lock_irqsave(get_ccwdev_lock(device->cdev), flags); + dasd_path_set_tbvpm(device, pm ? : dasd_path_get_notoperpm(device)); + dasd_schedule_device_bh(device); + spin_unlock_irqrestore(get_ccwdev_lock(device->cdev), flags); +} + static int dasd_eckd_read_features(struct dasd_device *device) { struct dasd_eckd_private *private = device->private; @@ -1652,32 +1691,6 @@ static void dasd_eckd_kick_validate_server(struct dasd_device *device) dasd_put_device(device); } -static u32 get_fcx_max_data(struct dasd_device *device) -{ - struct dasd_eckd_private *private = device->private; - int fcx_in_css, fcx_in_gneq, fcx_in_features; - int tpm, mdc; - - if (dasd_nofcx) - return 0; - /* is transport mode supported? */ - fcx_in_css = css_general_characteristics.fcx; - fcx_in_gneq = private->gneq->reserved2[7] & 0x04; - fcx_in_features = private->features.feature[40] & 0x80; - tpm = fcx_in_css && fcx_in_gneq && fcx_in_features; - - if (!tpm) - return 0; - - mdc = ccw_device_get_mdc(device->cdev, 0); - if (mdc < 0) { - dev_warn(&device->cdev->dev, "Detecting the maximum supported" - " data size for zHPF requests failed\n"); - return 0; - } else - return (u32)mdc * FCX_MAX_DATA_FACTOR; -} - /* * Check device characteristics. * If the device is accessible using ECKD discipline, the device is enabled. @@ -1729,10 +1742,11 @@ dasd_eckd_check_characteristics(struct dasd_device *device) if (rc) goto out_err1; - /* set default timeout */ + /* set some default values */ device->default_expires = DASD_EXPIRES; - /* set default retry count */ device->default_retries = DASD_RETRIES; + device->path_thrhld = DASD_ECKD_PATH_THRHLD; + device->path_interval = DASD_ECKD_PATH_INTERVAL; if (private->gneq) { value = 1; @@ -1839,13 +1853,16 @@ static void dasd_eckd_uncheck_device(struct dasd_device *device) private->gneq = NULL; private->conf_len = 0; for (i = 0; i < 8; i++) { - kfree(private->path_conf_data[i]); - if ((__u8 *)private->path_conf_data[i] == + kfree(device->path[i].conf_data); + if ((__u8 *)device->path[i].conf_data == private->conf_data) { private->conf_data = NULL; private->conf_len = 0; } - private->path_conf_data[i] = NULL; + device->path[i].conf_data = NULL; + device->path[i].cssid = 0; + device->path[i].ssid = 0; + device->path[i].chpid = 0; } kfree(private->conf_data); private->conf_data = NULL; @@ -2966,7 +2983,7 @@ static void dasd_eckd_handle_terminated_request(struct dasd_ccw_req *cqr) if (cqr->block && (cqr->startdev != cqr->block->base)) { dasd_eckd_reset_ccw_to_base_io(cqr); cqr->startdev = cqr->block->base; - cqr->lpm = cqr->block->base->path_data.opm; + cqr->lpm = dasd_path_get_opm(cqr->block->base); } }; @@ -3251,7 +3268,7 @@ static struct dasd_ccw_req *dasd_eckd_build_cp_cmd_single( cqr->memdev = startdev; cqr->block = block; cqr->expires = startdev->default_expires * HZ; /* default 5 minutes */ - cqr->lpm = startdev->path_data.ppm; + cqr->lpm = dasd_path_get_ppm(startdev); cqr->retries = startdev->default_retries; cqr->buildclk = get_tod_clock(); cqr->status = DASD_CQR_FILLED; @@ -3426,7 +3443,7 @@ static struct dasd_ccw_req *dasd_eckd_build_cp_cmd_track( cqr->memdev = startdev; cqr->block = block; cqr->expires = startdev->default_expires * HZ; /* default 5 minutes */ - cqr->lpm = startdev->path_data.ppm; + cqr->lpm = dasd_path_get_ppm(startdev); cqr->retries = startdev->default_retries; cqr->buildclk = get_tod_clock(); cqr->status = DASD_CQR_FILLED; @@ -3735,7 +3752,7 @@ static struct dasd_ccw_req *dasd_eckd_build_cp_tpm_track( cqr->memdev = startdev; cqr->block = block; cqr->expires = startdev->default_expires * HZ; /* default 5 minutes */ - cqr->lpm = startdev->path_data.ppm; + cqr->lpm = dasd_path_get_ppm(startdev); cqr->retries = startdev->default_retries; cqr->buildclk = get_tod_clock(); cqr->status = DASD_CQR_FILLED; @@ -3962,7 +3979,7 @@ static struct dasd_ccw_req *dasd_raw_build_cp(struct dasd_device *startdev, cqr->memdev = startdev; cqr->block = block; cqr->expires = startdev->default_expires * HZ; - cqr->lpm = startdev->path_data.ppm; + cqr->lpm = dasd_path_get_ppm(startdev); cqr->retries = startdev->default_retries; cqr->buildclk = get_tod_clock(); cqr->status = DASD_CQR_FILLED; @@ -4783,7 +4800,8 @@ static void dasd_eckd_dump_sense_tcw(struct dasd_device *device, req, scsw_cc(&irb->scsw), scsw_fctl(&irb->scsw), scsw_actl(&irb->scsw), scsw_stctl(&irb->scsw), scsw_dstat(&irb->scsw), scsw_cstat(&irb->scsw), - irb->scsw.tm.fcxs, irb->scsw.tm.schxs, + irb->scsw.tm.fcxs, + (irb->scsw.tm.ifob << 7) | irb->scsw.tm.sesq, req ? req->intrc : 0); len += sprintf(page + len, PRINTK_HEADER " device %s: Failing TCW: %p\n", @@ -5306,11 +5324,10 @@ static int dasd_hosts_print(struct dasd_device *device, struct seq_file *m) */ static int dasd_eckd_psf_cuir_response(struct dasd_device *device, int response, - __u32 message_id, - struct channel_path_desc *desc, - struct subchannel_id sch_id) + __u32 message_id, __u8 lpum) { struct dasd_psf_cuir_response *psf_cuir; + int pos = pathmask_to_pos(lpum); struct dasd_ccw_req *cqr; struct ccw1 *ccw; int rc; @@ -5328,11 +5345,10 @@ dasd_eckd_psf_cuir_response(struct dasd_device *device, int response, psf_cuir = (struct dasd_psf_cuir_response *)cqr->data; psf_cuir->order = PSF_ORDER_CUIR_RESPONSE; psf_cuir->cc = response; - if (desc) - psf_cuir->chpid = desc->chpid; + psf_cuir->chpid = device->path[pos].chpid; psf_cuir->message_id = message_id; - psf_cuir->cssid = sch_id.cssid; - psf_cuir->ssid = sch_id.ssid; + psf_cuir->cssid = device->path[pos].cssid; + psf_cuir->ssid = device->path[pos].ssid; ccw = cqr->cpaddr; ccw->cmd_code = DASD_ECKD_CCW_PSF; ccw->cda = (__u32)(addr_t)psf_cuir; @@ -5363,20 +5379,19 @@ static struct dasd_conf_data *dasd_eckd_get_ref_conf(struct dasd_device *device, __u8 lpum, struct dasd_cuir_message *cuir) { - struct dasd_eckd_private *private = device->private; struct dasd_conf_data *conf_data; int path, pos; if (cuir->record_selector == 0) goto out; for (path = 0x80, pos = 0; path; path >>= 1, pos++) { - conf_data = private->path_conf_data[pos]; + conf_data = device->path[pos].conf_data; if (conf_data->gneq.record_selector == cuir->record_selector) return conf_data; } out: - return private->path_conf_data[pathmask_to_pos(lpum)]; + return device->path[pathmask_to_pos(lpum)].conf_data; } /* @@ -5391,7 +5406,6 @@ out: static int dasd_eckd_cuir_scope(struct dasd_device *device, __u8 lpum, struct dasd_cuir_message *cuir) { - struct dasd_eckd_private *private = device->private; struct dasd_conf_data *ref_conf_data; unsigned long bitmask = 0, mask = 0; struct dasd_conf_data *conf_data; @@ -5417,11 +5431,10 @@ static int dasd_eckd_cuir_scope(struct dasd_device *device, __u8 lpum, mask |= cuir->neq_map[1] << 8; mask |= cuir->neq_map[0] << 16; - for (path = 0x80; path; path >>= 1) { + for (path = 0; path < 8; path++) { /* initialise data per path */ bitmask = mask; - pos = pathmask_to_pos(path); - conf_data = private->path_conf_data[pos]; + conf_data = device->path[path].conf_data; pos = 8 - ffs(cuir->ned_map); ned = (char *) &conf_data->neds[pos]; /* compare reference ned and per path ned */ @@ -5442,33 +5455,29 @@ static int dasd_eckd_cuir_scope(struct dasd_device *device, __u8 lpum, continue; /* device and path match the reference values add path to CUIR scope */ - tbcpm |= path; + tbcpm |= 0x80 >> path; } return tbcpm; } static void dasd_eckd_cuir_notify_user(struct dasd_device *device, - unsigned long paths, - struct subchannel_id sch_id, int action) + unsigned long paths, int action) { - struct channel_path_desc *desc; int pos; while (paths) { /* get position of bit in mask */ - pos = ffs(paths) - 1; + pos = 8 - ffs(paths); /* get channel path descriptor from this position */ - desc = ccw_device_get_chp_desc(device->cdev, 7 - pos); if (action == CUIR_QUIESCE) - pr_warn("Service on the storage server caused path " - "%x.%02x to go offline", sch_id.cssid, - desc ? desc->chpid : 0); + pr_warn("Service on the storage server caused path %x.%02x to go offline", + device->path[pos].cssid, + device->path[pos].chpid); else if (action == CUIR_RESUME) - pr_info("Path %x.%02x is back online after service " - "on the storage server", sch_id.cssid, - desc ? desc->chpid : 0); - kfree(desc); - clear_bit(pos, &paths); + pr_info("Path %x.%02x is back online after service on the storage server", + device->path[pos].cssid, + device->path[pos].chpid); + clear_bit(7 - pos, &paths); } } @@ -5479,16 +5488,16 @@ static int dasd_eckd_cuir_remove_path(struct dasd_device *device, __u8 lpum, tbcpm = dasd_eckd_cuir_scope(device, lpum, cuir); /* nothing to do if path is not in use */ - if (!(device->path_data.opm & tbcpm)) + if (!(dasd_path_get_opm(device) & tbcpm)) return 0; - if (!(device->path_data.opm & ~tbcpm)) { + if (!(dasd_path_get_opm(device) & ~tbcpm)) { /* no path would be left if the CUIR action is taken return error */ return -EINVAL; } /* remove device from operational path mask */ - device->path_data.opm &= ~tbcpm; - device->path_data.cuirpm |= tbcpm; + dasd_path_remove_opm(device, tbcpm); + dasd_path_add_cuirpm(device, tbcpm); return tbcpm; } @@ -5501,7 +5510,6 @@ static int dasd_eckd_cuir_remove_path(struct dasd_device *device, __u8 lpum, * notify the already set offline devices again */ static int dasd_eckd_cuir_quiesce(struct dasd_device *device, __u8 lpum, - struct subchannel_id sch_id, struct dasd_cuir_message *cuir) { struct dasd_eckd_private *private = device->private; @@ -5556,14 +5564,13 @@ static int dasd_eckd_cuir_quiesce(struct dasd_device *device, __u8 lpum, } } /* notify user about all paths affected by CUIR action */ - dasd_eckd_cuir_notify_user(device, paths, sch_id, CUIR_QUIESCE); + dasd_eckd_cuir_notify_user(device, paths, CUIR_QUIESCE); return 0; out_err: return tbcpm; } static int dasd_eckd_cuir_resume(struct dasd_device *device, __u8 lpum, - struct subchannel_id sch_id, struct dasd_cuir_message *cuir) { struct dasd_eckd_private *private = device->private; @@ -5581,8 +5588,8 @@ static int dasd_eckd_cuir_resume(struct dasd_device *device, __u8 lpum, alias_list) { tbcpm = dasd_eckd_cuir_scope(dev, lpum, cuir); paths |= tbcpm; - if (!(dev->path_data.opm & tbcpm)) { - dev->path_data.tbvpm |= tbcpm; + if (!(dasd_path_get_opm(dev) & tbcpm)) { + dasd_path_add_tbvpm(dev, tbcpm); dasd_schedule_device_bh(dev); } } @@ -5591,8 +5598,8 @@ static int dasd_eckd_cuir_resume(struct dasd_device *device, __u8 lpum, alias_list) { tbcpm = dasd_eckd_cuir_scope(dev, lpum, cuir); paths |= tbcpm; - if (!(dev->path_data.opm & tbcpm)) { - dev->path_data.tbvpm |= tbcpm; + if (!(dasd_path_get_opm(dev) & tbcpm)) { + dasd_path_add_tbvpm(dev, tbcpm); dasd_schedule_device_bh(dev); } } @@ -5605,8 +5612,8 @@ static int dasd_eckd_cuir_resume(struct dasd_device *device, __u8 lpum, alias_list) { tbcpm = dasd_eckd_cuir_scope(dev, lpum, cuir); paths |= tbcpm; - if (!(dev->path_data.opm & tbcpm)) { - dev->path_data.tbvpm |= tbcpm; + if (!(dasd_path_get_opm(dev) & tbcpm)) { + dasd_path_add_tbvpm(dev, tbcpm); dasd_schedule_device_bh(dev); } } @@ -5615,14 +5622,14 @@ static int dasd_eckd_cuir_resume(struct dasd_device *device, __u8 lpum, alias_list) { tbcpm = dasd_eckd_cuir_scope(dev, lpum, cuir); paths |= tbcpm; - if (!(dev->path_data.opm & tbcpm)) { - dev->path_data.tbvpm |= tbcpm; + if (!(dasd_path_get_opm(dev) & tbcpm)) { + dasd_path_add_tbvpm(dev, tbcpm); dasd_schedule_device_bh(dev); } } } /* notify user about all paths affected by CUIR action */ - dasd_eckd_cuir_notify_user(device, paths, sch_id, CUIR_RESUME); + dasd_eckd_cuir_notify_user(device, paths, CUIR_RESUME); return 0; } @@ -5630,38 +5637,31 @@ static void dasd_eckd_handle_cuir(struct dasd_device *device, void *messages, __u8 lpum) { struct dasd_cuir_message *cuir = messages; - struct channel_path_desc *desc; - struct subchannel_id sch_id; - int pos, response; + int response; DBF_DEV_EVENT(DBF_WARNING, device, "CUIR request: %016llx %016llx %016llx %08x", ((u64 *)cuir)[0], ((u64 *)cuir)[1], ((u64 *)cuir)[2], ((u32 *)cuir)[3]); - ccw_device_get_schid(device->cdev, &sch_id); - pos = pathmask_to_pos(lpum); - desc = ccw_device_get_chp_desc(device->cdev, pos); if (cuir->code == CUIR_QUIESCE) { /* quiesce */ - if (dasd_eckd_cuir_quiesce(device, lpum, sch_id, cuir)) + if (dasd_eckd_cuir_quiesce(device, lpum, cuir)) response = PSF_CUIR_LAST_PATH; else response = PSF_CUIR_COMPLETED; } else if (cuir->code == CUIR_RESUME) { /* resume */ - dasd_eckd_cuir_resume(device, lpum, sch_id, cuir); + dasd_eckd_cuir_resume(device, lpum, cuir); response = PSF_CUIR_COMPLETED; } else response = PSF_CUIR_NOT_SUPPORTED; dasd_eckd_psf_cuir_response(device, response, - cuir->message_id, desc, sch_id); + cuir->message_id, lpum); DBF_DEV_EVENT(DBF_WARNING, device, "CUIR response: %d on message ID %08x", response, cuir->message_id); - /* free descriptor copy */ - kfree(desc); /* to make sure there is no attention left schedule work again */ device->discipline->check_attention(device, lpum); } @@ -5708,6 +5708,63 @@ static int dasd_eckd_check_attention(struct dasd_device *device, __u8 lpum) return 0; } +static int dasd_eckd_disable_hpf_path(struct dasd_device *device, __u8 lpum) +{ + if (~lpum & dasd_path_get_opm(device)) { + dasd_path_add_nohpfpm(device, lpum); + dasd_path_remove_opm(device, lpum); + dev_err(&device->cdev->dev, + "Channel path %02X lost HPF functionality and is disabled\n", + lpum); + return 1; + } + return 0; +} + +static void dasd_eckd_disable_hpf_device(struct dasd_device *device) +{ + struct dasd_eckd_private *private = device->private; + + dev_err(&device->cdev->dev, + "High Performance FICON disabled\n"); + private->fcx_max_data = 0; +} + +static int dasd_eckd_hpf_enabled(struct dasd_device *device) +{ + struct dasd_eckd_private *private = device->private; + + return private->fcx_max_data ? 1 : 0; +} + +static void dasd_eckd_handle_hpf_error(struct dasd_device *device, + struct irb *irb) +{ + struct dasd_eckd_private *private = device->private; + + if (!private->fcx_max_data) { + /* sanity check for no HPF, the error makes no sense */ + DBF_DEV_EVENT(DBF_WARNING, device, "%s", + "Trying to disable HPF for a non HPF device"); + return; + } + if (irb->scsw.tm.sesq == SCSW_SESQ_DEV_NOFCX) { + dasd_eckd_disable_hpf_device(device); + } else if (irb->scsw.tm.sesq == SCSW_SESQ_PATH_NOFCX) { + if (dasd_eckd_disable_hpf_path(device, irb->esw.esw1.lpum)) + return; + dasd_eckd_disable_hpf_device(device); + dasd_path_set_tbvpm(device, + dasd_path_get_hpfpm(device)); + } + /* + * prevent that any new I/O ist started on the device and schedule a + * requeue of existing requests + */ + dasd_device_set_stop_bits(device, DASD_STOPPED_NOT_ACC); + dasd_schedule_requeue(device); +} + static struct ccw_driver dasd_eckd_driver = { .driver = { .name = "dasd-eckd", @@ -5776,6 +5833,10 @@ static struct dasd_discipline dasd_eckd_discipline = { .check_attention = dasd_eckd_check_attention, .host_access_count = dasd_eckd_host_access_count, .hosts_print = dasd_hosts_print, + .handle_hpf_error = dasd_eckd_handle_hpf_error, + .disable_hpf = dasd_eckd_disable_hpf_device, + .hpf_enabled = dasd_eckd_hpf_enabled, + .reset_path = dasd_eckd_reset_path, }; static int __init diff --git a/drivers/s390/block/dasd_eckd.h b/drivers/s390/block/dasd_eckd.h index 5980362..e2a710c 100644 --- a/drivers/s390/block/dasd_eckd.h +++ b/drivers/s390/block/dasd_eckd.h @@ -94,6 +94,8 @@ #define FCX_MAX_DATA_FACTOR 65536 #define DASD_ECKD_RCD_DATA_SIZE 256 +#define DASD_ECKD_PATH_THRHLD 256 +#define DASD_ECKD_PATH_INTERVAL 300 /***************************************************************************** * SECTION: Type Definitions @@ -535,8 +537,7 @@ struct dasd_eckd_private { struct dasd_eckd_characteristics rdc_data; u8 *conf_data; int conf_len; - /* per path configuration data */ - struct dasd_conf_data *path_conf_data[8]; + /* pointers to specific parts in the conf_data */ struct dasd_ned *ned; struct dasd_sneq *sneq; diff --git a/drivers/s390/block/dasd_eer.c b/drivers/s390/block/dasd_eer.c index 21ef63c..6c5d671 100644 --- a/drivers/s390/block/dasd_eer.c +++ b/drivers/s390/block/dasd_eer.c @@ -454,20 +454,30 @@ static void dasd_eer_snss_cb(struct dasd_ccw_req *cqr, void *data) */ int dasd_eer_enable(struct dasd_device *device) { - struct dasd_ccw_req *cqr; + struct dasd_ccw_req *cqr = NULL; unsigned long flags; struct ccw1 *ccw; + int rc = 0; + spin_lock_irqsave(get_ccwdev_lock(device->cdev), flags); if (device->eer_cqr) - return 0; + goto out; + else if (!device->discipline || + strcmp(device->discipline->name, "ECKD")) + rc = -EMEDIUMTYPE; + else if (test_bit(DASD_FLAG_OFFLINE, &device->flags)) + rc = -EBUSY; - if (!device->discipline || strcmp(device->discipline->name, "ECKD")) - return -EPERM; /* FIXME: -EMEDIUMTYPE ? */ + if (rc) + goto out; cqr = dasd_kmalloc_request(DASD_ECKD_MAGIC, 1 /* SNSS */, SNSS_DATA_SIZE, device); - if (IS_ERR(cqr)) - return -ENOMEM; + if (IS_ERR(cqr)) { + rc = -ENOMEM; + cqr = NULL; + goto out; + } cqr->startdev = device; cqr->retries = 255; @@ -485,15 +495,18 @@ int dasd_eer_enable(struct dasd_device *device) cqr->status = DASD_CQR_FILLED; cqr->callback = dasd_eer_snss_cb; - spin_lock_irqsave(get_ccwdev_lock(device->cdev), flags); if (!device->eer_cqr) { device->eer_cqr = cqr; cqr = NULL; } + +out: spin_unlock_irqrestore(get_ccwdev_lock(device->cdev), flags); + if (cqr) dasd_kfree_request(cqr, device); - return 0; + + return rc; } /* diff --git a/drivers/s390/block/dasd_erp.c b/drivers/s390/block/dasd_erp.c index d138d01..113c1c1 100644 --- a/drivers/s390/block/dasd_erp.c +++ b/drivers/s390/block/dasd_erp.c @@ -96,7 +96,7 @@ dasd_default_erp_action(struct dasd_ccw_req *cqr) "default ERP called (%i retries left)", cqr->retries); if (!test_bit(DASD_CQR_VERIFY_PATH, &cqr->flags)) - cqr->lpm = device->path_data.opm; + cqr->lpm = dasd_path_get_opm(device); cqr->status = DASD_CQR_FILLED; } else { pr_err("%s: default ERP has run out of retries and failed\n", diff --git a/drivers/s390/block/dasd_fba.c b/drivers/s390/block/dasd_fba.c index d7b5b55..462cab5 100644 --- a/drivers/s390/block/dasd_fba.c +++ b/drivers/s390/block/dasd_fba.c @@ -168,7 +168,7 @@ dasd_fba_check_characteristics(struct dasd_device *device) device->default_expires = DASD_EXPIRES; device->default_retries = FBA_DEFAULT_RETRIES; - device->path_data.opm = LPM_ANYPATH; + dasd_path_set_opm(device, LPM_ANYPATH); readonly = dasd_device_is_ro(device); if (readonly) diff --git a/drivers/s390/block/dasd_int.h b/drivers/s390/block/dasd_int.h index 87ff6ce..24be210 100644 --- a/drivers/s390/block/dasd_int.h +++ b/drivers/s390/block/dasd_int.h @@ -55,6 +55,7 @@ #include <asm/debug.h> #include <asm/dasd.h> #include <asm/idals.h> +#include <linux/bitops.h> /* DASD discipline magic */ #define DASD_ECKD_MAGIC 0xC5C3D2C4 @@ -377,6 +378,10 @@ struct dasd_discipline { int (*check_attention)(struct dasd_device *, __u8); int (*host_access_count)(struct dasd_device *); int (*hosts_print)(struct dasd_device *, struct seq_file *); + void (*handle_hpf_error)(struct dasd_device *, struct irb *); + void (*disable_hpf)(struct dasd_device *); + int (*hpf_enabled)(struct dasd_device *); + void (*reset_path)(struct dasd_device *, __u8); }; extern struct dasd_discipline *dasd_diag_discipline_pointer; @@ -397,17 +402,31 @@ extern struct dasd_discipline *dasd_diag_discipline_pointer; #define DASD_EER_STATECHANGE 3 #define DASD_EER_PPRCSUSPEND 4 +/* DASD path handling */ + +#define DASD_PATH_OPERATIONAL 1 +#define DASD_PATH_TBV 2 +#define DASD_PATH_PP 3 +#define DASD_PATH_NPP 4 +#define DASD_PATH_MISCABLED 5 +#define DASD_PATH_NOHPF 6 +#define DASD_PATH_CUIR 7 +#define DASD_PATH_IFCC 8 + +#define DASD_THRHLD_MAX 4294967295U +#define DASD_INTERVAL_MAX 4294967295U + struct dasd_path { - __u8 opm; - __u8 tbvpm; - __u8 ppm; - __u8 npm; - /* paths that are not used because of a special condition */ - __u8 cablepm; /* miss-cabled */ - __u8 hpfpm; /* the HPF requirements of the other paths are not met */ - __u8 cuirpm; /* CUIR varied offline */ + unsigned long flags; + u8 cssid; + u8 ssid; + u8 chpid; + struct dasd_conf_data *conf_data; + atomic_t error_count; + unsigned long long errorclk; }; + struct dasd_profile_info { /* legacy part of profile data, as in dasd_profile_info_t */ unsigned int dasd_io_reqs; /* number of requests processed */ @@ -458,7 +477,8 @@ struct dasd_device { struct dasd_discipline *discipline; struct dasd_discipline *base_discipline; void *private; - struct dasd_path path_data; + struct dasd_path path[8]; + __u8 opm; /* Device state and target state. */ int state, target; @@ -483,6 +503,7 @@ struct dasd_device { struct work_struct reload_device; struct work_struct kick_validate; struct work_struct suc_work; + struct work_struct requeue_requests; struct timer_list timer; debug_info_t *debug_area; @@ -498,6 +519,9 @@ struct dasd_device { unsigned long blk_timeout; + unsigned long path_thrhld; + unsigned long path_interval; + struct dentry *debugfs_dentry; struct dentry *hosts_dentry; struct dasd_profile profile; @@ -707,6 +731,7 @@ void dasd_set_target_state(struct dasd_device *, int); void dasd_kick_device(struct dasd_device *); void dasd_restore_device(struct dasd_device *); void dasd_reload_device(struct dasd_device *); +void dasd_schedule_requeue(struct dasd_device *); void dasd_add_request_head(struct dasd_ccw_req *); void dasd_add_request_tail(struct dasd_ccw_req *); @@ -835,4 +860,410 @@ static inline int dasd_eer_enabled(struct dasd_device *device) #define dasd_eer_enabled(d) (0) #endif /* CONFIG_DASD_ERR */ + +/* DASD path handling functions */ + +/* + * helper functions to modify bit masks for a given channel path for a device + */ +static inline int dasd_path_is_operational(struct dasd_device *device, int chp) +{ + return test_bit(DASD_PATH_OPERATIONAL, &device->path[chp].flags); +} + +static inline int dasd_path_need_verify(struct dasd_device *device, int chp) +{ + return test_bit(DASD_PATH_TBV, &device->path[chp].flags); +} + +static inline void dasd_path_verify(struct dasd_device *device, int chp) +{ + __set_bit(DASD_PATH_TBV, &device->path[chp].flags); +} + +static inline void dasd_path_clear_verify(struct dasd_device *device, int chp) +{ + __clear_bit(DASD_PATH_TBV, &device->path[chp].flags); +} + +static inline void dasd_path_clear_all_verify(struct dasd_device *device) +{ + int chp; + + for (chp = 0; chp < 8; chp++) + dasd_path_clear_verify(device, chp); +} + +static inline void dasd_path_operational(struct dasd_device *device, int chp) +{ + __set_bit(DASD_PATH_OPERATIONAL, &device->path[chp].flags); + device->opm |= (0x80 >> chp); +} + +static inline void dasd_path_nonpreferred(struct dasd_device *device, int chp) +{ + __set_bit(DASD_PATH_NPP, &device->path[chp].flags); +} + +static inline int dasd_path_is_nonpreferred(struct dasd_device *device, int chp) +{ + return test_bit(DASD_PATH_NPP, &device->path[chp].flags); +} + +static inline void dasd_path_clear_nonpreferred(struct dasd_device *device, + int chp) +{ + __clear_bit(DASD_PATH_NPP, &device->path[chp].flags); +} + +static inline void dasd_path_preferred(struct dasd_device *device, int chp) +{ + __set_bit(DASD_PATH_PP, &device->path[chp].flags); +} + +static inline int dasd_path_is_preferred(struct dasd_device *device, int chp) +{ + return test_bit(DASD_PATH_PP, &device->path[chp].flags); +} + +static inline void dasd_path_clear_preferred(struct dasd_device *device, + int chp) +{ + __clear_bit(DASD_PATH_PP, &device->path[chp].flags); +} + +static inline void dasd_path_clear_oper(struct dasd_device *device, int chp) +{ + __clear_bit(DASD_PATH_OPERATIONAL, &device->path[chp].flags); + device->opm &= ~(0x80 >> chp); +} + +static inline void dasd_path_clear_cable(struct dasd_device *device, int chp) +{ + __clear_bit(DASD_PATH_MISCABLED, &device->path[chp].flags); +} + +static inline void dasd_path_cuir(struct dasd_device *device, int chp) +{ + __set_bit(DASD_PATH_CUIR, &device->path[chp].flags); +} + +static inline int dasd_path_is_cuir(struct dasd_device *device, int chp) +{ + return test_bit(DASD_PATH_CUIR, &device->path[chp].flags); +} + +static inline void dasd_path_clear_cuir(struct dasd_device *device, int chp) +{ + __clear_bit(DASD_PATH_CUIR, &device->path[chp].flags); +} + +static inline void dasd_path_ifcc(struct dasd_device *device, int chp) +{ + set_bit(DASD_PATH_IFCC, &device->path[chp].flags); +} + +static inline int dasd_path_is_ifcc(struct dasd_device *device, int chp) +{ + return test_bit(DASD_PATH_IFCC, &device->path[chp].flags); +} + +static inline void dasd_path_clear_ifcc(struct dasd_device *device, int chp) +{ + clear_bit(DASD_PATH_IFCC, &device->path[chp].flags); +} + +static inline void dasd_path_clear_nohpf(struct dasd_device *device, int chp) +{ + __clear_bit(DASD_PATH_NOHPF, &device->path[chp].flags); +} + +static inline void dasd_path_miscabled(struct dasd_device *device, int chp) +{ + __set_bit(DASD_PATH_MISCABLED, &device->path[chp].flags); +} + +static inline int dasd_path_is_miscabled(struct dasd_device *device, int chp) +{ + return test_bit(DASD_PATH_MISCABLED, &device->path[chp].flags); +} + +static inline void dasd_path_nohpf(struct dasd_device *device, int chp) +{ + __set_bit(DASD_PATH_NOHPF, &device->path[chp].flags); +} + +static inline int dasd_path_is_nohpf(struct dasd_device *device, int chp) +{ + return test_bit(DASD_PATH_NOHPF, &device->path[chp].flags); +} + +/* + * get functions for path masks + * will return a path masks for the given device + */ + +static inline __u8 dasd_path_get_opm(struct dasd_device *device) +{ + return device->opm; +} + +static inline __u8 dasd_path_get_tbvpm(struct dasd_device *device) +{ + int chp; + __u8 tbvpm = 0x00; + + for (chp = 0; chp < 8; chp++) + if (dasd_path_need_verify(device, chp)) + tbvpm |= 0x80 >> chp; + return tbvpm; +} + +static inline __u8 dasd_path_get_nppm(struct dasd_device *device) +{ + int chp; + __u8 npm = 0x00; + + for (chp = 0; chp < 8; chp++) { + if (dasd_path_is_nonpreferred(device, chp)) + npm |= 0x80 >> chp; + } + return npm; +} + +static inline __u8 dasd_path_get_ppm(struct dasd_device *device) +{ + int chp; + __u8 ppm = 0x00; + + for (chp = 0; chp < 8; chp++) + if (dasd_path_is_preferred(device, chp)) + ppm |= 0x80 >> chp; + return ppm; +} + +static inline __u8 dasd_path_get_cablepm(struct dasd_device *device) +{ + int chp; + __u8 cablepm = 0x00; + + for (chp = 0; chp < 8; chp++) + if (dasd_path_is_miscabled(device, chp)) + cablepm |= 0x80 >> chp; + return cablepm; +} + +static inline __u8 dasd_path_get_cuirpm(struct dasd_device *device) +{ + int chp; + __u8 cuirpm = 0x00; + + for (chp = 0; chp < 8; chp++) + if (dasd_path_is_cuir(device, chp)) + cuirpm |= 0x80 >> chp; + return cuirpm; +} + +static inline __u8 dasd_path_get_ifccpm(struct dasd_device *device) +{ + int chp; + __u8 ifccpm = 0x00; + + for (chp = 0; chp < 8; chp++) + if (dasd_path_is_ifcc(device, chp)) + ifccpm |= 0x80 >> chp; + return ifccpm; +} + +static inline __u8 dasd_path_get_hpfpm(struct dasd_device *device) +{ + int chp; + __u8 hpfpm = 0x00; + + for (chp = 0; chp < 8; chp++) + if (dasd_path_is_nohpf(device, chp)) + hpfpm |= 0x80 >> chp; + return hpfpm; +} + +/* + * add functions for path masks + * the existing path mask will be extended by the given path mask + */ +static inline void dasd_path_add_tbvpm(struct dasd_device *device, __u8 pm) +{ + int chp; + + for (chp = 0; chp < 8; chp++) + if (pm & (0x80 >> chp)) + dasd_path_verify(device, chp); +} + +static inline __u8 dasd_path_get_notoperpm(struct dasd_device *device) +{ + int chp; + __u8 nopm = 0x00; + + for (chp = 0; chp < 8; chp++) + if (dasd_path_is_nohpf(device, chp) || + dasd_path_is_ifcc(device, chp) || + dasd_path_is_cuir(device, chp) || + dasd_path_is_miscabled(device, chp)) + nopm |= 0x80 >> chp; + return nopm; +} + +static inline void dasd_path_add_opm(struct dasd_device *device, __u8 pm) +{ + int chp; + + for (chp = 0; chp < 8; chp++) + if (pm & (0x80 >> chp)) { + dasd_path_operational(device, chp); + /* + * if the path is used + * it should not be in one of the negative lists + */ + dasd_path_clear_nohpf(device, chp); + dasd_path_clear_cuir(device, chp); + dasd_path_clear_cable(device, chp); + dasd_path_clear_ifcc(device, chp); + } +} + +static inline void dasd_path_add_cablepm(struct dasd_device *device, __u8 pm) +{ + int chp; + + for (chp = 0; chp < 8; chp++) + if (pm & (0x80 >> chp)) + dasd_path_miscabled(device, chp); +} + +static inline void dasd_path_add_cuirpm(struct dasd_device *device, __u8 pm) +{ + int chp; + + for (chp = 0; chp < 8; chp++) + if (pm & (0x80 >> chp)) + dasd_path_cuir(device, chp); +} + +static inline void dasd_path_add_ifccpm(struct dasd_device *device, __u8 pm) +{ + int chp; + + for (chp = 0; chp < 8; chp++) + if (pm & (0x80 >> chp)) + dasd_path_ifcc(device, chp); +} + +static inline void dasd_path_add_nppm(struct dasd_device *device, __u8 pm) +{ + int chp; + + for (chp = 0; chp < 8; chp++) + if (pm & (0x80 >> chp)) + dasd_path_nonpreferred(device, chp); +} + +static inline void dasd_path_add_nohpfpm(struct dasd_device *device, __u8 pm) +{ + int chp; + + for (chp = 0; chp < 8; chp++) + if (pm & (0x80 >> chp)) + dasd_path_nohpf(device, chp); +} + +static inline void dasd_path_add_ppm(struct dasd_device *device, __u8 pm) +{ + int chp; + + for (chp = 0; chp < 8; chp++) + if (pm & (0x80 >> chp)) + dasd_path_preferred(device, chp); +} + +/* + * set functions for path masks + * the existing path mask will be replaced by the given path mask + */ +static inline void dasd_path_set_tbvpm(struct dasd_device *device, __u8 pm) +{ + int chp; + + for (chp = 0; chp < 8; chp++) + if (pm & (0x80 >> chp)) + dasd_path_verify(device, chp); + else + dasd_path_clear_verify(device, chp); +} + +static inline void dasd_path_set_opm(struct dasd_device *device, __u8 pm) +{ + int chp; + + for (chp = 0; chp < 8; chp++) { + dasd_path_clear_oper(device, chp); + if (pm & (0x80 >> chp)) { + dasd_path_operational(device, chp); + /* + * if the path is used + * it should not be in one of the negative lists + */ + dasd_path_clear_nohpf(device, chp); + dasd_path_clear_cuir(device, chp); + dasd_path_clear_cable(device, chp); + dasd_path_clear_ifcc(device, chp); + } + } +} + +/* + * remove functions for path masks + * the existing path mask will be cleared with the given path mask + */ +static inline void dasd_path_remove_opm(struct dasd_device *device, __u8 pm) +{ + int chp; + + for (chp = 0; chp < 8; chp++) { + if (pm & (0x80 >> chp)) + dasd_path_clear_oper(device, chp); + } +} + +/* + * add the newly available path to the to be verified pm and remove it from + * normal operation until it is verified + */ +static inline void dasd_path_available(struct dasd_device *device, int chp) +{ + dasd_path_clear_oper(device, chp); + dasd_path_verify(device, chp); +} + +static inline void dasd_path_notoper(struct dasd_device *device, int chp) +{ + dasd_path_clear_oper(device, chp); + dasd_path_clear_preferred(device, chp); + dasd_path_clear_nonpreferred(device, chp); +} + +/* + * remove all paths from normal operation + */ +static inline void dasd_path_no_path(struct dasd_device *device) +{ + int chp; + + for (chp = 0; chp < 8; chp++) + dasd_path_notoper(device, chp); + + dasd_path_clear_all_verify(device); +} + +/* end - path handling */ + #endif /* DASD_H */ diff --git a/drivers/s390/char/con3215.c b/drivers/s390/char/con3215.c index 931d10e..1b8d825 100644 --- a/drivers/s390/char/con3215.c +++ b/drivers/s390/char/con3215.c @@ -9,7 +9,6 @@ * Dan Morrison, IBM Corporation <dmorriso@cse.buffalo.edu> */ -#include <linux/module.h> #include <linux/types.h> #include <linux/kdev_t.h> #include <linux/tty.h> @@ -1215,13 +1214,4 @@ static int __init tty3215_init(void) tty3215_driver = driver; return 0; } - -static void __exit tty3215_exit(void) -{ - tty_unregister_driver(tty3215_driver); - put_tty_driver(tty3215_driver); - ccw_driver_unregister(&raw3215_ccw_driver); -} - -module_init(tty3215_init); -module_exit(tty3215_exit); +device_initcall(tty3215_init); diff --git a/drivers/s390/char/sclp.h b/drivers/s390/char/sclp.h index 7a10c56..e1fc7eb 100644 --- a/drivers/s390/char/sclp.h +++ b/drivers/s390/char/sclp.h @@ -59,6 +59,7 @@ typedef unsigned int sclp_cmdw_t; +#define SCLP_CMDW_READ_CPU_INFO 0x00010001 #define SCLP_CMDW_READ_EVENT_DATA 0x00770005 #define SCLP_CMDW_WRITE_EVENT_DATA 0x00760005 #define SCLP_CMDW_WRITE_EVENT_MASK 0x00780005 @@ -102,6 +103,28 @@ struct init_sccb { sccb_mask_t sclp_send_mask; } __attribute__((packed)); +struct read_cpu_info_sccb { + struct sccb_header header; + u16 nr_configured; + u16 offset_configured; + u16 nr_standby; + u16 offset_standby; + u8 reserved[4096 - 16]; +} __attribute__((packed, aligned(PAGE_SIZE))); + +static inline void sclp_fill_core_info(struct sclp_core_info *info, + struct read_cpu_info_sccb *sccb) +{ + char *page = (char *) sccb; + + memset(info, 0, sizeof(*info)); + info->configured = sccb->nr_configured; + info->standby = sccb->nr_standby; + info->combined = sccb->nr_configured + sccb->nr_standby; + memcpy(&info->core, page + sccb->offset_configured, + info->combined * sizeof(struct sclp_core_entry)); +} + #define SCLP_HAS_CHP_INFO (sclp.facilities & 0x8000000000000000ULL) #define SCLP_HAS_CHP_RECONFIG (sclp.facilities & 0x2000000000000000ULL) #define SCLP_HAS_CPU_INFO (sclp.facilities & 0x0800000000000000ULL) diff --git a/drivers/s390/char/sclp_cmd.c b/drivers/s390/char/sclp_cmd.c index e3fc753..b9c5522 100644 --- a/drivers/s390/char/sclp_cmd.c +++ b/drivers/s390/char/sclp_cmd.c @@ -80,33 +80,10 @@ out: * CPU configuration related functions. */ -#define SCLP_CMDW_READ_CPU_INFO 0x00010001 #define SCLP_CMDW_CONFIGURE_CPU 0x00110001 #define SCLP_CMDW_DECONFIGURE_CPU 0x00100001 -struct read_cpu_info_sccb { - struct sccb_header header; - u16 nr_configured; - u16 offset_configured; - u16 nr_standby; - u16 offset_standby; - u8 reserved[4096 - 16]; -} __attribute__((packed, aligned(PAGE_SIZE))); - -static void sclp_fill_core_info(struct sclp_core_info *info, - struct read_cpu_info_sccb *sccb) -{ - char *page = (char *) sccb; - - memset(info, 0, sizeof(*info)); - info->configured = sccb->nr_configured; - info->standby = sccb->nr_standby; - info->combined = sccb->nr_configured + sccb->nr_standby; - memcpy(&info->core, page + sccb->offset_configured, - info->combined * sizeof(struct sclp_core_entry)); -} - -int sclp_get_core_info(struct sclp_core_info *info) +int _sclp_get_core_info(struct sclp_core_info *info) { int rc; struct read_cpu_info_sccb *sccb; diff --git a/drivers/s390/char/sclp_early.c b/drivers/s390/char/sclp_early.c index c71df0c..f8e46c2 100644 --- a/drivers/s390/char/sclp_early.c +++ b/drivers/s390/char/sclp_early.c @@ -221,6 +221,36 @@ static int __init sclp_set_event_mask(struct init_sccb *sccb, return sclp_cmd_early(SCLP_CMDW_WRITE_EVENT_MASK, sccb); } +static struct sclp_core_info sclp_core_info_early __initdata; +static int sclp_core_info_early_valid __initdata; + +static void __init sclp_init_core_info_early(struct read_cpu_info_sccb *sccb) +{ + int rc; + + if (!SCLP_HAS_CPU_INFO) + return; + memset(sccb, 0, sizeof(*sccb)); + sccb->header.length = sizeof(*sccb); + do { + rc = sclp_cmd_sync_early(SCLP_CMDW_READ_CPU_INFO, sccb); + } while (rc == -EBUSY); + if (rc) + return; + if (sccb->header.response_code != 0x0010) + return; + sclp_fill_core_info(&sclp_core_info_early, sccb); + sclp_core_info_early_valid = 1; +} + +int __init _sclp_get_core_info_early(struct sclp_core_info *info) +{ + if (!sclp_core_info_early_valid) + return -EIO; + *info = sclp_core_info_early; + return 0; +} + static long __init sclp_hsa_size_init(struct sdias_sccb *sccb) { sccb_init_eq_size(sccb); @@ -293,6 +323,7 @@ void __init sclp_early_detect(void) void *sccb = &sccb_early; sclp_facilities_detect(sccb); + sclp_init_core_info_early(sccb); sclp_hsa_size_detect(sccb); /* Turn off SCLP event notifications. Also save remote masks in the diff --git a/drivers/s390/char/sclp_quiesce.c b/drivers/s390/char/sclp_quiesce.c index 475e470..e495851 100644 --- a/drivers/s390/char/sclp_quiesce.c +++ b/drivers/s390/char/sclp_quiesce.c @@ -6,7 +6,6 @@ * Peter Oberparleiter <peter.oberparleiter@de.ibm.com> */ -#include <linux/module.h> #include <linux/types.h> #include <linux/cpumask.h> #include <linux/smp.h> @@ -80,5 +79,4 @@ static int __init sclp_quiesce_init(void) { return sclp_register(&sclp_quiesce_event); } - -module_init(sclp_quiesce_init); +device_initcall(sclp_quiesce_init); diff --git a/drivers/s390/char/sclp_tty.c b/drivers/s390/char/sclp_tty.c index 3c6e174..9259017 100644 --- a/drivers/s390/char/sclp_tty.c +++ b/drivers/s390/char/sclp_tty.c @@ -7,7 +7,6 @@ * Martin Schwidefsky <schwidefsky@de.ibm.com> */ -#include <linux/module.h> #include <linux/kmod.h> #include <linux/tty.h> #include <linux/tty_driver.h> @@ -573,4 +572,4 @@ sclp_tty_init(void) sclp_tty_driver = driver; return 0; } -module_init(sclp_tty_init); +device_initcall(sclp_tty_init); diff --git a/drivers/s390/char/vmlogrdr.c b/drivers/s390/char/vmlogrdr.c index e883063..3167e85 100644 --- a/drivers/s390/char/vmlogrdr.c +++ b/drivers/s390/char/vmlogrdr.c @@ -870,7 +870,7 @@ static int __init vmlogrdr_init(void) goto cleanup; for (i=0; i < MAXMINOR; ++i ) { - sys_ser[i].buffer = (char *) get_zeroed_page(GFP_KERNEL); + sys_ser[i].buffer = (char *) get_zeroed_page(GFP_KERNEL | GFP_DMA); if (!sys_ser[i].buffer) { rc = -ENOMEM; break; diff --git a/drivers/s390/char/zcore.c b/drivers/s390/char/zcore.c index 16992e2..f771e5e 100644 --- a/drivers/s390/char/zcore.c +++ b/drivers/s390/char/zcore.c @@ -7,6 +7,7 @@ * * Copyright IBM Corp. 2003, 2008 * Author(s): Michael Holzheu + * License: GPL */ #define KMSG_COMPONENT "zdump" @@ -16,7 +17,6 @@ #include <linux/slab.h> #include <linux/miscdevice.h> #include <linux/debugfs.h> -#include <linux/module.h> #include <linux/memblock.h> #include <asm/asm-offsets.h> @@ -320,7 +320,7 @@ static int __init zcore_init(void) goto fail; } - pr_alert("DETECTED 'S390X (64 bit) OS'\n"); + pr_alert("The dump process started for a 64-bit operating system\n"); rc = init_cpu_info(); if (rc) goto fail; @@ -364,22 +364,4 @@ fail: diag308(DIAG308_REL_HSA, NULL); return rc; } - -static void __exit zcore_exit(void) -{ - debug_unregister(zcore_dbf); - sclp_sdias_exit(); - free_page((unsigned long) ipl_block); - debugfs_remove(zcore_hsa_file); - debugfs_remove(zcore_reipl_file); - debugfs_remove(zcore_memmap_file); - debugfs_remove(zcore_dir); - diag308(DIAG308_REL_HSA, NULL); -} - -MODULE_AUTHOR("Copyright IBM Corp. 2003,2008"); -MODULE_DESCRIPTION("zcore module for zfcpdump support"); -MODULE_LICENSE("GPL"); - subsys_initcall(zcore_init); -module_exit(zcore_exit); diff --git a/drivers/s390/cio/cmf.c b/drivers/s390/cio/cmf.c index 268aa23..6b6386e 100644 --- a/drivers/s390/cio/cmf.c +++ b/drivers/s390/cio/cmf.c @@ -30,7 +30,7 @@ #include <linux/device.h> #include <linux/init.h> #include <linux/list.h> -#include <linux/module.h> +#include <linux/export.h> #include <linux/moduleparam.h> #include <linux/slab.h> #include <linux/timex.h> /* get_tod_clock() */ @@ -1389,13 +1389,7 @@ static int __init init_cmf(void) "%s (mode %s)\n", format_string, detect_string); return 0; } -module_init(init_cmf); - - -MODULE_AUTHOR("Arnd Bergmann <arndb@de.ibm.com>"); -MODULE_LICENSE("GPL"); -MODULE_DESCRIPTION("channel measurement facility base driver\n" - "Copyright IBM Corp. 2003\n"); +device_initcall(init_cmf); EXPORT_SYMBOL_GPL(enable_cmf); EXPORT_SYMBOL_GPL(disable_cmf); diff --git a/drivers/s390/cio/css.c b/drivers/s390/cio/css.c index 3d2b20e..bc099b6 100644 --- a/drivers/s390/cio/css.c +++ b/drivers/s390/cio/css.c @@ -5,12 +5,14 @@ * * Author(s): Arnd Bergmann (arndb@de.ibm.com) * Cornelia Huck (cornelia.huck@de.ibm.com) + * + * License: GPL */ #define KMSG_COMPONENT "cio" #define pr_fmt(fmt) KMSG_COMPONENT ": " fmt -#include <linux/module.h> +#include <linux/export.h> #include <linux/init.h> #include <linux/device.h> #include <linux/slab.h> @@ -1285,5 +1287,3 @@ void css_driver_unregister(struct css_driver *cdrv) driver_unregister(&cdrv->drv); } EXPORT_SYMBOL_GPL(css_driver_unregister); - -MODULE_LICENSE("GPL"); diff --git a/drivers/s390/cio/device.c b/drivers/s390/cio/device.c index 6a58bc8..79823ee 100644 --- a/drivers/s390/cio/device.c +++ b/drivers/s390/cio/device.c @@ -5,12 +5,14 @@ * Author(s): Arnd Bergmann (arndb@de.ibm.com) * Cornelia Huck (cornelia.huck@de.ibm.com) * Martin Schwidefsky (schwidefsky@de.ibm.com) + * + * License: GPL */ #define KMSG_COMPONENT "cio" #define pr_fmt(fmt) KMSG_COMPONENT ": " fmt -#include <linux/module.h> +#include <linux/export.h> #include <linux/init.h> #include <linux/spinlock.h> #include <linux/errno.h> @@ -145,7 +147,6 @@ static struct css_device_id io_subchannel_ids[] = { { .match_flags = 0x1, .type = SUBCHANNEL_TYPE_IO, }, { /* end of list */ }, }; -MODULE_DEVICE_TABLE(css, io_subchannel_ids); static int io_subchannel_prepare(struct subchannel *sch) { @@ -2150,7 +2151,6 @@ int ccw_device_siosl(struct ccw_device *cdev) } EXPORT_SYMBOL_GPL(ccw_device_siosl); -MODULE_LICENSE("GPL"); EXPORT_SYMBOL(ccw_device_set_online); EXPORT_SYMBOL(ccw_device_set_offline); EXPORT_SYMBOL(ccw_driver_register); diff --git a/drivers/s390/cio/device.h b/drivers/s390/cio/device.h index 065b1be..ec497af 100644 --- a/drivers/s390/cio/device.h +++ b/drivers/s390/cio/device.h @@ -13,7 +13,6 @@ */ enum dev_state { DEV_STATE_NOT_OPER, - DEV_STATE_SENSE_PGID, DEV_STATE_SENSE_ID, DEV_STATE_OFFLINE, DEV_STATE_VERIFY, diff --git a/drivers/s390/cio/device_fsm.c b/drivers/s390/cio/device_fsm.c index 8327d47..9afb5ce 100644 --- a/drivers/s390/cio/device_fsm.c +++ b/drivers/s390/cio/device_fsm.c @@ -1058,12 +1058,6 @@ fsm_func_t *dev_jumptable[NR_DEV_STATES][NR_DEV_EVENTS] = { [DEV_EVENT_TIMEOUT] = ccw_device_nop, [DEV_EVENT_VERIFY] = ccw_device_nop, }, - [DEV_STATE_SENSE_PGID] = { - [DEV_EVENT_NOTOPER] = ccw_device_request_event, - [DEV_EVENT_INTERRUPT] = ccw_device_request_event, - [DEV_EVENT_TIMEOUT] = ccw_device_request_event, - [DEV_EVENT_VERIFY] = ccw_device_nop, - }, [DEV_STATE_SENSE_ID] = { [DEV_EVENT_NOTOPER] = ccw_device_request_event, [DEV_EVENT_INTERRUPT] = ccw_device_request_event, diff --git a/drivers/s390/cio/device_ops.c b/drivers/s390/cio/device_ops.c index 877d9f6..cf8c4ac 100644 --- a/drivers/s390/cio/device_ops.c +++ b/drivers/s390/cio/device_ops.c @@ -3,8 +3,10 @@ * * Author(s): Martin Schwidefsky (schwidefsky@de.ibm.com) * Cornelia Huck (cornelia.huck@de.ibm.com) + * + * License: GPL */ -#include <linux/module.h> +#include <linux/export.h> #include <linux/init.h> #include <linux/errno.h> #include <linux/slab.h> @@ -676,7 +678,6 @@ void ccw_device_get_schid(struct ccw_device *cdev, struct subchannel_id *schid) } EXPORT_SYMBOL_GPL(ccw_device_get_schid); -MODULE_LICENSE("GPL"); EXPORT_SYMBOL(ccw_device_set_options_mask); EXPORT_SYMBOL(ccw_device_set_options); EXPORT_SYMBOL(ccw_device_clear_options); diff --git a/drivers/s390/crypto/ap_bus.c b/drivers/s390/crypto/ap_bus.c index ed92fb0..f407b4f 100644 --- a/drivers/s390/crypto/ap_bus.c +++ b/drivers/s390/crypto/ap_bus.c @@ -1273,7 +1273,7 @@ static int ap_uevent (struct device *dev, struct kobj_uevent_env *env) return retval; } -static int ap_dev_suspend(struct device *dev, pm_message_t state) +static int ap_dev_suspend(struct device *dev) { struct ap_device *ap_dev = to_ap_dev(dev); @@ -1287,11 +1287,6 @@ static int ap_dev_suspend(struct device *dev, pm_message_t state) return 0; } -static int ap_dev_resume(struct device *dev) -{ - return 0; -} - static void ap_bus_suspend(void) { ap_suspend_flag = 1; @@ -1356,12 +1351,13 @@ static struct notifier_block ap_power_notifier = { .notifier_call = ap_power_event, }; +static SIMPLE_DEV_PM_OPS(ap_bus_pm_ops, ap_dev_suspend, NULL); + static struct bus_type ap_bus_type = { .name = "ap", .match = &ap_bus_match, .uevent = &ap_uevent, - .suspend = ap_dev_suspend, - .resume = ap_dev_resume, + .pm = &ap_bus_pm_ops, }; void ap_device_init_reply(struct ap_device *ap_dev, diff --git a/drivers/s390/scsi/zfcp_ext.h b/drivers/s390/scsi/zfcp_ext.h index c8fed9f..968a0ab 100644 --- a/drivers/s390/scsi/zfcp_ext.h +++ b/drivers/s390/scsi/zfcp_ext.h @@ -84,8 +84,8 @@ extern void zfcp_fc_link_test_work(struct work_struct *); extern void zfcp_fc_wka_ports_force_offline(struct zfcp_fc_wka_ports *); extern int zfcp_fc_gs_setup(struct zfcp_adapter *); extern void zfcp_fc_gs_destroy(struct zfcp_adapter *); -extern int zfcp_fc_exec_bsg_job(struct fc_bsg_job *); -extern int zfcp_fc_timeout_bsg_job(struct fc_bsg_job *); +extern int zfcp_fc_exec_bsg_job(struct bsg_job *); +extern int zfcp_fc_timeout_bsg_job(struct bsg_job *); extern void zfcp_fc_sym_name_update(struct work_struct *); extern unsigned int zfcp_fc_port_scan_backoff(void); extern void zfcp_fc_conditional_port_scan(struct zfcp_adapter *); diff --git a/drivers/s390/scsi/zfcp_fc.c b/drivers/s390/scsi/zfcp_fc.c index 237688a..7331eea 100644 --- a/drivers/s390/scsi/zfcp_fc.c +++ b/drivers/s390/scsi/zfcp_fc.c @@ -13,6 +13,7 @@ #include <linux/slab.h> #include <linux/utsname.h> #include <linux/random.h> +#include <linux/bsg-lib.h> #include <scsi/fc/fc_els.h> #include <scsi/libfc.h> #include "zfcp_ext.h" @@ -885,26 +886,30 @@ out_free: static void zfcp_fc_ct_els_job_handler(void *data) { - struct fc_bsg_job *job = data; + struct bsg_job *job = data; struct zfcp_fsf_ct_els *zfcp_ct_els = job->dd_data; struct fc_bsg_reply *jr = job->reply; jr->reply_payload_rcv_len = job->reply_payload.payload_len; jr->reply_data.ctels_reply.status = FC_CTELS_STATUS_OK; jr->result = zfcp_ct_els->status ? -EIO : 0; - job->job_done(job); + bsg_job_done(job, jr->result, jr->reply_payload_rcv_len); } -static struct zfcp_fc_wka_port *zfcp_fc_job_wka_port(struct fc_bsg_job *job) +static struct zfcp_fc_wka_port *zfcp_fc_job_wka_port(struct bsg_job *job) { u32 preamble_word1; u8 gs_type; struct zfcp_adapter *adapter; + struct fc_bsg_request *bsg_request = job->request; + struct fc_rport *rport = fc_bsg_to_rport(job); + struct Scsi_Host *shost; - preamble_word1 = job->request->rqst_data.r_ct.preamble_word1; + preamble_word1 = bsg_request->rqst_data.r_ct.preamble_word1; gs_type = (preamble_word1 & 0xff000000) >> 24; - adapter = (struct zfcp_adapter *) job->shost->hostdata[0]; + shost = rport ? rport_to_shost(rport) : fc_bsg_to_shost(job); + adapter = (struct zfcp_adapter *) shost->hostdata[0]; switch (gs_type) { case FC_FST_ALIAS: @@ -924,7 +929,7 @@ static struct zfcp_fc_wka_port *zfcp_fc_job_wka_port(struct fc_bsg_job *job) static void zfcp_fc_ct_job_handler(void *data) { - struct fc_bsg_job *job = data; + struct bsg_job *job = data; struct zfcp_fc_wka_port *wka_port; wka_port = zfcp_fc_job_wka_port(job); @@ -933,11 +938,12 @@ static void zfcp_fc_ct_job_handler(void *data) zfcp_fc_ct_els_job_handler(data); } -static int zfcp_fc_exec_els_job(struct fc_bsg_job *job, +static int zfcp_fc_exec_els_job(struct bsg_job *job, struct zfcp_adapter *adapter) { struct zfcp_fsf_ct_els *els = job->dd_data; - struct fc_rport *rport = job->rport; + struct fc_rport *rport = fc_bsg_to_rport(job); + struct fc_bsg_request *bsg_request = job->request; struct zfcp_port *port; u32 d_id; @@ -949,13 +955,13 @@ static int zfcp_fc_exec_els_job(struct fc_bsg_job *job, d_id = port->d_id; put_device(&port->dev); } else - d_id = ntoh24(job->request->rqst_data.h_els.port_id); + d_id = ntoh24(bsg_request->rqst_data.h_els.port_id); els->handler = zfcp_fc_ct_els_job_handler; return zfcp_fsf_send_els(adapter, d_id, els, job->req->timeout / HZ); } -static int zfcp_fc_exec_ct_job(struct fc_bsg_job *job, +static int zfcp_fc_exec_ct_job(struct bsg_job *job, struct zfcp_adapter *adapter) { int ret; @@ -978,13 +984,15 @@ static int zfcp_fc_exec_ct_job(struct fc_bsg_job *job, return ret; } -int zfcp_fc_exec_bsg_job(struct fc_bsg_job *job) +int zfcp_fc_exec_bsg_job(struct bsg_job *job) { struct Scsi_Host *shost; struct zfcp_adapter *adapter; struct zfcp_fsf_ct_els *ct_els = job->dd_data; + struct fc_bsg_request *bsg_request = job->request; + struct fc_rport *rport = fc_bsg_to_rport(job); - shost = job->rport ? rport_to_shost(job->rport) : job->shost; + shost = rport ? rport_to_shost(rport) : fc_bsg_to_shost(job); adapter = (struct zfcp_adapter *)shost->hostdata[0]; if (!(atomic_read(&adapter->status) & ZFCP_STATUS_COMMON_OPEN)) @@ -994,7 +1002,7 @@ int zfcp_fc_exec_bsg_job(struct fc_bsg_job *job) ct_els->resp = job->reply_payload.sg_list; ct_els->handler_data = job; - switch (job->request->msgcode) { + switch (bsg_request->msgcode) { case FC_BSG_RPT_ELS: case FC_BSG_HST_ELS_NOLOGIN: return zfcp_fc_exec_els_job(job, adapter); @@ -1006,7 +1014,7 @@ int zfcp_fc_exec_bsg_job(struct fc_bsg_job *job) } } -int zfcp_fc_timeout_bsg_job(struct fc_bsg_job *job) +int zfcp_fc_timeout_bsg_job(struct bsg_job *job) { /* hardware tracks timeout, reset bsg timeout to not interfere */ return -EAGAIN; diff --git a/drivers/s390/virtio/virtio_ccw.c b/drivers/s390/virtio/virtio_ccw.c index 8688ad4..639ed4e 100644 --- a/drivers/s390/virtio/virtio_ccw.c +++ b/drivers/s390/virtio/virtio_ccw.c @@ -24,7 +24,7 @@ #include <linux/wait.h> #include <linux/list.h> #include <linux/bitops.h> -#include <linux/module.h> +#include <linux/moduleparam.h> #include <linux/io.h> #include <linux/kvm_para.h> #include <linux/notifier.h> @@ -235,16 +235,6 @@ static struct airq_info *new_airq_info(void) return info; } -static void destroy_airq_info(struct airq_info *info) -{ - if (!info) - return; - - unregister_adapter_interrupt(&info->airq); - airq_iv_release(info->aiv); - kfree(info); -} - static unsigned long get_airq_indicator(struct virtqueue *vqs[], int nvqs, u64 *first, void **airq_info) { @@ -1294,7 +1284,6 @@ static struct ccw_device_id virtio_ids[] = { { CCW_DEVICE(0x3832, 0) }, {}, }; -MODULE_DEVICE_TABLE(ccw, virtio_ids); static struct ccw_driver virtio_ccw_driver = { .driver = { @@ -1406,14 +1395,4 @@ static int __init virtio_ccw_init(void) no_auto_parse(); return ccw_driver_register(&virtio_ccw_driver); } -module_init(virtio_ccw_init); - -static void __exit virtio_ccw_exit(void) -{ - int i; - - ccw_driver_unregister(&virtio_ccw_driver); - for (i = 0; i < MAX_AIRQ_AREAS; i++) - destroy_airq_info(airq_areas[i]); -} -module_exit(virtio_ccw_exit); +device_initcall(virtio_ccw_init); diff --git a/drivers/scsi/Kconfig b/drivers/scsi/Kconfig index 3e2bdb9..dfa9334 100644 --- a/drivers/scsi/Kconfig +++ b/drivers/scsi/Kconfig @@ -263,6 +263,7 @@ config SCSI_SPI_ATTRS config SCSI_FC_ATTRS tristate "FiberChannel Transport Attributes" depends on SCSI && NET + select BLK_DEV_BSGLIB select SCSI_NETLINK help If you wish to export transport-specific information about @@ -743,40 +744,18 @@ config SCSI_ISCI control unit found in the Intel(R) C600 series chipset. config SCSI_GENERIC_NCR5380 - tristate "Generic NCR5380/53c400 SCSI PIO support" - depends on ISA && SCSI + tristate "Generic NCR5380/53c400 SCSI ISA card support" + depends on ISA && SCSI && HAS_IOPORT_MAP select SCSI_SPI_ATTRS ---help--- - This is a driver for the old NCR 53c80 series of SCSI controllers - on boards using PIO. Most boards such as the Trantor T130 fit this - category, along with a large number of ISA 8bit controllers shipped - for free with SCSI scanners. If you have a PAS16, T128 or DMX3191 - you should select the specific driver for that card rather than - generic 5380 support. - - It is explained in section 3.8 of the SCSI-HOWTO, available from - <http://www.tldp.org/docs.html#howto>. If it doesn't work out - of the box, you may have to change some settings in - <file:drivers/scsi/g_NCR5380.h>. + This is a driver for old ISA card SCSI controllers based on a + NCR 5380, 53C80, 53C400, 53C400A, or DTC 436 device. + Most boards such as the Trantor T130 fit this category, as do + various 8-bit and 16-bit ISA cards bundled with SCSI scanners. To compile this driver as a module, choose M here: the module will be called g_NCR5380. -config SCSI_GENERIC_NCR5380_MMIO - tristate "Generic NCR5380/53c400 SCSI MMIO support" - depends on ISA && SCSI - select SCSI_SPI_ATTRS - ---help--- - This is a driver for the old NCR 53c80 series of SCSI controllers - on boards using memory mapped I/O. - It is explained in section 3.8 of the SCSI-HOWTO, available from - <http://www.tldp.org/docs.html#howto>. If it doesn't work out - of the box, you may have to change some settings in - <file:drivers/scsi/g_NCR5380.h>. - - To compile this driver as a module, choose M here: the - module will be called g_NCR5380_mmio. - config SCSI_IPS tristate "IBM ServeRAID support" depends on PCI && SCSI diff --git a/drivers/scsi/Makefile b/drivers/scsi/Makefile index 1520596..a2d0395 100644 --- a/drivers/scsi/Makefile +++ b/drivers/scsi/Makefile @@ -74,7 +74,6 @@ obj-$(CONFIG_SCSI_ISCI) += isci/ obj-$(CONFIG_SCSI_IPS) += ips.o obj-$(CONFIG_SCSI_FUTURE_DOMAIN)+= fdomain.o obj-$(CONFIG_SCSI_GENERIC_NCR5380) += g_NCR5380.o -obj-$(CONFIG_SCSI_GENERIC_NCR5380_MMIO) += g_NCR5380_mmio.o obj-$(CONFIG_SCSI_NCR53C406A) += NCR53c406a.o obj-$(CONFIG_SCSI_NCR_D700) += 53c700.o NCR_D700.o obj-$(CONFIG_SCSI_NCR_Q720) += NCR_Q720_mod.o diff --git a/drivers/scsi/NCR5380.c b/drivers/scsi/NCR5380.c index 790babc..d849ffa 100644 --- a/drivers/scsi/NCR5380.c +++ b/drivers/scsi/NCR5380.c @@ -121,9 +121,10 @@ * * Either real DMA *or* pseudo DMA may be implemented * - * NCR5380_dma_write_setup(instance, src, count) - initialize - * NCR5380_dma_read_setup(instance, dst, count) - initialize - * NCR5380_dma_residual(instance); - residual count + * NCR5380_dma_xfer_len - determine size of DMA/PDMA transfer + * NCR5380_dma_send_setup - execute DMA/PDMA from memory to 5380 + * NCR5380_dma_recv_setup - execute DMA/PDMA from 5380 to memory + * NCR5380_dma_residual - residual byte count * * The generic driver is initialized by calling NCR5380_init(instance), * after setting the appropriate host specific fields and ID. If the @@ -178,7 +179,7 @@ static inline void initialize_SCp(struct scsi_cmnd *cmd) /** * NCR5380_poll_politely2 - wait for two chip register values - * @instance: controller to poll + * @hostdata: host private data * @reg1: 5380 register to poll * @bit1: Bitmask to check * @val1: Expected value @@ -195,18 +196,14 @@ static inline void initialize_SCp(struct scsi_cmnd *cmd) * Returns 0 if either or both event(s) occurred otherwise -ETIMEDOUT. */ -static int NCR5380_poll_politely2(struct Scsi_Host *instance, - int reg1, int bit1, int val1, - int reg2, int bit2, int val2, int wait) +static int NCR5380_poll_politely2(struct NCR5380_hostdata *hostdata, + unsigned int reg1, u8 bit1, u8 val1, + unsigned int reg2, u8 bit2, u8 val2, + unsigned long wait) { - struct NCR5380_hostdata *hostdata = shost_priv(instance); + unsigned long n = hostdata->poll_loops; unsigned long deadline = jiffies + wait; - unsigned long n; - /* Busy-wait for up to 10 ms */ - n = min(10000U, jiffies_to_usecs(wait)); - n *= hostdata->accesses_per_ms; - n /= 2000; do { if ((NCR5380_read(reg1) & bit1) == val1) return 0; @@ -288,6 +285,7 @@ mrs[] = { static void NCR5380_print(struct Scsi_Host *instance) { + struct NCR5380_hostdata *hostdata = shost_priv(instance); unsigned char status, data, basr, mr, icr, i; data = NCR5380_read(CURRENT_SCSI_DATA_REG); @@ -337,6 +335,7 @@ static struct { static void NCR5380_print_phase(struct Scsi_Host *instance) { + struct NCR5380_hostdata *hostdata = shost_priv(instance); unsigned char status; int i; @@ -441,14 +440,14 @@ static void prepare_info(struct Scsi_Host *instance) struct NCR5380_hostdata *hostdata = shost_priv(instance); snprintf(hostdata->info, sizeof(hostdata->info), - "%s, io_port 0x%lx, n_io_port %d, " - "base 0x%lx, irq %d, " + "%s, irq %d, " + "io_port 0x%lx, base 0x%lx, " "can_queue %d, cmd_per_lun %d, " "sg_tablesize %d, this_id %d, " "flags { %s%s%s}, " "options { %s} ", - instance->hostt->name, instance->io_port, instance->n_io_port, - instance->base, instance->irq, + instance->hostt->name, instance->irq, + hostdata->io_port, hostdata->base, instance->can_queue, instance->cmd_per_lun, instance->sg_tablesize, instance->this_id, hostdata->flags & FLAG_DMA_FIXUP ? "DMA_FIXUP " : "", @@ -482,6 +481,7 @@ static int NCR5380_init(struct Scsi_Host *instance, int flags) struct NCR5380_hostdata *hostdata = shost_priv(instance); int i; unsigned long deadline; + unsigned long accesses_per_ms; instance->max_lun = 7; @@ -530,7 +530,8 @@ static int NCR5380_init(struct Scsi_Host *instance, int flags) ++i; cpu_relax(); } while (time_is_after_jiffies(deadline)); - hostdata->accesses_per_ms = i / 256; + accesses_per_ms = i / 256; + hostdata->poll_loops = NCR5380_REG_POLL_TIME * accesses_per_ms / 2; return 0; } @@ -560,7 +561,7 @@ static int NCR5380_maybe_reset_bus(struct Scsi_Host *instance) case 3: case 5: shost_printk(KERN_ERR, instance, "SCSI bus busy, waiting up to five seconds\n"); - NCR5380_poll_politely(instance, + NCR5380_poll_politely(hostdata, STATUS_REG, SR_BSY, 0, 5 * HZ); break; case 2: @@ -871,7 +872,7 @@ static void NCR5380_dma_complete(struct Scsi_Host *instance) NCR5380_write(INITIATOR_COMMAND_REG, ICR_BASE); NCR5380_read(RESET_PARITY_INTERRUPT_REG); - transferred = hostdata->dma_len - NCR5380_dma_residual(instance); + transferred = hostdata->dma_len - NCR5380_dma_residual(hostdata); hostdata->dma_len = 0; data = (unsigned char **)&hostdata->connected->SCp.ptr; @@ -994,7 +995,7 @@ static irqreturn_t __maybe_unused NCR5380_intr(int irq, void *dev_id) } handled = 1; } else { - shost_printk(KERN_NOTICE, instance, "interrupt without IRQ bit\n"); + dsprintk(NDEBUG_INTR, instance, "interrupt without IRQ bit\n"); #ifdef SUN3_SCSI_VME dregs->csr |= CSR_DMA_ENABLE; #endif @@ -1075,7 +1076,7 @@ static struct scsi_cmnd *NCR5380_select(struct Scsi_Host *instance, */ spin_unlock_irq(&hostdata->lock); - err = NCR5380_poll_politely2(instance, MODE_REG, MR_ARBITRATE, 0, + err = NCR5380_poll_politely2(hostdata, MODE_REG, MR_ARBITRATE, 0, INITIATOR_COMMAND_REG, ICR_ARBITRATION_PROGRESS, ICR_ARBITRATION_PROGRESS, HZ); spin_lock_irq(&hostdata->lock); @@ -1201,7 +1202,7 @@ static struct scsi_cmnd *NCR5380_select(struct Scsi_Host *instance, * selection. */ - err = NCR5380_poll_politely(instance, STATUS_REG, SR_BSY, SR_BSY, + err = NCR5380_poll_politely(hostdata, STATUS_REG, SR_BSY, SR_BSY, msecs_to_jiffies(250)); if ((NCR5380_read(STATUS_REG) & (SR_SEL | SR_IO)) == (SR_SEL | SR_IO)) { @@ -1247,7 +1248,7 @@ static struct scsi_cmnd *NCR5380_select(struct Scsi_Host *instance, /* Wait for start of REQ/ACK handshake */ - err = NCR5380_poll_politely(instance, STATUS_REG, SR_REQ, SR_REQ, HZ); + err = NCR5380_poll_politely(hostdata, STATUS_REG, SR_REQ, SR_REQ, HZ); spin_lock_irq(&hostdata->lock); if (err < 0) { shost_printk(KERN_ERR, instance, "select: REQ timeout\n"); @@ -1318,6 +1319,7 @@ static int NCR5380_transfer_pio(struct Scsi_Host *instance, unsigned char *phase, int *count, unsigned char **data) { + struct NCR5380_hostdata *hostdata = shost_priv(instance); unsigned char p = *phase, tmp; int c = *count; unsigned char *d = *data; @@ -1336,7 +1338,7 @@ static int NCR5380_transfer_pio(struct Scsi_Host *instance, * valid */ - if (NCR5380_poll_politely(instance, STATUS_REG, SR_REQ, SR_REQ, HZ) < 0) + if (NCR5380_poll_politely(hostdata, STATUS_REG, SR_REQ, SR_REQ, HZ) < 0) break; dsprintk(NDEBUG_HANDSHAKE, instance, "REQ asserted\n"); @@ -1381,7 +1383,7 @@ static int NCR5380_transfer_pio(struct Scsi_Host *instance, NCR5380_write(INITIATOR_COMMAND_REG, ICR_BASE | ICR_ASSERT_ACK); } - if (NCR5380_poll_politely(instance, + if (NCR5380_poll_politely(hostdata, STATUS_REG, SR_REQ, 0, 5 * HZ) < 0) break; @@ -1440,6 +1442,7 @@ static int NCR5380_transfer_pio(struct Scsi_Host *instance, static void do_reset(struct Scsi_Host *instance) { + struct NCR5380_hostdata __maybe_unused *hostdata = shost_priv(instance); unsigned long flags; local_irq_save(flags); @@ -1462,6 +1465,7 @@ static void do_reset(struct Scsi_Host *instance) static int do_abort(struct Scsi_Host *instance) { + struct NCR5380_hostdata *hostdata = shost_priv(instance); unsigned char *msgptr, phase, tmp; int len; int rc; @@ -1479,7 +1483,7 @@ static int do_abort(struct Scsi_Host *instance) * the target sees, so we just handshake. */ - rc = NCR5380_poll_politely(instance, STATUS_REG, SR_REQ, SR_REQ, 10 * HZ); + rc = NCR5380_poll_politely(hostdata, STATUS_REG, SR_REQ, SR_REQ, 10 * HZ); if (rc < 0) goto timeout; @@ -1490,7 +1494,7 @@ static int do_abort(struct Scsi_Host *instance) if (tmp != PHASE_MSGOUT) { NCR5380_write(INITIATOR_COMMAND_REG, ICR_BASE | ICR_ASSERT_ATN | ICR_ASSERT_ACK); - rc = NCR5380_poll_politely(instance, STATUS_REG, SR_REQ, 0, 3 * HZ); + rc = NCR5380_poll_politely(hostdata, STATUS_REG, SR_REQ, 0, 3 * HZ); if (rc < 0) goto timeout; NCR5380_write(INITIATOR_COMMAND_REG, ICR_BASE | ICR_ASSERT_ATN); @@ -1575,9 +1579,9 @@ static int NCR5380_transfer_dma(struct Scsi_Host *instance, * starting the NCR. This is also the cleaner way for the TT. */ if (p & SR_IO) - result = NCR5380_dma_recv_setup(instance, d, c); + result = NCR5380_dma_recv_setup(hostdata, d, c); else - result = NCR5380_dma_send_setup(instance, d, c); + result = NCR5380_dma_send_setup(hostdata, d, c); } /* @@ -1609,9 +1613,9 @@ static int NCR5380_transfer_dma(struct Scsi_Host *instance, * NCR access, else the DMA setup gets trashed! */ if (p & SR_IO) - result = NCR5380_dma_recv_setup(instance, d, c); + result = NCR5380_dma_recv_setup(hostdata, d, c); else - result = NCR5380_dma_send_setup(instance, d, c); + result = NCR5380_dma_send_setup(hostdata, d, c); } /* On failure, NCR5380_dma_xxxx_setup() returns a negative int. */ @@ -1678,12 +1682,12 @@ static int NCR5380_transfer_dma(struct Scsi_Host *instance, * byte. */ - if (NCR5380_poll_politely(instance, BUS_AND_STATUS_REG, + if (NCR5380_poll_politely(hostdata, BUS_AND_STATUS_REG, BASR_DRQ, BASR_DRQ, HZ) < 0) { result = -1; shost_printk(KERN_ERR, instance, "PDMA read: DRQ timeout\n"); } - if (NCR5380_poll_politely(instance, STATUS_REG, + if (NCR5380_poll_politely(hostdata, STATUS_REG, SR_REQ, 0, HZ) < 0) { result = -1; shost_printk(KERN_ERR, instance, "PDMA read: !REQ timeout\n"); @@ -1694,7 +1698,7 @@ static int NCR5380_transfer_dma(struct Scsi_Host *instance, * Wait for the last byte to be sent. If REQ is being asserted for * the byte we're interested, we'll ACK it and it will go false. */ - if (NCR5380_poll_politely2(instance, + if (NCR5380_poll_politely2(hostdata, BUS_AND_STATUS_REG, BASR_DRQ, BASR_DRQ, BUS_AND_STATUS_REG, BASR_PHASE_MATCH, 0, HZ) < 0) { result = -1; @@ -1751,22 +1755,26 @@ static void NCR5380_information_transfer(struct Scsi_Host *instance) NCR5380_dprint_phase(NDEBUG_INFORMATION, instance); } #ifdef CONFIG_SUN3 - if (phase == PHASE_CMDOUT) { - void *d; - unsigned long count; + if (phase == PHASE_CMDOUT && + sun3_dma_setup_done != cmd) { + int count; if (!cmd->SCp.this_residual && cmd->SCp.buffers_residual) { - count = cmd->SCp.buffer->length; - d = sg_virt(cmd->SCp.buffer); - } else { - count = cmd->SCp.this_residual; - d = cmd->SCp.ptr; + ++cmd->SCp.buffer; + --cmd->SCp.buffers_residual; + cmd->SCp.this_residual = cmd->SCp.buffer->length; + cmd->SCp.ptr = sg_virt(cmd->SCp.buffer); } - if (sun3_dma_setup_done != cmd && - sun3scsi_dma_xfer_len(count, cmd) > 0) { - sun3scsi_dma_setup(instance, d, count, - rq_data_dir(cmd->request)); + count = sun3scsi_dma_xfer_len(hostdata, cmd); + + if (count > 0) { + if (rq_data_dir(cmd->request)) + sun3scsi_dma_send_setup(hostdata, + cmd->SCp.ptr, count); + else + sun3scsi_dma_recv_setup(hostdata, + cmd->SCp.ptr, count); sun3_dma_setup_done = cmd; } #ifdef SUN3_SCSI_VME @@ -1827,7 +1835,7 @@ static void NCR5380_information_transfer(struct Scsi_Host *instance) transfersize = 0; if (!cmd->device->borken) - transfersize = NCR5380_dma_xfer_len(instance, cmd, phase); + transfersize = NCR5380_dma_xfer_len(hostdata, cmd); if (transfersize > 0) { len = transfersize; @@ -2073,7 +2081,7 @@ static void NCR5380_information_transfer(struct Scsi_Host *instance) } /* switch(phase) */ } else { spin_unlock_irq(&hostdata->lock); - NCR5380_poll_politely(instance, STATUS_REG, SR_REQ, SR_REQ, HZ); + NCR5380_poll_politely(hostdata, STATUS_REG, SR_REQ, SR_REQ, HZ); spin_lock_irq(&hostdata->lock); } } @@ -2119,7 +2127,7 @@ static void NCR5380_reselect(struct Scsi_Host *instance) */ NCR5380_write(INITIATOR_COMMAND_REG, ICR_BASE | ICR_ASSERT_BSY); - if (NCR5380_poll_politely(instance, + if (NCR5380_poll_politely(hostdata, STATUS_REG, SR_SEL, 0, 2 * HZ) < 0) { NCR5380_write(INITIATOR_COMMAND_REG, ICR_BASE); return; @@ -2130,7 +2138,7 @@ static void NCR5380_reselect(struct Scsi_Host *instance) * Wait for target to go into MSGIN. */ - if (NCR5380_poll_politely(instance, + if (NCR5380_poll_politely(hostdata, STATUS_REG, SR_REQ, SR_REQ, 2 * HZ) < 0) { do_abort(instance); return; @@ -2204,22 +2212,25 @@ static void NCR5380_reselect(struct Scsi_Host *instance) } #ifdef CONFIG_SUN3 - { - void *d; - unsigned long count; + if (sun3_dma_setup_done != tmp) { + int count; if (!tmp->SCp.this_residual && tmp->SCp.buffers_residual) { - count = tmp->SCp.buffer->length; - d = sg_virt(tmp->SCp.buffer); - } else { - count = tmp->SCp.this_residual; - d = tmp->SCp.ptr; + ++tmp->SCp.buffer; + --tmp->SCp.buffers_residual; + tmp->SCp.this_residual = tmp->SCp.buffer->length; + tmp->SCp.ptr = sg_virt(tmp->SCp.buffer); } - if (sun3_dma_setup_done != tmp && - sun3scsi_dma_xfer_len(count, tmp) > 0) { - sun3scsi_dma_setup(instance, d, count, - rq_data_dir(tmp->request)); + count = sun3scsi_dma_xfer_len(hostdata, tmp); + + if (count > 0) { + if (rq_data_dir(tmp->request)) + sun3scsi_dma_send_setup(hostdata, + tmp->SCp.ptr, count); + else + sun3scsi_dma_recv_setup(hostdata, + tmp->SCp.ptr, count); sun3_dma_setup_done = tmp; } } diff --git a/drivers/scsi/NCR5380.h b/drivers/scsi/NCR5380.h index 965d923..3c6ce54 100644 --- a/drivers/scsi/NCR5380.h +++ b/drivers/scsi/NCR5380.h @@ -219,27 +219,32 @@ #define FLAG_TOSHIBA_DELAY 128 /* Allow for borken CD-ROMs */ struct NCR5380_hostdata { - NCR5380_implementation_fields; /* implementation specific */ - struct Scsi_Host *host; /* Host backpointer */ - unsigned char id_mask, id_higher_mask; /* 1 << id, all bits greater */ - unsigned char busy[8]; /* index = target, bit = lun */ - int dma_len; /* requested length of DMA */ - unsigned char last_message; /* last message OUT */ - struct scsi_cmnd *connected; /* currently connected cmnd */ - struct scsi_cmnd *selecting; /* cmnd to be connected */ - struct list_head unissued; /* waiting to be issued */ - struct list_head autosense; /* priority issue queue */ - struct list_head disconnected; /* waiting for reconnect */ - spinlock_t lock; /* protects this struct */ - int flags; - struct scsi_eh_save ses; - struct scsi_cmnd *sensing; + NCR5380_implementation_fields; /* Board-specific data */ + u8 __iomem *io; /* Remapped 5380 address */ + u8 __iomem *pdma_io; /* Remapped PDMA address */ + unsigned long poll_loops; /* Register polling limit */ + spinlock_t lock; /* Protects this struct */ + struct scsi_cmnd *connected; /* Currently connected cmnd */ + struct list_head disconnected; /* Waiting for reconnect */ + struct Scsi_Host *host; /* SCSI host backpointer */ + struct workqueue_struct *work_q; /* SCSI host work queue */ + struct work_struct main_task; /* Work item for main loop */ + int flags; /* Board-specific quirks */ + int dma_len; /* Requested length of DMA */ + int read_overruns; /* Transfer size reduction for DMA erratum */ + unsigned long io_port; /* Device IO port */ + unsigned long base; /* Device base address */ + struct list_head unissued; /* Waiting to be issued */ + struct scsi_cmnd *selecting; /* Cmnd to be connected */ + struct list_head autosense; /* Priority cmnd queue */ + struct scsi_cmnd *sensing; /* Cmnd needing autosense */ + struct scsi_eh_save ses; /* Cmnd state saved for EH */ + unsigned char busy[8]; /* Index = target, bit = lun */ + unsigned char id_mask; /* 1 << Host ID */ + unsigned char id_higher_mask; /* All bits above id_mask */ + unsigned char last_message; /* Last Message Out */ + unsigned long region_size; /* Size of address/port range */ char info[256]; - int read_overruns; /* number of bytes to cut from a - * transfer to handle chip overruns */ - struct work_struct main_task; - struct workqueue_struct *work_q; - unsigned long accesses_per_ms; /* chip register accesses per ms */ }; #ifdef __KERNEL__ @@ -252,6 +257,9 @@ struct NCR5380_cmd { #define NCR5380_PIO_CHUNK_SIZE 256 +/* Time limit (ms) to poll registers when IRQs are disabled, e.g. during PDMA */ +#define NCR5380_REG_POLL_TIME 15 + static inline struct scsi_cmnd *NCR5380_to_scmd(struct NCR5380_cmd *ncmd_ptr) { return ((struct scsi_cmnd *)ncmd_ptr) - 1; @@ -294,14 +302,45 @@ static void NCR5380_reselect(struct Scsi_Host *instance); static struct scsi_cmnd *NCR5380_select(struct Scsi_Host *, struct scsi_cmnd *); static int NCR5380_transfer_dma(struct Scsi_Host *instance, unsigned char *phase, int *count, unsigned char **data); static int NCR5380_transfer_pio(struct Scsi_Host *instance, unsigned char *phase, int *count, unsigned char **data); -static int NCR5380_poll_politely2(struct Scsi_Host *, int, int, int, int, int, int, int); +static int NCR5380_poll_politely2(struct NCR5380_hostdata *, + unsigned int, u8, u8, + unsigned int, u8, u8, unsigned long); -static inline int NCR5380_poll_politely(struct Scsi_Host *instance, - int reg, int bit, int val, int wait) +static inline int NCR5380_poll_politely(struct NCR5380_hostdata *hostdata, + unsigned int reg, u8 bit, u8 val, + unsigned long wait) { - return NCR5380_poll_politely2(instance, reg, bit, val, + if ((NCR5380_read(reg) & bit) == val) + return 0; + + return NCR5380_poll_politely2(hostdata, reg, bit, val, reg, bit, val, wait); } +static int NCR5380_dma_xfer_len(struct NCR5380_hostdata *, + struct scsi_cmnd *); +static int NCR5380_dma_send_setup(struct NCR5380_hostdata *, + unsigned char *, int); +static int NCR5380_dma_recv_setup(struct NCR5380_hostdata *, + unsigned char *, int); +static int NCR5380_dma_residual(struct NCR5380_hostdata *); + +static inline int NCR5380_dma_xfer_none(struct NCR5380_hostdata *hostdata, + struct scsi_cmnd *cmd) +{ + return 0; +} + +static inline int NCR5380_dma_setup_none(struct NCR5380_hostdata *hostdata, + unsigned char *data, int count) +{ + return 0; +} + +static inline int NCR5380_dma_residual_none(struct NCR5380_hostdata *hostdata) +{ + return 0; +} + #endif /* __KERNEL__ */ #endif /* NCR5380_H */ diff --git a/drivers/scsi/aacraid/aacraid.h b/drivers/scsi/aacraid/aacraid.h index 969c312..f059c14 100644 --- a/drivers/scsi/aacraid/aacraid.h +++ b/drivers/scsi/aacraid/aacraid.h @@ -1246,7 +1246,6 @@ struct aac_dev u32 max_msix; /* max. MSI-X vectors */ u32 vector_cap; /* MSI-X vector capab.*/ int msi_enabled; /* MSI/MSI-X enabled */ - struct msix_entry msixentry[AAC_MAX_MSIX]; struct aac_msix_ctx aac_msix[AAC_MAX_MSIX]; /* context */ u8 adapter_shutdown; u32 handle_pci_error; diff --git a/drivers/scsi/aacraid/comminit.c b/drivers/scsi/aacraid/comminit.c index 341ea32..4f56b10 100644 --- a/drivers/scsi/aacraid/comminit.c +++ b/drivers/scsi/aacraid/comminit.c @@ -378,16 +378,12 @@ void aac_define_int_mode(struct aac_dev *dev) if (msi_count > AAC_MAX_MSIX) msi_count = AAC_MAX_MSIX; - for (i = 0; i < msi_count; i++) - dev->msixentry[i].entry = i; - if (msi_count > 1 && pci_find_capability(dev->pdev, PCI_CAP_ID_MSIX)) { min_msix = 2; - i = pci_enable_msix_range(dev->pdev, - dev->msixentry, - min_msix, - msi_count); + i = pci_alloc_irq_vectors(dev->pdev, + min_msix, msi_count, + PCI_IRQ_MSIX | PCI_IRQ_AFFINITY); if (i > 0) { dev->msi_enabled = 1; msi_count = i; diff --git a/drivers/scsi/aacraid/commsup.c b/drivers/scsi/aacraid/commsup.c index 0aeecec..9e7551f 100644 --- a/drivers/scsi/aacraid/commsup.c +++ b/drivers/scsi/aacraid/commsup.c @@ -2043,30 +2043,22 @@ int aac_acquire_irq(struct aac_dev *dev) int i; int j; int ret = 0; - int cpu; - cpu = cpumask_first(cpu_online_mask); if (!dev->sync_mode && dev->msi_enabled && dev->max_msix > 1) { for (i = 0; i < dev->max_msix; i++) { dev->aac_msix[i].vector_no = i; dev->aac_msix[i].dev = dev; - if (request_irq(dev->msixentry[i].vector, + if (request_irq(pci_irq_vector(dev->pdev, i), dev->a_ops.adapter_intr, 0, "aacraid", &(dev->aac_msix[i]))) { printk(KERN_ERR "%s%d: Failed to register IRQ for vector %d.\n", dev->name, dev->id, i); for (j = 0 ; j < i ; j++) - free_irq(dev->msixentry[j].vector, + free_irq(pci_irq_vector(dev->pdev, j), &(dev->aac_msix[j])); pci_disable_msix(dev->pdev); ret = -1; } - if (irq_set_affinity_hint(dev->msixentry[i].vector, - get_cpu_mask(cpu))) { - printk(KERN_ERR "%s%d: Failed to set IRQ affinity for cpu %d\n", - dev->name, dev->id, cpu); - } - cpu = cpumask_next(cpu, cpu_online_mask); } } else { dev->aac_msix[0].vector_no = 0; @@ -2096,16 +2088,9 @@ void aac_free_irq(struct aac_dev *dev) dev->pdev->device == PMC_DEVICE_S8 || dev->pdev->device == PMC_DEVICE_S9) { if (dev->max_msix > 1) { - for (i = 0; i < dev->max_msix; i++) { - if (irq_set_affinity_hint( - dev->msixentry[i].vector, NULL)) { - printk(KERN_ERR "%s%d: Failed to reset IRQ affinity for cpu %d\n", - dev->name, dev->id, cpu); - } - cpu = cpumask_next(cpu, cpu_online_mask); - free_irq(dev->msixentry[i].vector, - &(dev->aac_msix[i])); - } + for (i = 0; i < dev->max_msix; i++) + free_irq(pci_irq_vector(dev->pdev, i), + &(dev->aac_msix[i])); } else { free_irq(dev->pdev->irq, &(dev->aac_msix[0])); } diff --git a/drivers/scsi/aacraid/linit.c b/drivers/scsi/aacraid/linit.c index 79871f3..e4f3e22 100644 --- a/drivers/scsi/aacraid/linit.c +++ b/drivers/scsi/aacraid/linit.c @@ -1071,7 +1071,6 @@ static struct scsi_host_template aac_driver_template = { static void __aac_shutdown(struct aac_dev * aac) { int i; - int cpu; aac_send_shutdown(aac); @@ -1087,24 +1086,13 @@ static void __aac_shutdown(struct aac_dev * aac) kthread_stop(aac->thread); } aac_adapter_disable_int(aac); - cpu = cpumask_first(cpu_online_mask); if (aac->pdev->device == PMC_DEVICE_S6 || aac->pdev->device == PMC_DEVICE_S7 || aac->pdev->device == PMC_DEVICE_S8 || aac->pdev->device == PMC_DEVICE_S9) { if (aac->max_msix > 1) { for (i = 0; i < aac->max_msix; i++) { - if (irq_set_affinity_hint( - aac->msixentry[i].vector, - NULL)) { - printk(KERN_ERR "%s%d: Failed to reset IRQ affinity for cpu %d\n", - aac->name, - aac->id, - cpu); - } - cpu = cpumask_next(cpu, - cpu_online_mask); - free_irq(aac->msixentry[i].vector, + free_irq(pci_irq_vector(aac->pdev, i), &(aac->aac_msix[i])); } } else { @@ -1350,7 +1338,7 @@ static void aac_release_resources(struct aac_dev *aac) aac->pdev->device == PMC_DEVICE_S9) { if (aac->max_msix > 1) { for (i = 0; i < aac->max_msix; i++) - free_irq(aac->msixentry[i].vector, + free_irq(pci_irq_vector(aac->pdev, i), &(aac->aac_msix[i])); } else { free_irq(aac->pdev->irq, &(aac->aac_msix[0])); @@ -1396,13 +1384,13 @@ static int aac_acquire_resources(struct aac_dev *dev) dev->aac_msix[i].vector_no = i; dev->aac_msix[i].dev = dev; - if (request_irq(dev->msixentry[i].vector, + if (request_irq(pci_irq_vector(dev->pdev, i), dev->a_ops.adapter_intr, 0, "aacraid", &(dev->aac_msix[i]))) { printk(KERN_ERR "%s%d: Failed to register IRQ for vector %d.\n", name, instance, i); for (j = 0 ; j < i ; j++) - free_irq(dev->msixentry[j].vector, + free_irq(pci_irq_vector(dev->pdev, j), &(dev->aac_msix[j])); pci_disable_msix(dev->pdev); goto error_iounmap; diff --git a/drivers/scsi/advansys.c b/drivers/scsi/advansys.c index febbd83..81dd092 100644 --- a/drivers/scsi/advansys.c +++ b/drivers/scsi/advansys.c @@ -11030,6 +11030,9 @@ static int advansys_board_found(struct Scsi_Host *shost, unsigned int iop, ASC_DBG(2, "AdvInitGetConfig()\n"); ret = AdvInitGetConfig(pdev, shost) ? -ENODEV : 0; +#else + share_irq = 0; + ret = -ENODEV; #endif /* CONFIG_PCI */ } diff --git a/drivers/scsi/aic94xx/aic94xx_hwi.c b/drivers/scsi/aic94xx/aic94xx_hwi.c index 7c713f7..f2671a8 100644 --- a/drivers/scsi/aic94xx/aic94xx_hwi.c +++ b/drivers/scsi/aic94xx/aic94xx_hwi.c @@ -228,8 +228,11 @@ static int asd_init_scbs(struct asd_ha_struct *asd_ha) bitmap_bytes = (asd_ha->seq.tc_index_bitmap_bits+7)/8; bitmap_bytes = BITS_TO_LONGS(bitmap_bytes*8)*sizeof(unsigned long); asd_ha->seq.tc_index_bitmap = kzalloc(bitmap_bytes, GFP_KERNEL); - if (!asd_ha->seq.tc_index_bitmap) + if (!asd_ha->seq.tc_index_bitmap) { + kfree(asd_ha->seq.tc_index_array); + asd_ha->seq.tc_index_array = NULL; return -ENOMEM; + } spin_lock_init(&seq->tc_index_lock); diff --git a/drivers/scsi/arcmsr/arcmsr.h b/drivers/scsi/arcmsr/arcmsr.h index cf99f8c..a254b32 100644 --- a/drivers/scsi/arcmsr/arcmsr.h +++ b/drivers/scsi/arcmsr/arcmsr.h @@ -629,7 +629,6 @@ struct AdapterControlBlock struct pci_dev * pdev; struct Scsi_Host * host; unsigned long vir2phy_offset; - struct msix_entry entries[ARCMST_NUM_MSIX_VECTORS]; /* Offset is used in making arc cdb physical to virtual calculations */ uint32_t outbound_int_enable; uint32_t cdb_phyaddr_hi32; @@ -671,8 +670,6 @@ struct AdapterControlBlock /* iop init */ #define ACB_F_ABORT 0x0200 #define ACB_F_FIRMWARE_TRAP 0x0400 - #define ACB_F_MSI_ENABLED 0x1000 - #define ACB_F_MSIX_ENABLED 0x2000 struct CommandControlBlock * pccb_pool[ARCMSR_MAX_FREECCB_NUM]; /* used for memory free */ struct list_head ccb_free_list; @@ -725,7 +722,7 @@ struct AdapterControlBlock atomic_t rq_map_token; atomic_t ante_token_value; uint32_t maxOutstanding; - int msix_vector_count; + int vector_count; };/* HW_DEVICE_EXTENSION */ /* ******************************************************************************* diff --git a/drivers/scsi/arcmsr/arcmsr_hba.c b/drivers/scsi/arcmsr/arcmsr_hba.c index f0cfb04..9e45749 100644 --- a/drivers/scsi/arcmsr/arcmsr_hba.c +++ b/drivers/scsi/arcmsr/arcmsr_hba.c @@ -720,51 +720,39 @@ static void arcmsr_message_isr_bh_fn(struct work_struct *work) static int arcmsr_request_irq(struct pci_dev *pdev, struct AdapterControlBlock *acb) { - int i, j, r; - struct msix_entry entries[ARCMST_NUM_MSIX_VECTORS]; - - for (i = 0; i < ARCMST_NUM_MSIX_VECTORS; i++) - entries[i].entry = i; - r = pci_enable_msix_range(pdev, entries, 1, ARCMST_NUM_MSIX_VECTORS); - if (r < 0) - goto msi_int; - acb->msix_vector_count = r; - for (i = 0; i < r; i++) { - if (request_irq(entries[i].vector, - arcmsr_do_interrupt, 0, "arcmsr", acb)) { + unsigned long flags; + int nvec, i; + + nvec = pci_alloc_irq_vectors(pdev, 1, ARCMST_NUM_MSIX_VECTORS, + PCI_IRQ_MSIX); + if (nvec > 0) { + pr_info("arcmsr%d: msi-x enabled\n", acb->host->host_no); + flags = 0; + } else { + nvec = pci_alloc_irq_vectors(pdev, 1, 1, + PCI_IRQ_MSI | PCI_IRQ_LEGACY); + if (nvec < 1) + return FAILED; + + flags = IRQF_SHARED; + } + + acb->vector_count = nvec; + for (i = 0; i < nvec; i++) { + if (request_irq(pci_irq_vector(pdev, i), arcmsr_do_interrupt, + flags, "arcmsr", acb)) { pr_warn("arcmsr%d: request_irq =%d failed!\n", - acb->host->host_no, entries[i].vector); - for (j = 0 ; j < i ; j++) - free_irq(entries[j].vector, acb); - pci_disable_msix(pdev); - goto msi_int; + acb->host->host_no, pci_irq_vector(pdev, i)); + goto out_free_irq; } - acb->entries[i] = entries[i]; - } - acb->acb_flags |= ACB_F_MSIX_ENABLED; - pr_info("arcmsr%d: msi-x enabled\n", acb->host->host_no); - return SUCCESS; -msi_int: - if (pci_enable_msi_exact(pdev, 1) < 0) - goto legacy_int; - if (request_irq(pdev->irq, arcmsr_do_interrupt, - IRQF_SHARED, "arcmsr", acb)) { - pr_warn("arcmsr%d: request_irq =%d failed!\n", - acb->host->host_no, pdev->irq); - pci_disable_msi(pdev); - goto legacy_int; - } - acb->acb_flags |= ACB_F_MSI_ENABLED; - pr_info("arcmsr%d: msi enabled\n", acb->host->host_no); - return SUCCESS; -legacy_int: - if (request_irq(pdev->irq, arcmsr_do_interrupt, - IRQF_SHARED, "arcmsr", acb)) { - pr_warn("arcmsr%d: request_irq = %d failed!\n", - acb->host->host_no, pdev->irq); - return FAILED; } + return SUCCESS; +out_free_irq: + while (--i >= 0) + free_irq(pci_irq_vector(pdev, i), acb); + pci_free_irq_vectors(pdev); + return FAILED; } static int arcmsr_probe(struct pci_dev *pdev, const struct pci_device_id *id) @@ -886,15 +874,9 @@ static void arcmsr_free_irq(struct pci_dev *pdev, { int i; - if (acb->acb_flags & ACB_F_MSI_ENABLED) { - free_irq(pdev->irq, acb); - pci_disable_msi(pdev); - } else if (acb->acb_flags & ACB_F_MSIX_ENABLED) { - for (i = 0; i < acb->msix_vector_count; i++) - free_irq(acb->entries[i].vector, acb); - pci_disable_msix(pdev); - } else - free_irq(pdev->irq, acb); + for (i = 0; i < acb->vector_count; i++) + free_irq(pci_irq_vector(pdev, i), acb); + pci_free_irq_vectors(pdev); } static int arcmsr_suspend(struct pci_dev *pdev, pm_message_t state) diff --git a/drivers/scsi/arm/cumana_1.c b/drivers/scsi/arm/cumana_1.c index 8e9cfe8..a87b99c 100644 --- a/drivers/scsi/arm/cumana_1.c +++ b/drivers/scsi/arm/cumana_1.c @@ -14,49 +14,48 @@ #include <scsi/scsi_host.h> #define priv(host) ((struct NCR5380_hostdata *)(host)->hostdata) -#define NCR5380_read(reg) cumanascsi_read(instance, reg) -#define NCR5380_write(reg, value) cumanascsi_write(instance, reg, value) +#define NCR5380_read(reg) cumanascsi_read(hostdata, reg) +#define NCR5380_write(reg, value) cumanascsi_write(hostdata, reg, value) -#define NCR5380_dma_xfer_len(instance, cmd, phase) (cmd->transfersize) +#define NCR5380_dma_xfer_len cumanascsi_dma_xfer_len #define NCR5380_dma_recv_setup cumanascsi_pread #define NCR5380_dma_send_setup cumanascsi_pwrite -#define NCR5380_dma_residual(instance) (0) +#define NCR5380_dma_residual NCR5380_dma_residual_none #define NCR5380_intr cumanascsi_intr #define NCR5380_queue_command cumanascsi_queue_command #define NCR5380_info cumanascsi_info #define NCR5380_implementation_fields \ - unsigned ctrl; \ - void __iomem *base; \ - void __iomem *dma + unsigned ctrl -#include "../NCR5380.h" +struct NCR5380_hostdata; +static u8 cumanascsi_read(struct NCR5380_hostdata *, unsigned int); +static void cumanascsi_write(struct NCR5380_hostdata *, unsigned int, u8); -void cumanascsi_setup(char *str, int *ints) -{ -} +#include "../NCR5380.h" #define CTRL 0x16fc #define STAT 0x2004 #define L(v) (((v)<<16)|((v) & 0x0000ffff)) #define H(v) (((v)>>16)|((v) & 0xffff0000)) -static inline int cumanascsi_pwrite(struct Scsi_Host *host, +static inline int cumanascsi_pwrite(struct NCR5380_hostdata *hostdata, unsigned char *addr, int len) { unsigned long *laddr; - void __iomem *dma = priv(host)->dma + 0x2000; + u8 __iomem *base = hostdata->io; + u8 __iomem *dma = hostdata->pdma_io + 0x2000; if(!len) return 0; - writeb(0x02, priv(host)->base + CTRL); + writeb(0x02, base + CTRL); laddr = (unsigned long *)addr; while(len >= 32) { unsigned int status; unsigned long v; - status = readb(priv(host)->base + STAT); + status = readb(base + STAT); if(status & 0x80) goto end; if(!(status & 0x40)) @@ -75,12 +74,12 @@ static inline int cumanascsi_pwrite(struct Scsi_Host *host, } addr = (unsigned char *)laddr; - writeb(0x12, priv(host)->base + CTRL); + writeb(0x12, base + CTRL); while(len > 0) { unsigned int status; - status = readb(priv(host)->base + STAT); + status = readb(base + STAT); if(status & 0x80) goto end; if(status & 0x40) @@ -90,7 +89,7 @@ static inline int cumanascsi_pwrite(struct Scsi_Host *host, break; } - status = readb(priv(host)->base + STAT); + status = readb(base + STAT); if(status & 0x80) goto end; if(status & 0x40) @@ -101,27 +100,28 @@ static inline int cumanascsi_pwrite(struct Scsi_Host *host, } } end: - writeb(priv(host)->ctrl | 0x40, priv(host)->base + CTRL); + writeb(hostdata->ctrl | 0x40, base + CTRL); if (len) return -1; return 0; } -static inline int cumanascsi_pread(struct Scsi_Host *host, +static inline int cumanascsi_pread(struct NCR5380_hostdata *hostdata, unsigned char *addr, int len) { unsigned long *laddr; - void __iomem *dma = priv(host)->dma + 0x2000; + u8 __iomem *base = hostdata->io; + u8 __iomem *dma = hostdata->pdma_io + 0x2000; if(!len) return 0; - writeb(0x00, priv(host)->base + CTRL); + writeb(0x00, base + CTRL); laddr = (unsigned long *)addr; while(len >= 32) { unsigned int status; - status = readb(priv(host)->base + STAT); + status = readb(base + STAT); if(status & 0x80) goto end; if(!(status & 0x40)) @@ -140,12 +140,12 @@ static inline int cumanascsi_pread(struct Scsi_Host *host, } addr = (unsigned char *)laddr; - writeb(0x10, priv(host)->base + CTRL); + writeb(0x10, base + CTRL); while(len > 0) { unsigned int status; - status = readb(priv(host)->base + STAT); + status = readb(base + STAT); if(status & 0x80) goto end; if(status & 0x40) @@ -155,7 +155,7 @@ static inline int cumanascsi_pread(struct Scsi_Host *host, break; } - status = readb(priv(host)->base + STAT); + status = readb(base + STAT); if(status & 0x80) goto end; if(status & 0x40) @@ -166,37 +166,45 @@ static inline int cumanascsi_pread(struct Scsi_Host *host, } } end: - writeb(priv(host)->ctrl | 0x40, priv(host)->base + CTRL); + writeb(hostdata->ctrl | 0x40, base + CTRL); if (len) return -1; return 0; } -static unsigned char cumanascsi_read(struct Scsi_Host *host, unsigned int reg) +static int cumanascsi_dma_xfer_len(struct NCR5380_hostdata *hostdata, + struct scsi_cmnd *cmd) +{ + return cmd->transfersize; +} + +static u8 cumanascsi_read(struct NCR5380_hostdata *hostdata, + unsigned int reg) { - void __iomem *base = priv(host)->base; - unsigned char val; + u8 __iomem *base = hostdata->io; + u8 val; writeb(0, base + CTRL); val = readb(base + 0x2100 + (reg << 2)); - priv(host)->ctrl = 0x40; + hostdata->ctrl = 0x40; writeb(0x40, base + CTRL); return val; } -static void cumanascsi_write(struct Scsi_Host *host, unsigned int reg, unsigned int value) +static void cumanascsi_write(struct NCR5380_hostdata *hostdata, + unsigned int reg, u8 value) { - void __iomem *base = priv(host)->base; + u8 __iomem *base = hostdata->io; writeb(0, base + CTRL); writeb(value, base + 0x2100 + (reg << 2)); - priv(host)->ctrl = 0x40; + hostdata->ctrl = 0x40; writeb(0x40, base + CTRL); } @@ -235,11 +243,11 @@ static int cumanascsi1_probe(struct expansion_card *ec, goto out_release; } - priv(host)->base = ioremap(ecard_resource_start(ec, ECARD_RES_IOCSLOW), - ecard_resource_len(ec, ECARD_RES_IOCSLOW)); - priv(host)->dma = ioremap(ecard_resource_start(ec, ECARD_RES_MEMC), - ecard_resource_len(ec, ECARD_RES_MEMC)); - if (!priv(host)->base || !priv(host)->dma) { + priv(host)->io = ioremap(ecard_resource_start(ec, ECARD_RES_IOCSLOW), + ecard_resource_len(ec, ECARD_RES_IOCSLOW)); + priv(host)->pdma_io = ioremap(ecard_resource_start(ec, ECARD_RES_MEMC), + ecard_resource_len(ec, ECARD_RES_MEMC)); + if (!priv(host)->io || !priv(host)->pdma_io) { ret = -ENOMEM; goto out_unmap; } @@ -253,7 +261,7 @@ static int cumanascsi1_probe(struct expansion_card *ec, NCR5380_maybe_reset_bus(host); priv(host)->ctrl = 0; - writeb(0, priv(host)->base + CTRL); + writeb(0, priv(host)->io + CTRL); ret = request_irq(host->irq, cumanascsi_intr, 0, "CumanaSCSI-1", host); @@ -275,8 +283,8 @@ static int cumanascsi1_probe(struct expansion_card *ec, out_exit: NCR5380_exit(host); out_unmap: - iounmap(priv(host)->base); - iounmap(priv(host)->dma); + iounmap(priv(host)->io); + iounmap(priv(host)->pdma_io); scsi_host_put(host); out_release: ecard_release_resources(ec); @@ -287,15 +295,17 @@ static int cumanascsi1_probe(struct expansion_card *ec, static void cumanascsi1_remove(struct expansion_card *ec) { struct Scsi_Host *host = ecard_get_drvdata(ec); + void __iomem *base = priv(host)->io; + void __iomem *dma = priv(host)->pdma_io; ecard_set_drvdata(ec, NULL); scsi_remove_host(host); free_irq(host->irq, host); NCR5380_exit(host); - iounmap(priv(host)->base); - iounmap(priv(host)->dma); scsi_host_put(host); + iounmap(base); + iounmap(dma); ecard_release_resources(ec); } diff --git a/drivers/scsi/arm/oak.c b/drivers/scsi/arm/oak.c index a396024..6be6666 100644 --- a/drivers/scsi/arm/oak.c +++ b/drivers/scsi/arm/oak.c @@ -16,21 +16,18 @@ #define priv(host) ((struct NCR5380_hostdata *)(host)->hostdata) -#define NCR5380_read(reg) \ - readb(priv(instance)->base + ((reg) << 2)) -#define NCR5380_write(reg, value) \ - writeb(value, priv(instance)->base + ((reg) << 2)) +#define NCR5380_read(reg) readb(hostdata->io + ((reg) << 2)) +#define NCR5380_write(reg, value) writeb(value, hostdata->io + ((reg) << 2)) -#define NCR5380_dma_xfer_len(instance, cmd, phase) (0) +#define NCR5380_dma_xfer_len NCR5380_dma_xfer_none #define NCR5380_dma_recv_setup oakscsi_pread #define NCR5380_dma_send_setup oakscsi_pwrite -#define NCR5380_dma_residual(instance) (0) +#define NCR5380_dma_residual NCR5380_dma_residual_none #define NCR5380_queue_command oakscsi_queue_command #define NCR5380_info oakscsi_info -#define NCR5380_implementation_fields \ - void __iomem *base +#define NCR5380_implementation_fields /* none */ #include "../NCR5380.h" @@ -40,10 +37,10 @@ #define STAT ((128 + 16) << 2) #define DATA ((128 + 8) << 2) -static inline int oakscsi_pwrite(struct Scsi_Host *instance, +static inline int oakscsi_pwrite(struct NCR5380_hostdata *hostdata, unsigned char *addr, int len) { - void __iomem *base = priv(instance)->base; + u8 __iomem *base = hostdata->io; printk("writing %p len %d\n",addr, len); @@ -55,10 +52,11 @@ printk("writing %p len %d\n",addr, len); return 0; } -static inline int oakscsi_pread(struct Scsi_Host *instance, +static inline int oakscsi_pread(struct NCR5380_hostdata *hostdata, unsigned char *addr, int len) { - void __iomem *base = priv(instance)->base; + u8 __iomem *base = hostdata->io; + printk("reading %p len %d\n", addr, len); while(len > 0) { @@ -133,15 +131,14 @@ static int oakscsi_probe(struct expansion_card *ec, const struct ecard_id *id) goto release; } - priv(host)->base = ioremap(ecard_resource_start(ec, ECARD_RES_MEMC), - ecard_resource_len(ec, ECARD_RES_MEMC)); - if (!priv(host)->base) { + priv(host)->io = ioremap(ecard_resource_start(ec, ECARD_RES_MEMC), + ecard_resource_len(ec, ECARD_RES_MEMC)); + if (!priv(host)->io) { ret = -ENOMEM; goto unreg; } host->irq = NO_IRQ; - host->n_io_port = 255; ret = NCR5380_init(host, FLAG_DMA_FIXUP | FLAG_LATE_DMA_SETUP); if (ret) @@ -159,7 +156,7 @@ static int oakscsi_probe(struct expansion_card *ec, const struct ecard_id *id) out_exit: NCR5380_exit(host); out_unmap: - iounmap(priv(host)->base); + iounmap(priv(host)->io); unreg: scsi_host_put(host); release: @@ -171,13 +168,14 @@ static int oakscsi_probe(struct expansion_card *ec, const struct ecard_id *id) static void oakscsi_remove(struct expansion_card *ec) { struct Scsi_Host *host = ecard_get_drvdata(ec); + void __iomem *base = priv(host)->io; ecard_set_drvdata(ec, NULL); scsi_remove_host(host); NCR5380_exit(host); - iounmap(priv(host)->base); scsi_host_put(host); + iounmap(base); ecard_release_resources(ec); } diff --git a/drivers/scsi/atari_scsi.c b/drivers/scsi/atari_scsi.c index a59ad94..105b353 100644 --- a/drivers/scsi/atari_scsi.c +++ b/drivers/scsi/atari_scsi.c @@ -57,6 +57,9 @@ #define NCR5380_implementation_fields /* none */ +static u8 (*atari_scsi_reg_read)(unsigned int); +static void (*atari_scsi_reg_write)(unsigned int, u8); + #define NCR5380_read(reg) atari_scsi_reg_read(reg) #define NCR5380_write(reg, value) atari_scsi_reg_write(reg, value) @@ -64,14 +67,10 @@ #define NCR5380_abort atari_scsi_abort #define NCR5380_info atari_scsi_info -#define NCR5380_dma_recv_setup(instance, data, count) \ - atari_scsi_dma_setup(instance, data, count, 0) -#define NCR5380_dma_send_setup(instance, data, count) \ - atari_scsi_dma_setup(instance, data, count, 1) -#define NCR5380_dma_residual(instance) \ - atari_scsi_dma_residual(instance) -#define NCR5380_dma_xfer_len(instance, cmd, phase) \ - atari_dma_xfer_len(cmd->SCp.this_residual, cmd, !((phase) & SR_IO)) +#define NCR5380_dma_xfer_len atari_scsi_dma_xfer_len +#define NCR5380_dma_recv_setup atari_scsi_dma_recv_setup +#define NCR5380_dma_send_setup atari_scsi_dma_send_setup +#define NCR5380_dma_residual atari_scsi_dma_residual #define NCR5380_acquire_dma_irq(instance) falcon_get_lock(instance) #define NCR5380_release_dma_irq(instance) falcon_release_lock() @@ -126,9 +125,6 @@ static inline unsigned long SCSI_DMA_GETADR(void) static void atari_scsi_fetch_restbytes(void); -static unsigned char (*atari_scsi_reg_read)(unsigned char reg); -static void (*atari_scsi_reg_write)(unsigned char reg, unsigned char value); - static unsigned long atari_dma_residual, atari_dma_startaddr; static short atari_dma_active; /* pointer to the dribble buffer */ @@ -457,15 +453,14 @@ static int __init atari_scsi_setup(char *str) __setup("atascsi=", atari_scsi_setup); #endif /* !MODULE */ - -static unsigned long atari_scsi_dma_setup(struct Scsi_Host *instance, +static unsigned long atari_scsi_dma_setup(struct NCR5380_hostdata *hostdata, void *data, unsigned long count, int dir) { unsigned long addr = virt_to_phys(data); - dprintk(NDEBUG_DMA, "scsi%d: setting up dma, data = %p, phys = %lx, count = %ld, " - "dir = %d\n", instance->host_no, data, addr, count, dir); + dprintk(NDEBUG_DMA, "scsi%d: setting up dma, data = %p, phys = %lx, count = %ld, dir = %d\n", + hostdata->host->host_no, data, addr, count, dir); if (!IS_A_TT() && !STRAM_ADDR(addr)) { /* If we have a non-DMAable address on a Falcon, use the dribble @@ -522,8 +517,19 @@ static unsigned long atari_scsi_dma_setup(struct Scsi_Host *instance, return count; } +static inline int atari_scsi_dma_recv_setup(struct NCR5380_hostdata *hostdata, + unsigned char *data, int count) +{ + return atari_scsi_dma_setup(hostdata, data, count, 0); +} + +static inline int atari_scsi_dma_send_setup(struct NCR5380_hostdata *hostdata, + unsigned char *data, int count) +{ + return atari_scsi_dma_setup(hostdata, data, count, 1); +} -static long atari_scsi_dma_residual(struct Scsi_Host *instance) +static int atari_scsi_dma_residual(struct NCR5380_hostdata *hostdata) { return atari_dma_residual; } @@ -564,10 +570,11 @@ static int falcon_classify_cmd(struct scsi_cmnd *cmd) * the overrun problem, so this question is academic :-) */ -static unsigned long atari_dma_xfer_len(unsigned long wanted_len, - struct scsi_cmnd *cmd, int write_flag) +static int atari_scsi_dma_xfer_len(struct NCR5380_hostdata *hostdata, + struct scsi_cmnd *cmd) { - unsigned long possible_len, limit; + int wanted_len = cmd->SCp.this_residual; + int possible_len, limit; if (wanted_len < DMA_MIN_SIZE) return 0; @@ -604,7 +611,7 @@ static unsigned long atari_dma_xfer_len(unsigned long wanted_len, * use the dribble buffer and thus can do only STRAM_BUFFER_SIZE bytes. */ - if (write_flag) { + if (cmd->sc_data_direction == DMA_TO_DEVICE) { /* Write operation can always use the DMA, but the transfer size must * be rounded up to the next multiple of 512 (atari_dma_setup() does * this). @@ -644,8 +651,8 @@ static unsigned long atari_dma_xfer_len(unsigned long wanted_len, possible_len = limit; if (possible_len != wanted_len) - dprintk(NDEBUG_DMA, "Sorry, must cut DMA transfer size to %ld bytes " - "instead of %ld\n", possible_len, wanted_len); + dprintk(NDEBUG_DMA, "DMA transfer now %d bytes instead of %d\n", + possible_len, wanted_len); return possible_len; } @@ -658,26 +665,38 @@ static unsigned long atari_dma_xfer_len(unsigned long wanted_len, * NCR5380_write call these functions via function pointers. */ -static unsigned char atari_scsi_tt_reg_read(unsigned char reg) +static u8 atari_scsi_tt_reg_read(unsigned int reg) { return tt_scsi_regp[reg * 2]; } -static void atari_scsi_tt_reg_write(unsigned char reg, unsigned char value) +static void atari_scsi_tt_reg_write(unsigned int reg, u8 value) { tt_scsi_regp[reg * 2] = value; } -static unsigned char atari_scsi_falcon_reg_read(unsigned char reg) +static u8 atari_scsi_falcon_reg_read(unsigned int reg) { - dma_wd.dma_mode_status= (u_short)(0x88 + reg); - return (u_char)dma_wd.fdc_acces_seccount; + unsigned long flags; + u8 result; + + reg += 0x88; + local_irq_save(flags); + dma_wd.dma_mode_status = (u_short)reg; + result = (u8)dma_wd.fdc_acces_seccount; + local_irq_restore(flags); + return result; } -static void atari_scsi_falcon_reg_write(unsigned char reg, unsigned char value) +static void atari_scsi_falcon_reg_write(unsigned int reg, u8 value) { - dma_wd.dma_mode_status = (u_short)(0x88 + reg); + unsigned long flags; + + reg += 0x88; + local_irq_save(flags); + dma_wd.dma_mode_status = (u_short)reg; dma_wd.fdc_acces_seccount = (u_short)value; + local_irq_restore(flags); } diff --git a/drivers/scsi/be2iscsi/be_main.c b/drivers/scsi/be2iscsi/be_main.c index d9239c2..b5112d6 100644 --- a/drivers/scsi/be2iscsi/be_main.c +++ b/drivers/scsi/be2iscsi/be_main.c @@ -3049,8 +3049,10 @@ static int beiscsi_create_eqs(struct beiscsi_hba *phba, eq_vaddress = pci_alloc_consistent(phba->pcidev, num_eq_pages * PAGE_SIZE, &paddr); - if (!eq_vaddress) + if (!eq_vaddress) { + ret = -ENOMEM; goto create_eq_error; + } mem->va = eq_vaddress; ret = be_fill_queue(eq, phba->params.num_eq_entries, @@ -3113,8 +3115,10 @@ static int beiscsi_create_cqs(struct beiscsi_hba *phba, cq_vaddress = pci_alloc_consistent(phba->pcidev, num_cq_pages * PAGE_SIZE, &paddr); - if (!cq_vaddress) + if (!cq_vaddress) { + ret = -ENOMEM; goto create_cq_error; + } ret = be_fill_queue(cq, phba->params.num_cq_entries, sizeof(struct sol_cqe), cq_vaddress); diff --git a/drivers/scsi/bfa/bfa_ioc.h b/drivers/scsi/bfa/bfa_ioc.h index 713745d..0f9fab7 100644 --- a/drivers/scsi/bfa/bfa_ioc.h +++ b/drivers/scsi/bfa/bfa_ioc.h @@ -111,20 +111,24 @@ struct bfa_meminfo_s { struct bfa_mem_kva_s kva_info; }; -/* BFA memory segment setup macros */ -#define bfa_mem_dma_setup(_meminfo, _dm_ptr, _seg_sz) do { \ - ((bfa_mem_dma_t *)(_dm_ptr))->mem_len = (_seg_sz); \ - if (_seg_sz) \ - list_add_tail(&((bfa_mem_dma_t *)_dm_ptr)->qe, \ - &(_meminfo)->dma_info.qe); \ -} while (0) +/* BFA memory segment setup helpers */ +static inline void bfa_mem_dma_setup(struct bfa_meminfo_s *meminfo, + struct bfa_mem_dma_s *dm_ptr, + size_t seg_sz) +{ + dm_ptr->mem_len = seg_sz; + if (seg_sz) + list_add_tail(&dm_ptr->qe, &meminfo->dma_info.qe); +} -#define bfa_mem_kva_setup(_meminfo, _kva_ptr, _seg_sz) do { \ - ((bfa_mem_kva_t *)(_kva_ptr))->mem_len = (_seg_sz); \ - if (_seg_sz) \ - list_add_tail(&((bfa_mem_kva_t *)_kva_ptr)->qe, \ - &(_meminfo)->kva_info.qe); \ -} while (0) +static inline void bfa_mem_kva_setup(struct bfa_meminfo_s *meminfo, + struct bfa_mem_kva_s *kva_ptr, + size_t seg_sz) +{ + kva_ptr->mem_len = seg_sz; + if (seg_sz) + list_add_tail(&kva_ptr->qe, &meminfo->kva_info.qe); +} /* BFA dma memory segments iterator */ #define bfa_mem_dma_sptr(_mod, _i) (&(_mod)->dma_seg[(_i)]) diff --git a/drivers/scsi/bfa/bfad_bsg.c b/drivers/scsi/bfa/bfad_bsg.c index d1ad020..a9a0016 100644 --- a/drivers/scsi/bfa/bfad_bsg.c +++ b/drivers/scsi/bfa/bfad_bsg.c @@ -3130,11 +3130,12 @@ bfad_iocmd_handler(struct bfad_s *bfad, unsigned int cmd, void *iocmd, } static int -bfad_im_bsg_vendor_request(struct fc_bsg_job *job) +bfad_im_bsg_vendor_request(struct bsg_job *job) { - uint32_t vendor_cmd = job->request->rqst_data.h_vendor.vendor_cmd[0]; - struct bfad_im_port_s *im_port = - (struct bfad_im_port_s *) job->shost->hostdata[0]; + struct fc_bsg_request *bsg_request = job->request; + struct fc_bsg_reply *bsg_reply = job->reply; + uint32_t vendor_cmd = bsg_request->rqst_data.h_vendor.vendor_cmd[0]; + struct bfad_im_port_s *im_port = shost_priv(fc_bsg_to_shost(job)); struct bfad_s *bfad = im_port->bfad; struct request_queue *request_q = job->req->q; void *payload_kbuf; @@ -3175,18 +3176,19 @@ bfad_im_bsg_vendor_request(struct fc_bsg_job *job) /* Fill the BSG job reply data */ job->reply_len = job->reply_payload.payload_len; - job->reply->reply_payload_rcv_len = job->reply_payload.payload_len; - job->reply->result = rc; + bsg_reply->reply_payload_rcv_len = job->reply_payload.payload_len; + bsg_reply->result = rc; - job->job_done(job); + bsg_job_done(job, bsg_reply->result, + bsg_reply->reply_payload_rcv_len); return rc; error: /* free the command buffer */ kfree(payload_kbuf); out: - job->reply->result = rc; + bsg_reply->result = rc; job->reply_len = sizeof(uint32_t); - job->reply->reply_payload_rcv_len = 0; + bsg_reply->reply_payload_rcv_len = 0; return rc; } @@ -3312,7 +3314,7 @@ bfad_fcxp_free_mem(struct bfad_s *bfad, struct bfad_buf_info *buf_base, } int -bfad_fcxp_bsg_send(struct fc_bsg_job *job, struct bfad_fcxp *drv_fcxp, +bfad_fcxp_bsg_send(struct bsg_job *job, struct bfad_fcxp *drv_fcxp, bfa_bsg_fcpt_t *bsg_fcpt) { struct bfa_fcxp_s *hal_fcxp; @@ -3352,28 +3354,29 @@ bfad_fcxp_bsg_send(struct fc_bsg_job *job, struct bfad_fcxp *drv_fcxp, } int -bfad_im_bsg_els_ct_request(struct fc_bsg_job *job) +bfad_im_bsg_els_ct_request(struct bsg_job *job) { struct bfa_bsg_data *bsg_data; - struct bfad_im_port_s *im_port = - (struct bfad_im_port_s *) job->shost->hostdata[0]; + struct bfad_im_port_s *im_port = shost_priv(fc_bsg_to_shost(job)); struct bfad_s *bfad = im_port->bfad; bfa_bsg_fcpt_t *bsg_fcpt; struct bfad_fcxp *drv_fcxp; struct bfa_fcs_lport_s *fcs_port; struct bfa_fcs_rport_s *fcs_rport; - uint32_t command_type = job->request->msgcode; + struct fc_bsg_request *bsg_request = bsg_request; + struct fc_bsg_reply *bsg_reply = job->reply; + uint32_t command_type = bsg_request->msgcode; unsigned long flags; struct bfad_buf_info *rsp_buf_info; void *req_kbuf = NULL, *rsp_kbuf = NULL; int rc = -EINVAL; job->reply_len = sizeof(uint32_t); /* Atleast uint32_t reply_len */ - job->reply->reply_payload_rcv_len = 0; + bsg_reply->reply_payload_rcv_len = 0; /* Get the payload passed in from userspace */ - bsg_data = (struct bfa_bsg_data *) (((char *)job->request) + - sizeof(struct fc_bsg_request)); + bsg_data = (struct bfa_bsg_data *) (((char *)bsg_request) + + sizeof(struct fc_bsg_request)); if (bsg_data == NULL) goto out; @@ -3517,13 +3520,13 @@ bfad_im_bsg_els_ct_request(struct fc_bsg_job *job) /* fill the job->reply data */ if (drv_fcxp->req_status == BFA_STATUS_OK) { job->reply_len = drv_fcxp->rsp_len; - job->reply->reply_payload_rcv_len = drv_fcxp->rsp_len; - job->reply->reply_data.ctels_reply.status = FC_CTELS_STATUS_OK; + bsg_reply->reply_payload_rcv_len = drv_fcxp->rsp_len; + bsg_reply->reply_data.ctels_reply.status = FC_CTELS_STATUS_OK; } else { - job->reply->reply_payload_rcv_len = + bsg_reply->reply_payload_rcv_len = sizeof(struct fc_bsg_ctels_reply); job->reply_len = sizeof(uint32_t); - job->reply->reply_data.ctels_reply.status = + bsg_reply->reply_data.ctels_reply.status = FC_CTELS_STATUS_REJECT; } @@ -3549,20 +3552,23 @@ out_free_mem: kfree(bsg_fcpt); kfree(drv_fcxp); out: - job->reply->result = rc; + bsg_reply->result = rc; if (rc == BFA_STATUS_OK) - job->job_done(job); + bsg_job_done(job, bsg_reply->result, + bsg_reply->reply_payload_rcv_len); return rc; } int -bfad_im_bsg_request(struct fc_bsg_job *job) +bfad_im_bsg_request(struct bsg_job *job) { + struct fc_bsg_request *bsg_request = job->request; + struct fc_bsg_reply *bsg_reply = job->reply; uint32_t rc = BFA_STATUS_OK; - switch (job->request->msgcode) { + switch (bsg_request->msgcode) { case FC_BSG_HST_VENDOR: /* Process BSG HST Vendor requests */ rc = bfad_im_bsg_vendor_request(job); @@ -3575,8 +3581,8 @@ bfad_im_bsg_request(struct fc_bsg_job *job) rc = bfad_im_bsg_els_ct_request(job); break; default: - job->reply->result = rc = -EINVAL; - job->reply->reply_payload_rcv_len = 0; + bsg_reply->result = rc = -EINVAL; + bsg_reply->reply_payload_rcv_len = 0; break; } @@ -3584,7 +3590,7 @@ bfad_im_bsg_request(struct fc_bsg_job *job) } int -bfad_im_bsg_timeout(struct fc_bsg_job *job) +bfad_im_bsg_timeout(struct bsg_job *job) { /* Don't complete the BSG job request - return -EAGAIN * to reset bsg job timeout : for ELS/CT pass thru we diff --git a/drivers/scsi/bfa/bfad_im.h b/drivers/scsi/bfa/bfad_im.h index 836fdc2..c81ec2a 100644 --- a/drivers/scsi/bfa/bfad_im.h +++ b/drivers/scsi/bfa/bfad_im.h @@ -166,8 +166,8 @@ extern struct device_attribute *bfad_im_vport_attrs[]; irqreturn_t bfad_intx(int irq, void *dev_id); -int bfad_im_bsg_request(struct fc_bsg_job *job); -int bfad_im_bsg_timeout(struct fc_bsg_job *job); +int bfad_im_bsg_request(struct bsg_job *job); +int bfad_im_bsg_timeout(struct bsg_job *job); /* * Macro to set the SCSI device sdev_bflags - sdev_bflags are used by the diff --git a/drivers/scsi/bnx2fc/bnx2fc_fcoe.c b/drivers/scsi/bnx2fc/bnx2fc_fcoe.c index f9ddb61..0990130 100644 --- a/drivers/scsi/bnx2fc/bnx2fc_fcoe.c +++ b/drivers/scsi/bnx2fc/bnx2fc_fcoe.c @@ -970,7 +970,6 @@ static int bnx2fc_libfc_config(struct fc_lport *lport) sizeof(struct libfc_function_template)); fc_elsct_init(lport); fc_exch_init(lport); - fc_rport_init(lport); fc_disc_init(lport); fc_disc_config(lport, lport); return 0; diff --git a/drivers/scsi/bnx2fc/bnx2fc_tgt.c b/drivers/scsi/bnx2fc/bnx2fc_tgt.c index 08ec318..739bfb6 100644 --- a/drivers/scsi/bnx2fc/bnx2fc_tgt.c +++ b/drivers/scsi/bnx2fc/bnx2fc_tgt.c @@ -80,7 +80,6 @@ static void bnx2fc_offload_session(struct fcoe_port *port, struct bnx2fc_rport *tgt, struct fc_rport_priv *rdata) { - struct fc_lport *lport = rdata->local_port; struct fc_rport *rport = rdata->rport; struct bnx2fc_interface *interface = port->priv; struct bnx2fc_hba *hba = interface->hba; @@ -160,7 +159,7 @@ ofld_err: tgt_init_err: if (tgt->fcoe_conn_id != -1) bnx2fc_free_conn_id(hba, tgt->fcoe_conn_id); - lport->tt.rport_logoff(rdata); + fc_rport_logoff(rdata); } void bnx2fc_flush_active_ios(struct bnx2fc_rport *tgt) diff --git a/drivers/scsi/cxgbi/cxgb4i/cxgb4i.c b/drivers/scsi/cxgbi/cxgb4i/cxgb4i.c index 4655a9f..9e6f647 100644 --- a/drivers/scsi/cxgbi/cxgb4i/cxgb4i.c +++ b/drivers/scsi/cxgbi/cxgb4i/cxgb4i.c @@ -1411,7 +1411,7 @@ static int init_act_open(struct cxgbi_sock *csk) csk->atid = cxgb4_alloc_atid(lldi->tids, csk); if (csk->atid < 0) { pr_err("%s, NO atid available.\n", ndev->name); - return -EINVAL; + goto rel_resource_without_clip; } cxgbi_sock_set_flag(csk, CTPF_HAS_ATID); cxgbi_sock_get(csk); diff --git a/drivers/scsi/cxlflash/common.h b/drivers/scsi/cxlflash/common.h index 6e68155..0e9de5d 100644 --- a/drivers/scsi/cxlflash/common.h +++ b/drivers/scsi/cxlflash/common.h @@ -19,6 +19,7 @@ #include <linux/rwsem.h> #include <linux/types.h> #include <scsi/scsi.h> +#include <scsi/scsi_cmnd.h> #include <scsi/scsi_device.h> extern const struct file_operations cxlflash_cxl_fops; @@ -62,11 +63,6 @@ static inline void check_sizes(void) /* AFU defines a fixed size of 4K for command buffers (borrow 4K page define) */ #define CMD_BUFSIZE SIZE_4K -/* flags in IOA status area for host use */ -#define B_DONE 0x01 -#define B_ERROR 0x02 /* set with B_DONE */ -#define B_TIMEOUT 0x04 /* set with B_DONE & B_ERROR */ - enum cxlflash_lr_state { LINK_RESET_INVALID, LINK_RESET_REQUIRED, @@ -132,12 +128,9 @@ struct cxlflash_cfg { struct afu_cmd { struct sisl_ioarcb rcb; /* IOARCB (cache line aligned) */ struct sisl_ioasa sa; /* IOASA must follow IOARCB */ - spinlock_t slock; - struct completion cevent; - char *buf; /* per command buffer */ struct afu *parent; - int slot; - atomic_t free; + struct scsi_cmnd *scp; + struct completion cevent; u8 cmd_tmf:1; @@ -147,19 +140,31 @@ struct afu_cmd { */ } __aligned(cache_line_size()); +static inline struct afu_cmd *sc_to_afuc(struct scsi_cmnd *sc) +{ + return PTR_ALIGN(scsi_cmd_priv(sc), __alignof__(struct afu_cmd)); +} + +static inline struct afu_cmd *sc_to_afucz(struct scsi_cmnd *sc) +{ + struct afu_cmd *afuc = sc_to_afuc(sc); + + memset(afuc, 0, sizeof(*afuc)); + return afuc; +} + struct afu { /* Stuff requiring alignment go first. */ u64 rrq_entry[NUM_RRQ_ENTRY]; /* 2K RRQ */ - /* - * Command & data for AFU commands. - */ - struct afu_cmd cmd[CXLFLASH_NUM_CMDS]; /* Beware of alignment till here. Preferably introduce new * fields after this point */ + int (*send_cmd)(struct afu *, struct afu_cmd *); + void (*context_reset)(struct afu_cmd *); + /* AFU HW */ struct cxl_ioctl_start_work work; struct cxlflash_afu_map __iomem *afu_map; /* entire MMIO map */ @@ -173,10 +178,10 @@ struct afu { u64 *hrrq_end; u64 *hrrq_curr; bool toggle; - bool read_room; - atomic64_t room; + atomic_t cmds_active; /* Number of currently active AFU commands */ + s64 room; + spinlock_t rrin_slock; /* Lock to rrin queuing and cmd_room updates */ u64 hb; - u32 cmd_couts; /* Number of command checkouts */ u32 internal_lun; /* User-desired LUN mode for this AFU */ char version[16]; diff --git a/drivers/scsi/cxlflash/lunmgt.c b/drivers/scsi/cxlflash/lunmgt.c index a0923ca..6c318db9 100644 --- a/drivers/scsi/cxlflash/lunmgt.c +++ b/drivers/scsi/cxlflash/lunmgt.c @@ -254,8 +254,14 @@ int cxlflash_manage_lun(struct scsi_device *sdev, if (lli->parent->mode != MODE_NONE) rc = -EBUSY; else { + /* + * Clean up local LUN for this port and reset table + * tracking when no more references exist. + */ sdev->hostdata = NULL; lli->port_sel &= ~CHAN2PORT(chan); + if (lli->port_sel == 0U) + lli->in_table = false; } } diff --git a/drivers/scsi/cxlflash/main.c b/drivers/scsi/cxlflash/main.c index b301655..b17ebf6 100644 --- a/drivers/scsi/cxlflash/main.c +++ b/drivers/scsi/cxlflash/main.c @@ -35,67 +35,6 @@ MODULE_AUTHOR("Matthew R. Ochs <mrochs@linux.vnet.ibm.com>"); MODULE_LICENSE("GPL"); /** - * cmd_checkout() - checks out an AFU command - * @afu: AFU to checkout from. - * - * Commands are checked out in a round-robin fashion. Note that since - * the command pool is larger than the hardware queue, the majority of - * times we will only loop once or twice before getting a command. The - * buffer and CDB within the command are initialized (zeroed) prior to - * returning. - * - * Return: The checked out command or NULL when command pool is empty. - */ -static struct afu_cmd *cmd_checkout(struct afu *afu) -{ - int k, dec = CXLFLASH_NUM_CMDS; - struct afu_cmd *cmd; - - while (dec--) { - k = (afu->cmd_couts++ & (CXLFLASH_NUM_CMDS - 1)); - - cmd = &afu->cmd[k]; - - if (!atomic_dec_if_positive(&cmd->free)) { - pr_devel("%s: returning found index=%d cmd=%p\n", - __func__, cmd->slot, cmd); - memset(cmd->buf, 0, CMD_BUFSIZE); - memset(cmd->rcb.cdb, 0, sizeof(cmd->rcb.cdb)); - return cmd; - } - } - - return NULL; -} - -/** - * cmd_checkin() - checks in an AFU command - * @cmd: AFU command to checkin. - * - * Safe to pass commands that have already been checked in. Several - * internal tracking fields are reset as part of the checkin. Note - * that these are intentionally reset prior to toggling the free bit - * to avoid clobbering values in the event that the command is checked - * out right away. - */ -static void cmd_checkin(struct afu_cmd *cmd) -{ - cmd->rcb.scp = NULL; - cmd->rcb.timeout = 0; - cmd->sa.ioasc = 0; - cmd->cmd_tmf = false; - cmd->sa.host_use[0] = 0; /* clears both completion and retry bytes */ - - if (unlikely(atomic_inc_return(&cmd->free) != 1)) { - pr_err("%s: Freeing cmd (%d) that is not in use!\n", - __func__, cmd->slot); - return; - } - - pr_devel("%s: released cmd %p index=%d\n", __func__, cmd, cmd->slot); -} - -/** * process_cmd_err() - command error handler * @cmd: AFU command that experienced the error. * @scp: SCSI command associated with the AFU command in error. @@ -212,7 +151,7 @@ static void process_cmd_err(struct afu_cmd *cmd, struct scsi_cmnd *scp) * * Prepares and submits command that has either completed or timed out to * the SCSI stack. Checks AFU command back into command pool for non-internal - * (rcb.scp populated) commands. + * (cmd->scp populated) commands. */ static void cmd_complete(struct afu_cmd *cmd) { @@ -222,19 +161,14 @@ static void cmd_complete(struct afu_cmd *cmd) struct cxlflash_cfg *cfg = afu->parent; bool cmd_is_tmf; - spin_lock_irqsave(&cmd->slock, lock_flags); - cmd->sa.host_use_b[0] |= B_DONE; - spin_unlock_irqrestore(&cmd->slock, lock_flags); - - if (cmd->rcb.scp) { - scp = cmd->rcb.scp; + if (cmd->scp) { + scp = cmd->scp; if (unlikely(cmd->sa.ioasc)) process_cmd_err(cmd, scp); else scp->result = (DID_OK << 16); cmd_is_tmf = cmd->cmd_tmf; - cmd_checkin(cmd); /* Don't use cmd after here */ pr_debug_ratelimited("%s: calling scsi_done scp=%p result=%X " "ioasc=%d\n", __func__, scp, scp->result, @@ -254,49 +188,19 @@ static void cmd_complete(struct afu_cmd *cmd) } /** - * context_reset() - timeout handler for AFU commands + * context_reset_ioarrin() - reset command owner context via IOARRIN register * @cmd: AFU command that timed out. - * - * Sends a reset to the AFU. */ -static void context_reset(struct afu_cmd *cmd) +static void context_reset_ioarrin(struct afu_cmd *cmd) { int nretry = 0; u64 rrin = 0x1; - u64 room = 0; struct afu *afu = cmd->parent; - ulong lock_flags; + struct cxlflash_cfg *cfg = afu->parent; + struct device *dev = &cfg->dev->dev; pr_debug("%s: cmd=%p\n", __func__, cmd); - spin_lock_irqsave(&cmd->slock, lock_flags); - - /* Already completed? */ - if (cmd->sa.host_use_b[0] & B_DONE) { - spin_unlock_irqrestore(&cmd->slock, lock_flags); - return; - } - - cmd->sa.host_use_b[0] |= (B_DONE | B_ERROR | B_TIMEOUT); - spin_unlock_irqrestore(&cmd->slock, lock_flags); - - /* - * We really want to send this reset at all costs, so spread - * out wait time on successive retries for available room. - */ - do { - room = readq_be(&afu->host_map->cmd_room); - atomic64_set(&afu->room, room); - if (room) - goto write_rrin; - udelay(1 << nretry); - } while (nretry++ < MC_ROOM_RETRY_CNT); - - pr_err("%s: no cmd_room to send reset\n", __func__); - return; - -write_rrin: - nretry = 0; writeq_be(rrin, &afu->host_map->ioarrin); do { rrin = readq_be(&afu->host_map->ioarrin); @@ -305,93 +209,81 @@ write_rrin: /* Double delay each time */ udelay(1 << nretry); } while (nretry++ < MC_ROOM_RETRY_CNT); + + dev_dbg(dev, "%s: returning rrin=0x%016llX nretry=%d\n", + __func__, rrin, nretry); } /** - * send_cmd() - sends an AFU command + * send_cmd_ioarrin() - sends an AFU command via IOARRIN register * @afu: AFU associated with the host. * @cmd: AFU command to send. * * Return: * 0 on success, SCSI_MLQUEUE_HOST_BUSY on failure */ -static int send_cmd(struct afu *afu, struct afu_cmd *cmd) +static int send_cmd_ioarrin(struct afu *afu, struct afu_cmd *cmd) { struct cxlflash_cfg *cfg = afu->parent; struct device *dev = &cfg->dev->dev; - int nretry = 0; int rc = 0; - u64 room; - long newval; + s64 room; + ulong lock_flags; /* - * This routine is used by critical users such an AFU sync and to - * send a task management function (TMF). Thus we want to retry a - * bit before returning an error. To avoid the performance penalty - * of MMIO, we spread the update of 'room' over multiple commands. + * To avoid the performance penalty of MMIO, spread the update of + * 'room' over multiple commands. */ -retry: - newval = atomic64_dec_if_positive(&afu->room); - if (!newval) { - do { - room = readq_be(&afu->host_map->cmd_room); - atomic64_set(&afu->room, room); - if (room) - goto write_ioarrin; - udelay(1 << nretry); - } while (nretry++ < MC_ROOM_RETRY_CNT); - - dev_err(dev, "%s: no cmd_room to send 0x%X\n", - __func__, cmd->rcb.cdb[0]); - - goto no_room; - } else if (unlikely(newval < 0)) { - /* This should be rare. i.e. Only if two threads race and - * decrement before the MMIO read is done. In this case - * just benefit from the other thread having updated - * afu->room. - */ - if (nretry++ < MC_ROOM_RETRY_CNT) { - udelay(1 << nretry); - goto retry; + spin_lock_irqsave(&afu->rrin_slock, lock_flags); + if (--afu->room < 0) { + room = readq_be(&afu->host_map->cmd_room); + if (room <= 0) { + dev_dbg_ratelimited(dev, "%s: no cmd_room to send " + "0x%02X, room=0x%016llX\n", + __func__, cmd->rcb.cdb[0], room); + afu->room = 0; + rc = SCSI_MLQUEUE_HOST_BUSY; + goto out; } - - goto no_room; + afu->room = room - 1; } -write_ioarrin: writeq_be((u64)&cmd->rcb, &afu->host_map->ioarrin); out: + spin_unlock_irqrestore(&afu->rrin_slock, lock_flags); pr_devel("%s: cmd=%p len=%d ea=%p rc=%d\n", __func__, cmd, cmd->rcb.data_len, (void *)cmd->rcb.data_ea, rc); return rc; - -no_room: - afu->read_room = true; - kref_get(&cfg->afu->mapcount); - schedule_work(&cfg->work_q); - rc = SCSI_MLQUEUE_HOST_BUSY; - goto out; } /** * wait_resp() - polls for a response or timeout to a sent AFU command * @afu: AFU associated with the host. * @cmd: AFU command that was sent. + * + * Return: + * 0 on success, -1 on timeout/error */ -static void wait_resp(struct afu *afu, struct afu_cmd *cmd) +static int wait_resp(struct afu *afu, struct afu_cmd *cmd) { + int rc = 0; ulong timeout = msecs_to_jiffies(cmd->rcb.timeout * 2 * 1000); timeout = wait_for_completion_timeout(&cmd->cevent, timeout); - if (!timeout) - context_reset(cmd); + if (!timeout) { + afu->context_reset(cmd); + rc = -1; + } - if (unlikely(cmd->sa.ioasc != 0)) + if (unlikely(cmd->sa.ioasc != 0)) { pr_err("%s: CMD 0x%X failed, IOASC: flags 0x%X, afu_rc 0x%X, " "scsi_rc 0x%X, fc_rc 0x%X\n", __func__, cmd->rcb.cdb[0], cmd->sa.rc.flags, cmd->sa.rc.afu_rc, cmd->sa.rc.scsi_rc, cmd->sa.rc.fc_rc); + rc = -1; + } + + return rc; } /** @@ -405,24 +297,15 @@ static void wait_resp(struct afu *afu, struct afu_cmd *cmd) */ static int send_tmf(struct afu *afu, struct scsi_cmnd *scp, u64 tmfcmd) { - struct afu_cmd *cmd; - u32 port_sel = scp->device->channel + 1; - short lflag = 0; struct Scsi_Host *host = scp->device->host; struct cxlflash_cfg *cfg = (struct cxlflash_cfg *)host->hostdata; + struct afu_cmd *cmd = sc_to_afucz(scp); struct device *dev = &cfg->dev->dev; ulong lock_flags; int rc = 0; ulong to; - cmd = cmd_checkout(afu); - if (unlikely(!cmd)) { - dev_err(dev, "%s: could not get a free command\n", __func__); - rc = SCSI_MLQUEUE_HOST_BUSY; - goto out; - } - /* When Task Management Function is active do not send another */ spin_lock_irqsave(&cfg->tmf_slock, lock_flags); if (cfg->tmf_active) @@ -430,28 +313,23 @@ static int send_tmf(struct afu *afu, struct scsi_cmnd *scp, u64 tmfcmd) !cfg->tmf_active, cfg->tmf_slock); cfg->tmf_active = true; - cmd->cmd_tmf = true; spin_unlock_irqrestore(&cfg->tmf_slock, lock_flags); + cmd->scp = scp; + cmd->parent = afu; + cmd->cmd_tmf = true; + cmd->rcb.ctx_id = afu->ctx_hndl; + cmd->rcb.msi = SISL_MSI_RRQ_UPDATED; cmd->rcb.port_sel = port_sel; cmd->rcb.lun_id = lun_to_lunid(scp->device->lun); - - lflag = SISL_REQ_FLAGS_TMF_CMD; - cmd->rcb.req_flags = (SISL_REQ_FLAGS_PORT_LUN_ID | - SISL_REQ_FLAGS_SUP_UNDERRUN | lflag); - - /* Stash the scp in the reserved field, for reuse during interrupt */ - cmd->rcb.scp = scp; - - /* Copy the CDB from the cmd passed in */ + SISL_REQ_FLAGS_SUP_UNDERRUN | + SISL_REQ_FLAGS_TMF_CMD); memcpy(cmd->rcb.cdb, &tmfcmd, sizeof(tmfcmd)); - /* Send the command */ - rc = send_cmd(afu, cmd); + rc = afu->send_cmd(afu, cmd); if (unlikely(rc)) { - cmd_checkin(cmd); spin_lock_irqsave(&cfg->tmf_slock, lock_flags); cfg->tmf_active = false; spin_unlock_irqrestore(&cfg->tmf_slock, lock_flags); @@ -507,12 +385,12 @@ static int cxlflash_queuecommand(struct Scsi_Host *host, struct scsi_cmnd *scp) struct cxlflash_cfg *cfg = (struct cxlflash_cfg *)host->hostdata; struct afu *afu = cfg->afu; struct device *dev = &cfg->dev->dev; - struct afu_cmd *cmd; + struct afu_cmd *cmd = sc_to_afucz(scp); + struct scatterlist *sg = scsi_sglist(scp); u32 port_sel = scp->device->channel + 1; - int nseg, i, ncount; - struct scatterlist *sg; + u16 req_flags = SISL_REQ_FLAGS_SUP_UNDERRUN; ulong lock_flags; - short lflag = 0; + int nseg = 0; int rc = 0; int kref_got = 0; @@ -552,55 +430,38 @@ static int cxlflash_queuecommand(struct Scsi_Host *host, struct scsi_cmnd *scp) break; } - cmd = cmd_checkout(afu); - if (unlikely(!cmd)) { - dev_err(dev, "%s: could not get a free command\n", __func__); - rc = SCSI_MLQUEUE_HOST_BUSY; - goto out; - } - kref_get(&cfg->afu->mapcount); kref_got = 1; + if (likely(sg)) { + nseg = scsi_dma_map(scp); + if (unlikely(nseg < 0)) { + dev_err(dev, "%s: Fail DMA map!\n", __func__); + rc = SCSI_MLQUEUE_HOST_BUSY; + goto out; + } + + cmd->rcb.data_len = sg_dma_len(sg); + cmd->rcb.data_ea = sg_dma_address(sg); + } + + cmd->scp = scp; + cmd->parent = afu; + cmd->rcb.ctx_id = afu->ctx_hndl; + cmd->rcb.msi = SISL_MSI_RRQ_UPDATED; cmd->rcb.port_sel = port_sel; cmd->rcb.lun_id = lun_to_lunid(scp->device->lun); if (scp->sc_data_direction == DMA_TO_DEVICE) - lflag = SISL_REQ_FLAGS_HOST_WRITE; - else - lflag = SISL_REQ_FLAGS_HOST_READ; - - cmd->rcb.req_flags = (SISL_REQ_FLAGS_PORT_LUN_ID | - SISL_REQ_FLAGS_SUP_UNDERRUN | lflag); - - /* Stash the scp in the reserved field, for reuse during interrupt */ - cmd->rcb.scp = scp; - - nseg = scsi_dma_map(scp); - if (unlikely(nseg < 0)) { - dev_err(dev, "%s: Fail DMA map! nseg=%d\n", - __func__, nseg); - rc = SCSI_MLQUEUE_HOST_BUSY; - goto out; - } + req_flags |= SISL_REQ_FLAGS_HOST_WRITE; - ncount = scsi_sg_count(scp); - scsi_for_each_sg(scp, sg, ncount, i) { - cmd->rcb.data_len = sg_dma_len(sg); - cmd->rcb.data_ea = sg_dma_address(sg); - } - - /* Copy the CDB from the scsi_cmnd passed in */ + cmd->rcb.req_flags = req_flags; memcpy(cmd->rcb.cdb, scp->cmnd, sizeof(cmd->rcb.cdb)); - /* Send the command */ - rc = send_cmd(afu, cmd); - if (unlikely(rc)) { - cmd_checkin(cmd); + rc = afu->send_cmd(afu, cmd); + if (unlikely(rc)) scsi_dma_unmap(scp); - } - out: if (kref_got) kref_put(&afu->mapcount, afu_unmap); @@ -628,17 +489,9 @@ static void cxlflash_wait_for_pci_err_recovery(struct cxlflash_cfg *cfg) */ static void free_mem(struct cxlflash_cfg *cfg) { - int i; - char *buf = NULL; struct afu *afu = cfg->afu; if (cfg->afu) { - for (i = 0; i < CXLFLASH_NUM_CMDS; i++) { - buf = afu->cmd[i].buf; - if (!((u64)buf & (PAGE_SIZE - 1))) - free_page((ulong)buf); - } - free_pages((ulong)afu, get_order(sizeof(struct afu))); cfg->afu = NULL; } @@ -650,30 +503,16 @@ static void free_mem(struct cxlflash_cfg *cfg) * * Safe to call with AFU in a partially allocated/initialized state. * - * Cleans up all state associated with the command queue, and unmaps + * Waits for any active internal AFU commands to timeout and then unmaps * the MMIO space. - * - * - complete() will take care of commands we initiated (they'll be checked - * in as part of the cleanup that occurs after the completion) - * - * - cmd_checkin() will take care of entries that we did not initiate and that - * have not (and will not) complete because they are sitting on a [now stale] - * hardware queue */ static void stop_afu(struct cxlflash_cfg *cfg) { - int i; struct afu *afu = cfg->afu; - struct afu_cmd *cmd; if (likely(afu)) { - for (i = 0; i < CXLFLASH_NUM_CMDS; i++) { - cmd = &afu->cmd[i]; - complete(&cmd->cevent); - if (!atomic_read(&cmd->free)) - cmd_checkin(cmd); - } - + while (atomic_read(&afu->cmds_active)) + ssleep(1); if (likely(afu->afu_map)) { cxl_psa_unmap((void __iomem *)afu->afu_map); afu->afu_map = NULL; @@ -886,8 +725,6 @@ static void cxlflash_remove(struct pci_dev *pdev) static int alloc_mem(struct cxlflash_cfg *cfg) { int rc = 0; - int i; - char *buf = NULL; struct device *dev = &cfg->dev->dev; /* AFU is ~12k, i.e. only one 64k page or up to four 4k pages */ @@ -901,25 +738,6 @@ static int alloc_mem(struct cxlflash_cfg *cfg) } cfg->afu->parent = cfg; cfg->afu->afu_map = NULL; - - for (i = 0; i < CXLFLASH_NUM_CMDS; buf += CMD_BUFSIZE, i++) { - if (!((u64)buf & (PAGE_SIZE - 1))) { - buf = (void *)__get_free_page(GFP_KERNEL | __GFP_ZERO); - if (unlikely(!buf)) { - dev_err(dev, - "%s: Allocate command buffers fail!\n", - __func__); - rc = -ENOMEM; - free_mem(cfg); - goto out; - } - } - - cfg->afu->cmd[i].buf = buf; - atomic_set(&cfg->afu->cmd[i].free, 1); - cfg->afu->cmd[i].slot = i; - } - out: return rc; } @@ -1549,13 +1367,6 @@ static void init_pcr(struct cxlflash_cfg *cfg) /* Program the Endian Control for the master context */ writeq_be(SISL_ENDIAN_CTRL, &afu->host_map->endian_ctrl); - - /* Initialize cmd fields that never change */ - for (i = 0; i < CXLFLASH_NUM_CMDS; i++) { - afu->cmd[i].rcb.ctx_id = afu->ctx_hndl; - afu->cmd[i].rcb.msi = SISL_MSI_RRQ_UPDATED; - afu->cmd[i].rcb.rrq = 0x0; - } } /** @@ -1644,19 +1455,8 @@ out: static int start_afu(struct cxlflash_cfg *cfg) { struct afu *afu = cfg->afu; - struct afu_cmd *cmd; - - int i = 0; int rc = 0; - for (i = 0; i < CXLFLASH_NUM_CMDS; i++) { - cmd = &afu->cmd[i]; - - init_completion(&cmd->cevent); - spin_lock_init(&cmd->slock); - cmd->parent = afu; - } - init_pcr(cfg); /* After an AFU reset, RRQ entries are stale, clear them */ @@ -1829,6 +1629,9 @@ static int init_afu(struct cxlflash_cfg *cfg) goto err2; } + afu->send_cmd = send_cmd_ioarrin; + afu->context_reset = context_reset_ioarrin; + pr_debug("%s: afu version %s, interface version 0x%llX\n", __func__, afu->version, afu->interface_version); @@ -1840,7 +1643,8 @@ static int init_afu(struct cxlflash_cfg *cfg) } afu_err_intr_init(cfg->afu); - atomic64_set(&afu->room, readq_be(&afu->host_map->cmd_room)); + spin_lock_init(&afu->rrin_slock); + afu->room = readq_be(&afu->host_map->cmd_room); /* Restore the LUN mappings */ cxlflash_restore_luntable(cfg); @@ -1884,8 +1688,8 @@ int cxlflash_afu_sync(struct afu *afu, ctx_hndl_t ctx_hndl_u, struct cxlflash_cfg *cfg = afu->parent; struct device *dev = &cfg->dev->dev; struct afu_cmd *cmd = NULL; + char *buf = NULL; int rc = 0; - int retry_cnt = 0; static DEFINE_MUTEX(sync_active); if (cfg->state != STATE_NORMAL) { @@ -1894,27 +1698,23 @@ int cxlflash_afu_sync(struct afu *afu, ctx_hndl_t ctx_hndl_u, } mutex_lock(&sync_active); -retry: - cmd = cmd_checkout(afu); - if (unlikely(!cmd)) { - retry_cnt++; - udelay(1000 * retry_cnt); - if (retry_cnt < MC_RETRY_CNT) - goto retry; - dev_err(dev, "%s: could not get a free command\n", __func__); + atomic_inc(&afu->cmds_active); + buf = kzalloc(sizeof(*cmd) + __alignof__(*cmd) - 1, GFP_KERNEL); + if (unlikely(!buf)) { + dev_err(dev, "%s: no memory for command\n", __func__); rc = -1; goto out; } - pr_debug("%s: afu=%p cmd=%p %d\n", __func__, afu, cmd, ctx_hndl_u); + cmd = (struct afu_cmd *)PTR_ALIGN(buf, __alignof__(*cmd)); + init_completion(&cmd->cevent); + cmd->parent = afu; - memset(cmd->rcb.cdb, 0, sizeof(cmd->rcb.cdb)); + pr_debug("%s: afu=%p cmd=%p %d\n", __func__, afu, cmd, ctx_hndl_u); cmd->rcb.req_flags = SISL_REQ_FLAGS_AFU_CMD; - cmd->rcb.port_sel = 0x0; /* NA */ - cmd->rcb.lun_id = 0x0; /* NA */ - cmd->rcb.data_len = 0x0; - cmd->rcb.data_ea = 0x0; + cmd->rcb.ctx_id = afu->ctx_hndl; + cmd->rcb.msi = SISL_MSI_RRQ_UPDATED; cmd->rcb.timeout = MC_AFU_SYNC_TIMEOUT; cmd->rcb.cdb[0] = 0xC0; /* AFU Sync */ @@ -1924,20 +1724,17 @@ retry: *((__be16 *)&cmd->rcb.cdb[2]) = cpu_to_be16(ctx_hndl_u); *((__be32 *)&cmd->rcb.cdb[4]) = cpu_to_be32(res_hndl_u); - rc = send_cmd(afu, cmd); + rc = afu->send_cmd(afu, cmd); if (unlikely(rc)) goto out; - wait_resp(afu, cmd); - - /* Set on timeout */ - if (unlikely((cmd->sa.ioasc != 0) || - (cmd->sa.host_use_b[0] & B_ERROR))) + rc = wait_resp(afu, cmd); + if (unlikely(rc)) rc = -1; out: + atomic_dec(&afu->cmds_active); mutex_unlock(&sync_active); - if (cmd) - cmd_checkin(cmd); + kfree(buf); pr_debug("%s: returning rc=%d\n", __func__, rc); return rc; } @@ -2376,8 +2173,9 @@ static struct scsi_host_template driver_template = { .change_queue_depth = cxlflash_change_queue_depth, .cmd_per_lun = CXLFLASH_MAX_CMDS_PER_LUN, .can_queue = CXLFLASH_MAX_CMDS, + .cmd_size = sizeof(struct afu_cmd) + __alignof__(struct afu_cmd) - 1, .this_id = -1, - .sg_tablesize = SG_NONE, /* No scatter gather support */ + .sg_tablesize = 1, /* No scatter gather support */ .max_sectors = CXLFLASH_MAX_SECTORS, .use_clustering = ENABLE_CLUSTERING, .shost_attrs = cxlflash_host_attrs, @@ -2412,7 +2210,6 @@ MODULE_DEVICE_TABLE(pci, cxlflash_pci_table); * Handles the following events: * - Link reset which cannot be performed on interrupt context due to * blocking up to a few seconds - * - Read AFU command room * - Rescan the host */ static void cxlflash_worker_thread(struct work_struct *work) @@ -2449,11 +2246,6 @@ static void cxlflash_worker_thread(struct work_struct *work) cfg->lr_state = LINK_RESET_COMPLETE; } - if (afu->read_room) { - atomic64_set(&afu->room, readq_be(&afu->host_map->cmd_room)); - afu->read_room = false; - } - spin_unlock_irqrestore(cfg->host->host_lock, lock_flags); if (atomic_dec_if_positive(&cfg->scan_host_needed) >= 0) diff --git a/drivers/scsi/cxlflash/sislite.h b/drivers/scsi/cxlflash/sislite.h index 347fc16..1a2d09c 100644 --- a/drivers/scsi/cxlflash/sislite.h +++ b/drivers/scsi/cxlflash/sislite.h @@ -72,7 +72,7 @@ struct sisl_ioarcb { u16 timeout; /* in units specified by req_flags */ u32 rsvd1; u8 cdb[16]; /* must be in big endian */ - struct scsi_cmnd *scp; + u64 reserved; /* Reserved area */ } __packed; struct sisl_rc { diff --git a/drivers/scsi/device_handler/scsi_dh_alua.c b/drivers/scsi/device_handler/scsi_dh_alua.c index db03c49..d704752 100644 --- a/drivers/scsi/device_handler/scsi_dh_alua.c +++ b/drivers/scsi/device_handler/scsi_dh_alua.c @@ -95,7 +95,7 @@ struct alua_port_group { struct alua_dh_data { struct list_head node; - struct alua_port_group *pg; + struct alua_port_group __rcu *pg; int group_id; spinlock_t pg_lock; struct scsi_device *sdev; @@ -371,7 +371,7 @@ static int alua_check_vpd(struct scsi_device *sdev, struct alua_dh_data *h, /* Check for existing port group references */ spin_lock(&h->pg_lock); - old_pg = h->pg; + old_pg = rcu_dereference_protected(h->pg, lockdep_is_held(&h->pg_lock)); if (old_pg != pg) { /* port group has changed. Update to new port group */ if (h->pg) { @@ -390,7 +390,9 @@ static int alua_check_vpd(struct scsi_device *sdev, struct alua_dh_data *h, list_add_rcu(&h->node, &pg->dh_list); spin_unlock_irqrestore(&pg->lock, flags); - alua_rtpg_queue(h->pg, sdev, NULL, true); + alua_rtpg_queue(rcu_dereference_protected(h->pg, + lockdep_is_held(&h->pg_lock)), + sdev, NULL, true); spin_unlock(&h->pg_lock); if (old_pg) @@ -942,7 +944,7 @@ static int alua_initialize(struct scsi_device *sdev, struct alua_dh_data *h) static int alua_set_params(struct scsi_device *sdev, const char *params) { struct alua_dh_data *h = sdev->handler_data; - struct alua_port_group __rcu *pg = NULL; + struct alua_port_group *pg = NULL; unsigned int optimize = 0, argc; const char *p = params; int result = SCSI_DH_OK; @@ -989,7 +991,7 @@ static int alua_activate(struct scsi_device *sdev, struct alua_dh_data *h = sdev->handler_data; int err = SCSI_DH_OK; struct alua_queue_data *qdata; - struct alua_port_group __rcu *pg; + struct alua_port_group *pg; qdata = kzalloc(sizeof(*qdata), GFP_KERNEL); if (!qdata) { @@ -1053,7 +1055,7 @@ static void alua_check(struct scsi_device *sdev, bool force) static int alua_prep_fn(struct scsi_device *sdev, struct request *req) { struct alua_dh_data *h = sdev->handler_data; - struct alua_port_group __rcu *pg; + struct alua_port_group *pg; unsigned char state = SCSI_ACCESS_STATE_OPTIMAL; int ret = BLKPREP_OK; @@ -1123,7 +1125,7 @@ static void alua_bus_detach(struct scsi_device *sdev) struct alua_port_group *pg; spin_lock(&h->pg_lock); - pg = h->pg; + pg = rcu_dereference_protected(h->pg, lockdep_is_held(&h->pg_lock)); rcu_assign_pointer(h->pg, NULL); h->sdev = NULL; spin_unlock(&h->pg_lock); diff --git a/drivers/scsi/dmx3191d.c b/drivers/scsi/dmx3191d.c index 9b5a457..6af3394 100644 --- a/drivers/scsi/dmx3191d.c +++ b/drivers/scsi/dmx3191d.c @@ -34,13 +34,13 @@ * Definitions for the generic 5380 driver. */ -#define NCR5380_read(reg) inb(instance->io_port + reg) -#define NCR5380_write(reg, value) outb(value, instance->io_port + reg) +#define NCR5380_read(reg) inb(hostdata->base + (reg)) +#define NCR5380_write(reg, value) outb(value, hostdata->base + (reg)) -#define NCR5380_dma_xfer_len(instance, cmd, phase) (0) -#define NCR5380_dma_recv_setup(instance, dst, len) (0) -#define NCR5380_dma_send_setup(instance, src, len) (0) -#define NCR5380_dma_residual(instance) (0) +#define NCR5380_dma_xfer_len NCR5380_dma_xfer_none +#define NCR5380_dma_recv_setup NCR5380_dma_setup_none +#define NCR5380_dma_send_setup NCR5380_dma_setup_none +#define NCR5380_dma_residual NCR5380_dma_residual_none #define NCR5380_implementation_fields /* none */ @@ -71,6 +71,7 @@ static int dmx3191d_probe_one(struct pci_dev *pdev, const struct pci_device_id *id) { struct Scsi_Host *shost; + struct NCR5380_hostdata *hostdata; unsigned long io; int error = -ENODEV; @@ -88,7 +89,9 @@ static int dmx3191d_probe_one(struct pci_dev *pdev, sizeof(struct NCR5380_hostdata)); if (!shost) goto out_release_region; - shost->io_port = io; + + hostdata = shost_priv(shost); + hostdata->base = io; /* This card does not seem to raise an interrupt on pdev->irq. * Steam-powered SCSI controllers run without an IRQ anyway. @@ -125,7 +128,8 @@ out_host_put: static void dmx3191d_remove_one(struct pci_dev *pdev) { struct Scsi_Host *shost = pci_get_drvdata(pdev); - unsigned long io = shost->io_port; + struct NCR5380_hostdata *hostdata = shost_priv(shost); + unsigned long io = hostdata->base; scsi_remove_host(shost); @@ -149,18 +153,7 @@ static struct pci_driver dmx3191d_pci_driver = { .remove = dmx3191d_remove_one, }; -static int __init dmx3191d_init(void) -{ - return pci_register_driver(&dmx3191d_pci_driver); -} - -static void __exit dmx3191d_exit(void) -{ - pci_unregister_driver(&dmx3191d_pci_driver); -} - -module_init(dmx3191d_init); -module_exit(dmx3191d_exit); +module_pci_driver(dmx3191d_pci_driver); MODULE_AUTHOR("Massimo Piccioni <dafastidio@libero.it>"); MODULE_DESCRIPTION("Domex DMX3191D SCSI driver"); diff --git a/drivers/scsi/dpt_i2o.c b/drivers/scsi/dpt_i2o.c index 21c8d21..27c0dce 100644 --- a/drivers/scsi/dpt_i2o.c +++ b/drivers/scsi/dpt_i2o.c @@ -651,7 +651,6 @@ static u32 adpt_ioctl_to_context(adpt_hba * pHba, void *reply) } spin_unlock_irqrestore(pHba->host->host_lock, flags); if (i >= nr) { - kfree (reply); printk(KERN_WARNING"%s: Too many outstanding " "ioctl commands\n", pHba->name); return (u32)-1; @@ -1754,8 +1753,10 @@ static int adpt_i2o_passthru(adpt_hba* pHba, u32 __user *arg) sg_offset = (msg[0]>>4)&0xf; msg[2] = 0x40000000; // IOCTL context msg[3] = adpt_ioctl_to_context(pHba, reply); - if (msg[3] == (u32)-1) + if (msg[3] == (u32)-1) { + kfree(reply); return -EBUSY; + } memset(sg_list,0, sizeof(sg_list[0])*pHba->sg_tablesize); if(sg_offset) { @@ -3350,7 +3351,7 @@ static int adpt_i2o_query_scalar(adpt_hba* pHba, int tid, if (opblk_va == NULL) { dma_free_coherent(&pHba->pDev->dev, sizeof(u8) * (8+buflen), resblk_va, resblk_pa); - printk(KERN_CRIT "%s: query operatio failed; Out of memory.\n", + printk(KERN_CRIT "%s: query operation failed; Out of memory.\n", pHba->name); return -ENOMEM; } diff --git a/drivers/scsi/fcoe/fcoe.c b/drivers/scsi/fcoe/fcoe.c index 9bd41a3..59150ca 100644 --- a/drivers/scsi/fcoe/fcoe.c +++ b/drivers/scsi/fcoe/fcoe.c @@ -63,6 +63,14 @@ unsigned int fcoe_debug_logging; module_param_named(debug_logging, fcoe_debug_logging, int, S_IRUGO|S_IWUSR); MODULE_PARM_DESC(debug_logging, "a bit mask of logging levels"); +unsigned int fcoe_e_d_tov = 2 * 1000; +module_param_named(e_d_tov, fcoe_e_d_tov, int, S_IRUGO|S_IWUSR); +MODULE_PARM_DESC(e_d_tov, "E_D_TOV in ms, default 2000"); + +unsigned int fcoe_r_a_tov = 2 * 2 * 1000; +module_param_named(r_a_tov, fcoe_r_a_tov, int, S_IRUGO|S_IWUSR); +MODULE_PARM_DESC(r_a_tov, "R_A_TOV in ms, default 4000"); + static DEFINE_MUTEX(fcoe_config_mutex); static struct workqueue_struct *fcoe_wq; @@ -582,7 +590,8 @@ static void fcoe_fip_send(struct fcoe_ctlr *fip, struct sk_buff *skb) * Use default VLAN for FIP VLAN discovery protocol */ frame = (struct fip_frame *)skb->data; - if (frame->fip.fip_op == ntohs(FIP_OP_VLAN) && + if (ntohs(frame->eth.h_proto) == ETH_P_FIP && + ntohs(frame->fip.fip_op) == FIP_OP_VLAN && fcoe->realdev != fcoe->netdev) skb->dev = fcoe->realdev; else @@ -633,8 +642,8 @@ static int fcoe_lport_config(struct fc_lport *lport) lport->qfull = 0; lport->max_retry_count = 3; lport->max_rport_retry_count = 3; - lport->e_d_tov = 2 * 1000; /* FC-FS default */ - lport->r_a_tov = 2 * 2 * 1000; + lport->e_d_tov = fcoe_e_d_tov; + lport->r_a_tov = fcoe_r_a_tov; lport->service_params = (FCP_SPPF_INIT_FCN | FCP_SPPF_RD_XRDY_DIS | FCP_SPPF_RETRY | FCP_SPPF_CONF_COMPL); lport->does_npiv = 1; @@ -2160,11 +2169,13 @@ static bool fcoe_match(struct net_device *netdev) */ static void fcoe_dcb_create(struct fcoe_interface *fcoe) { + int ctlr_prio = TC_PRIO_BESTEFFORT; + int fcoe_prio = TC_PRIO_INTERACTIVE; + struct fcoe_ctlr *ctlr = fcoe_to_ctlr(fcoe); #ifdef CONFIG_DCB int dcbx; u8 fup, up; struct net_device *netdev = fcoe->realdev; - struct fcoe_ctlr *ctlr = fcoe_to_ctlr(fcoe); struct dcb_app app = { .priority = 0, .protocol = ETH_P_FCOE @@ -2186,10 +2197,12 @@ static void fcoe_dcb_create(struct fcoe_interface *fcoe) fup = dcb_getapp(netdev, &app); } - fcoe->priority = ffs(up) ? ffs(up) - 1 : 0; - ctlr->priority = ffs(fup) ? ffs(fup) - 1 : fcoe->priority; + fcoe_prio = ffs(up) ? ffs(up) - 1 : 0; + ctlr_prio = ffs(fup) ? ffs(fup) - 1 : fcoe_prio; } #endif + fcoe->priority = fcoe_prio; + ctlr->priority = ctlr_prio; } enum fcoe_create_link_state { diff --git a/drivers/scsi/fcoe/fcoe_ctlr.c b/drivers/scsi/fcoe/fcoe_ctlr.c index dcf3653..cea57e2 100644 --- a/drivers/scsi/fcoe/fcoe_ctlr.c +++ b/drivers/scsi/fcoe/fcoe_ctlr.c @@ -801,6 +801,8 @@ int fcoe_ctlr_els_send(struct fcoe_ctlr *fip, struct fc_lport *lport, return -EINPROGRESS; drop: kfree_skb(skb); + LIBFCOE_FIP_DBG(fip, "drop els_send op %u d_id %x\n", + op, ntoh24(fh->fh_d_id)); return -EINVAL; } EXPORT_SYMBOL(fcoe_ctlr_els_send); @@ -1316,7 +1318,7 @@ drop: * The overall length has already been checked. */ static void fcoe_ctlr_recv_clr_vlink(struct fcoe_ctlr *fip, - struct fip_header *fh) + struct sk_buff *skb) { struct fip_desc *desc; struct fip_mac_desc *mp; @@ -1331,14 +1333,18 @@ static void fcoe_ctlr_recv_clr_vlink(struct fcoe_ctlr *fip, int num_vlink_desc; int reset_phys_port = 0; struct fip_vn_desc **vlink_desc_arr = NULL; + struct fip_header *fh = (struct fip_header *)skb->data; + struct ethhdr *eh = eth_hdr(skb); LIBFCOE_FIP_DBG(fip, "Clear Virtual Link received\n"); - if (!fcf || !lport->port_id) { + if (!fcf) { /* * We are yet to select best FCF, but we got CVL in the * meantime. reset the ctlr and let it rediscover the FCF */ + LIBFCOE_FIP_DBG(fip, "Resetting fcoe_ctlr as FCF has not been " + "selected yet\n"); mutex_lock(&fip->ctlr_mutex); fcoe_ctlr_reset(fip); mutex_unlock(&fip->ctlr_mutex); @@ -1346,6 +1352,31 @@ static void fcoe_ctlr_recv_clr_vlink(struct fcoe_ctlr *fip, } /* + * If we've selected an FCF check that the CVL is from there to avoid + * processing CVLs from an unexpected source. If it is from an + * unexpected source drop it on the floor. + */ + if (!ether_addr_equal(eh->h_source, fcf->fcf_mac)) { + LIBFCOE_FIP_DBG(fip, "Dropping CVL due to source address " + "mismatch with FCF src=%pM\n", eh->h_source); + return; + } + + /* + * If we haven't logged into the fabric but receive a CVL we should + * reset everything and go back to solicitation. + */ + if (!lport->port_id) { + LIBFCOE_FIP_DBG(fip, "lport not logged in, resoliciting\n"); + mutex_lock(&fip->ctlr_mutex); + fcoe_ctlr_reset(fip); + mutex_unlock(&fip->ctlr_mutex); + fc_lport_reset(fip->lp); + fcoe_ctlr_solicit(fip, NULL); + return; + } + + /* * mask of required descriptors. Validating each one clears its bit. */ desc_mask = BIT(FIP_DT_MAC) | BIT(FIP_DT_NAME); @@ -1576,7 +1607,7 @@ static int fcoe_ctlr_recv_handler(struct fcoe_ctlr *fip, struct sk_buff *skb) if (op == FIP_OP_DISC && sub == FIP_SC_ADV) fcoe_ctlr_recv_adv(fip, skb); else if (op == FIP_OP_CTRL && sub == FIP_SC_CLR_VLINK) - fcoe_ctlr_recv_clr_vlink(fip, fiph); + fcoe_ctlr_recv_clr_vlink(fip, skb); kfree_skb(skb); return 0; drop: @@ -2122,7 +2153,7 @@ static void fcoe_ctlr_vn_rport_callback(struct fc_lport *lport, LIBFCOE_FIP_DBG(fip, "rport FLOGI limited port_id %6.6x\n", rdata->ids.port_id); - lport->tt.rport_logoff(rdata); + fc_rport_logoff(rdata); } break; default: @@ -2145,9 +2176,15 @@ static void fcoe_ctlr_disc_stop_locked(struct fc_lport *lport) { struct fc_rport_priv *rdata; + rcu_read_lock(); + list_for_each_entry_rcu(rdata, &lport->disc.rports, peers) { + if (kref_get_unless_zero(&rdata->kref)) { + fc_rport_logoff(rdata); + kref_put(&rdata->kref, fc_rport_destroy); + } + } + rcu_read_unlock(); mutex_lock(&lport->disc.disc_mutex); - list_for_each_entry_rcu(rdata, &lport->disc.rports, peers) - lport->tt.rport_logoff(rdata); lport->disc.disc_callback = NULL; mutex_unlock(&lport->disc.disc_mutex); } @@ -2178,7 +2215,7 @@ static void fcoe_ctlr_disc_stop(struct fc_lport *lport) static void fcoe_ctlr_disc_stop_final(struct fc_lport *lport) { fcoe_ctlr_disc_stop(lport); - lport->tt.rport_flush_queue(); + fc_rport_flush_queue(); synchronize_rcu(); } @@ -2393,6 +2430,8 @@ static void fcoe_ctlr_vn_probe_req(struct fcoe_ctlr *fip, switch (fip->state) { case FIP_ST_VNMP_CLAIM: case FIP_ST_VNMP_UP: + LIBFCOE_FIP_DBG(fip, "vn_probe_req: send reply, state %x\n", + fip->state); fcoe_ctlr_vn_send(fip, FIP_SC_VN_PROBE_REP, frport->enode_mac, 0); break; @@ -2407,15 +2446,21 @@ static void fcoe_ctlr_vn_probe_req(struct fcoe_ctlr *fip, */ if (fip->lp->wwpn > rdata->ids.port_name && !(frport->flags & FIP_FL_REC_OR_P2P)) { + LIBFCOE_FIP_DBG(fip, "vn_probe_req: " + "port_id collision\n"); fcoe_ctlr_vn_send(fip, FIP_SC_VN_PROBE_REP, frport->enode_mac, 0); break; } /* fall through */ case FIP_ST_VNMP_START: + LIBFCOE_FIP_DBG(fip, "vn_probe_req: " + "restart VN2VN negotiation\n"); fcoe_ctlr_vn_restart(fip); break; default: + LIBFCOE_FIP_DBG(fip, "vn_probe_req: ignore state %x\n", + fip->state); break; } } @@ -2437,9 +2482,12 @@ static void fcoe_ctlr_vn_probe_reply(struct fcoe_ctlr *fip, case FIP_ST_VNMP_PROBE1: case FIP_ST_VNMP_PROBE2: case FIP_ST_VNMP_CLAIM: + LIBFCOE_FIP_DBG(fip, "vn_probe_reply: restart state %x\n", + fip->state); fcoe_ctlr_vn_restart(fip); break; case FIP_ST_VNMP_UP: + LIBFCOE_FIP_DBG(fip, "vn_probe_reply: send claim notify\n"); fcoe_ctlr_vn_send_claim(fip); break; default: @@ -2467,26 +2515,33 @@ static void fcoe_ctlr_vn_add(struct fcoe_ctlr *fip, struct fc_rport_priv *new) return; mutex_lock(&lport->disc.disc_mutex); - rdata = lport->tt.rport_create(lport, port_id); + rdata = fc_rport_create(lport, port_id); if (!rdata) { mutex_unlock(&lport->disc.disc_mutex); return; } + mutex_lock(&rdata->rp_mutex); + mutex_unlock(&lport->disc.disc_mutex); rdata->ops = &fcoe_ctlr_vn_rport_ops; rdata->disc_id = lport->disc.disc_id; ids = &rdata->ids; if ((ids->port_name != -1 && ids->port_name != new->ids.port_name) || - (ids->node_name != -1 && ids->node_name != new->ids.node_name)) - lport->tt.rport_logoff(rdata); + (ids->node_name != -1 && ids->node_name != new->ids.node_name)) { + mutex_unlock(&rdata->rp_mutex); + LIBFCOE_FIP_DBG(fip, "vn_add rport logoff %6.6x\n", port_id); + fc_rport_logoff(rdata); + mutex_lock(&rdata->rp_mutex); + } ids->port_name = new->ids.port_name; ids->node_name = new->ids.node_name; - mutex_unlock(&lport->disc.disc_mutex); + mutex_unlock(&rdata->rp_mutex); frport = fcoe_ctlr_rport(rdata); - LIBFCOE_FIP_DBG(fip, "vn_add rport %6.6x %s\n", - port_id, frport->fcoe_len ? "old" : "new"); + LIBFCOE_FIP_DBG(fip, "vn_add rport %6.6x %s state %d\n", + port_id, frport->fcoe_len ? "old" : "new", + rdata->rp_state); *frport = *fcoe_ctlr_rport(new); frport->time = 0; } @@ -2506,12 +2561,12 @@ static int fcoe_ctlr_vn_lookup(struct fcoe_ctlr *fip, u32 port_id, u8 *mac) struct fcoe_rport *frport; int ret = -1; - rdata = lport->tt.rport_lookup(lport, port_id); + rdata = fc_rport_lookup(lport, port_id); if (rdata) { frport = fcoe_ctlr_rport(rdata); memcpy(mac, frport->enode_mac, ETH_ALEN); ret = 0; - kref_put(&rdata->kref, lport->tt.rport_destroy); + kref_put(&rdata->kref, fc_rport_destroy); } return ret; } @@ -2529,6 +2584,7 @@ static void fcoe_ctlr_vn_claim_notify(struct fcoe_ctlr *fip, struct fcoe_rport *frport = fcoe_ctlr_rport(new); if (frport->flags & FIP_FL_REC_OR_P2P) { + LIBFCOE_FIP_DBG(fip, "send probe req for P2P/REC\n"); fcoe_ctlr_vn_send(fip, FIP_SC_VN_PROBE_REQ, fcoe_all_vn2vn, 0); return; } @@ -2536,25 +2592,37 @@ static void fcoe_ctlr_vn_claim_notify(struct fcoe_ctlr *fip, case FIP_ST_VNMP_START: case FIP_ST_VNMP_PROBE1: case FIP_ST_VNMP_PROBE2: - if (new->ids.port_id == fip->port_id) + if (new->ids.port_id == fip->port_id) { + LIBFCOE_FIP_DBG(fip, "vn_claim_notify: " + "restart, state %d\n", + fip->state); fcoe_ctlr_vn_restart(fip); + } break; case FIP_ST_VNMP_CLAIM: case FIP_ST_VNMP_UP: if (new->ids.port_id == fip->port_id) { if (new->ids.port_name > fip->lp->wwpn) { + LIBFCOE_FIP_DBG(fip, "vn_claim_notify: " + "restart, port_id collision\n"); fcoe_ctlr_vn_restart(fip); break; } + LIBFCOE_FIP_DBG(fip, "vn_claim_notify: " + "send claim notify\n"); fcoe_ctlr_vn_send_claim(fip); break; } + LIBFCOE_FIP_DBG(fip, "vn_claim_notify: send reply to %x\n", + new->ids.port_id); fcoe_ctlr_vn_send(fip, FIP_SC_VN_CLAIM_REP, frport->enode_mac, min((u32)frport->fcoe_len, fcoe_ctlr_fcoe_size(fip))); fcoe_ctlr_vn_add(fip, new); break; default: + LIBFCOE_FIP_DBG(fip, "vn_claim_notify: " + "ignoring claim from %x\n", new->ids.port_id); break; } } @@ -2591,19 +2659,26 @@ static void fcoe_ctlr_vn_beacon(struct fcoe_ctlr *fip, frport = fcoe_ctlr_rport(new); if (frport->flags & FIP_FL_REC_OR_P2P) { + LIBFCOE_FIP_DBG(fip, "p2p beacon while in vn2vn mode\n"); fcoe_ctlr_vn_send(fip, FIP_SC_VN_PROBE_REQ, fcoe_all_vn2vn, 0); return; } - rdata = lport->tt.rport_lookup(lport, new->ids.port_id); + rdata = fc_rport_lookup(lport, new->ids.port_id); if (rdata) { if (rdata->ids.node_name == new->ids.node_name && rdata->ids.port_name == new->ids.port_name) { frport = fcoe_ctlr_rport(rdata); - if (!frport->time && fip->state == FIP_ST_VNMP_UP) - lport->tt.rport_login(rdata); + LIBFCOE_FIP_DBG(fip, "beacon from rport %x\n", + rdata->ids.port_id); + if (!frport->time && fip->state == FIP_ST_VNMP_UP) { + LIBFCOE_FIP_DBG(fip, "beacon expired " + "for rport %x\n", + rdata->ids.port_id); + fc_rport_login(rdata); + } frport->time = jiffies; } - kref_put(&rdata->kref, lport->tt.rport_destroy); + kref_put(&rdata->kref, fc_rport_destroy); return; } if (fip->state != FIP_ST_VNMP_UP) @@ -2638,11 +2713,15 @@ static unsigned long fcoe_ctlr_vn_age(struct fcoe_ctlr *fip) unsigned long deadline; next_time = jiffies + msecs_to_jiffies(FIP_VN_BEACON_INT * 10); - mutex_lock(&lport->disc.disc_mutex); + rcu_read_lock(); list_for_each_entry_rcu(rdata, &lport->disc.rports, peers) { + if (!kref_get_unless_zero(&rdata->kref)) + continue; frport = fcoe_ctlr_rport(rdata); - if (!frport->time) + if (!frport->time) { + kref_put(&rdata->kref, fc_rport_destroy); continue; + } deadline = frport->time + msecs_to_jiffies(FIP_VN_BEACON_INT * 25 / 10); if (time_after_eq(jiffies, deadline)) { @@ -2650,11 +2729,12 @@ static unsigned long fcoe_ctlr_vn_age(struct fcoe_ctlr *fip) LIBFCOE_FIP_DBG(fip, "port %16.16llx fc_id %6.6x beacon expired\n", rdata->ids.port_name, rdata->ids.port_id); - lport->tt.rport_logoff(rdata); + fc_rport_logoff(rdata); } else if (time_before(deadline, next_time)) next_time = deadline; + kref_put(&rdata->kref, fc_rport_destroy); } - mutex_unlock(&lport->disc.disc_mutex); + rcu_read_unlock(); return next_time; } @@ -2674,11 +2754,21 @@ static int fcoe_ctlr_vn_recv(struct fcoe_ctlr *fip, struct sk_buff *skb) struct fc_rport_priv rdata; struct fcoe_rport frport; } buf; - int rc; + int rc, vlan_id = 0; fiph = (struct fip_header *)skb->data; sub = fiph->fip_subcode; + if (fip->lp->vlan) + vlan_id = skb_vlan_tag_get_id(skb); + + if (vlan_id && vlan_id != fip->lp->vlan) { + LIBFCOE_FIP_DBG(fip, "vn_recv drop frame sub %x vlan %d\n", + sub, vlan_id); + rc = -EAGAIN; + goto drop; + } + rc = fcoe_ctlr_vn_parse(fip, skb, &buf.rdata); if (rc) { LIBFCOE_FIP_DBG(fip, "vn_recv vn_parse error %d\n", rc); @@ -2941,7 +3031,7 @@ static void fcoe_ctlr_disc_recv(struct fc_lport *lport, struct fc_frame *fp) rjt_data.reason = ELS_RJT_UNSUP; rjt_data.explan = ELS_EXPL_NONE; - lport->tt.seq_els_rsp_send(fp, ELS_LS_RJT, &rjt_data); + fc_seq_els_rsp_send(fp, ELS_LS_RJT, &rjt_data); fc_frame_free(fp); } @@ -2991,12 +3081,17 @@ static void fcoe_ctlr_vn_disc(struct fcoe_ctlr *fip) mutex_lock(&disc->disc_mutex); callback = disc->pending ? disc->disc_callback : NULL; disc->pending = 0; + mutex_unlock(&disc->disc_mutex); + rcu_read_lock(); list_for_each_entry_rcu(rdata, &disc->rports, peers) { + if (!kref_get_unless_zero(&rdata->kref)) + continue; frport = fcoe_ctlr_rport(rdata); if (frport->time) - lport->tt.rport_login(rdata); + fc_rport_login(rdata); + kref_put(&rdata->kref, fc_rport_destroy); } - mutex_unlock(&disc->disc_mutex); + rcu_read_unlock(); if (callback) callback(lport, DISC_EV_SUCCESS); } @@ -3015,11 +3110,13 @@ static void fcoe_ctlr_vn_timeout(struct fcoe_ctlr *fip) switch (fip->state) { case FIP_ST_VNMP_START: fcoe_ctlr_set_state(fip, FIP_ST_VNMP_PROBE1); + LIBFCOE_FIP_DBG(fip, "vn_timeout: send 1st probe request\n"); fcoe_ctlr_vn_send(fip, FIP_SC_VN_PROBE_REQ, fcoe_all_vn2vn, 0); next_time = jiffies + msecs_to_jiffies(FIP_VN_PROBE_WAIT); break; case FIP_ST_VNMP_PROBE1: fcoe_ctlr_set_state(fip, FIP_ST_VNMP_PROBE2); + LIBFCOE_FIP_DBG(fip, "vn_timeout: send 2nd probe request\n"); fcoe_ctlr_vn_send(fip, FIP_SC_VN_PROBE_REQ, fcoe_all_vn2vn, 0); next_time = jiffies + msecs_to_jiffies(FIP_VN_ANN_WAIT); break; @@ -3030,6 +3127,7 @@ static void fcoe_ctlr_vn_timeout(struct fcoe_ctlr *fip) hton24(mac + 3, new_port_id); fcoe_ctlr_map_dest(fip); fip->update_mac(fip->lp, mac); + LIBFCOE_FIP_DBG(fip, "vn_timeout: send claim notify\n"); fcoe_ctlr_vn_send_claim(fip); next_time = jiffies + msecs_to_jiffies(FIP_VN_ANN_WAIT); break; @@ -3041,6 +3139,7 @@ static void fcoe_ctlr_vn_timeout(struct fcoe_ctlr *fip) next_time = fip->sol_time + msecs_to_jiffies(FIP_VN_ANN_WAIT); if (time_after_eq(jiffies, next_time)) { fcoe_ctlr_set_state(fip, FIP_ST_VNMP_UP); + LIBFCOE_FIP_DBG(fip, "vn_timeout: send vn2vn beacon\n"); fcoe_ctlr_vn_send(fip, FIP_SC_VN_BEACON, fcoe_all_vn2vn, 0); next_time = jiffies + msecs_to_jiffies(FIP_VN_ANN_WAIT); @@ -3051,6 +3150,7 @@ static void fcoe_ctlr_vn_timeout(struct fcoe_ctlr *fip) case FIP_ST_VNMP_UP: next_time = fcoe_ctlr_vn_age(fip); if (time_after_eq(jiffies, fip->port_ka_time)) { + LIBFCOE_FIP_DBG(fip, "vn_timeout: send vn2vn beacon\n"); fcoe_ctlr_vn_send(fip, FIP_SC_VN_BEACON, fcoe_all_vn2vn, 0); fip->port_ka_time = jiffies + @@ -3135,7 +3235,6 @@ int fcoe_libfc_config(struct fc_lport *lport, struct fcoe_ctlr *fip, fc_exch_init(lport); fc_elsct_init(lport); fc_lport_init(lport); - fc_rport_init(lport); fc_disc_init(lport); fcoe_ctlr_mode_set(lport, fip, fip->mode); return 0; diff --git a/drivers/scsi/fcoe/fcoe_sysfs.c b/drivers/scsi/fcoe/fcoe_sysfs.c index 0675fd1..9cf3d56 100644 --- a/drivers/scsi/fcoe/fcoe_sysfs.c +++ b/drivers/scsi/fcoe/fcoe_sysfs.c @@ -335,16 +335,24 @@ static ssize_t store_ctlr_enabled(struct device *dev, const char *buf, size_t count) { struct fcoe_ctlr_device *ctlr = dev_to_ctlr(dev); + bool enabled; int rc; + if (*buf == '1') + enabled = true; + else if (*buf == '0') + enabled = false; + else + return -EINVAL; + switch (ctlr->enabled) { case FCOE_CTLR_ENABLED: - if (*buf == '1') + if (enabled) return count; ctlr->enabled = FCOE_CTLR_DISABLED; break; case FCOE_CTLR_DISABLED: - if (*buf == '0') + if (!enabled) return count; ctlr->enabled = FCOE_CTLR_ENABLED; break; @@ -424,6 +432,75 @@ static FCOE_DEVICE_ATTR(ctlr, fip_vlan_responder, S_IRUGO | S_IWUSR, store_ctlr_fip_resp); static ssize_t +fcoe_ctlr_var_store(u32 *var, const char *buf, size_t count) +{ + int err; + unsigned long v; + + err = kstrtoul(buf, 10, &v); + if (err || v > UINT_MAX) + return -EINVAL; + + *var = v; + + return count; +} + +static ssize_t store_ctlr_r_a_tov(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t count) +{ + struct fcoe_ctlr_device *ctlr_dev = dev_to_ctlr(dev); + struct fcoe_ctlr *ctlr = fcoe_ctlr_device_priv(ctlr_dev); + + if (ctlr_dev->enabled == FCOE_CTLR_ENABLED) + return -EBUSY; + if (ctlr_dev->enabled == FCOE_CTLR_DISABLED) + return fcoe_ctlr_var_store(&ctlr->lp->r_a_tov, buf, count); + return -ENOTSUPP; +} + +static ssize_t show_ctlr_r_a_tov(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + struct fcoe_ctlr_device *ctlr_dev = dev_to_ctlr(dev); + struct fcoe_ctlr *ctlr = fcoe_ctlr_device_priv(ctlr_dev); + + return sprintf(buf, "%d\n", ctlr->lp->r_a_tov); +} + +static FCOE_DEVICE_ATTR(ctlr, r_a_tov, S_IRUGO | S_IWUSR, + show_ctlr_r_a_tov, store_ctlr_r_a_tov); + +static ssize_t store_ctlr_e_d_tov(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t count) +{ + struct fcoe_ctlr_device *ctlr_dev = dev_to_ctlr(dev); + struct fcoe_ctlr *ctlr = fcoe_ctlr_device_priv(ctlr_dev); + + if (ctlr_dev->enabled == FCOE_CTLR_ENABLED) + return -EBUSY; + if (ctlr_dev->enabled == FCOE_CTLR_DISABLED) + return fcoe_ctlr_var_store(&ctlr->lp->e_d_tov, buf, count); + return -ENOTSUPP; +} + +static ssize_t show_ctlr_e_d_tov(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + struct fcoe_ctlr_device *ctlr_dev = dev_to_ctlr(dev); + struct fcoe_ctlr *ctlr = fcoe_ctlr_device_priv(ctlr_dev); + + return sprintf(buf, "%d\n", ctlr->lp->e_d_tov); +} + +static FCOE_DEVICE_ATTR(ctlr, e_d_tov, S_IRUGO | S_IWUSR, + show_ctlr_e_d_tov, store_ctlr_e_d_tov); + +static ssize_t store_private_fcoe_ctlr_fcf_dev_loss_tmo(struct device *dev, struct device_attribute *attr, const char *buf, size_t count) @@ -507,6 +584,8 @@ static struct attribute_group fcoe_ctlr_lesb_attr_group = { static struct attribute *fcoe_ctlr_attrs[] = { &device_attr_fcoe_ctlr_fip_vlan_responder.attr, &device_attr_fcoe_ctlr_fcf_dev_loss_tmo.attr, + &device_attr_fcoe_ctlr_r_a_tov.attr, + &device_attr_fcoe_ctlr_e_d_tov.attr, &device_attr_fcoe_ctlr_enabled.attr, &device_attr_fcoe_ctlr_mode.attr, NULL, diff --git a/drivers/scsi/fnic/fnic_scsi.c b/drivers/scsi/fnic/fnic_scsi.c index d9fd2f8..2544a37 100644 --- a/drivers/scsi/fnic/fnic_scsi.c +++ b/drivers/scsi/fnic/fnic_scsi.c @@ -441,30 +441,38 @@ static int fnic_queuecommand_lck(struct scsi_cmnd *sc, void (*done)(struct scsi_ unsigned long ptr; spinlock_t *io_lock = NULL; int io_lock_acquired = 0; + struct fc_rport_libfc_priv *rp; if (unlikely(fnic_chk_state_flags_locked(fnic, FNIC_FLAGS_IO_BLOCKED))) return SCSI_MLQUEUE_HOST_BUSY; rport = starget_to_rport(scsi_target(sc->device)); + if (!rport) { + FNIC_SCSI_DBG(KERN_DEBUG, fnic->lport->host, + "returning DID_NO_CONNECT for IO as rport is NULL\n"); + sc->result = DID_NO_CONNECT << 16; + done(sc); + return 0; + } + ret = fc_remote_port_chkready(rport); if (ret) { + FNIC_SCSI_DBG(KERN_DEBUG, fnic->lport->host, + "rport is not ready\n"); atomic64_inc(&fnic_stats->misc_stats.rport_not_ready); sc->result = ret; done(sc); return 0; } - if (rport) { - struct fc_rport_libfc_priv *rp = rport->dd_data; - - if (!rp || rp->rp_state != RPORT_ST_READY) { - FNIC_SCSI_DBG(KERN_DEBUG, fnic->lport->host, + rp = rport->dd_data; + if (!rp || rp->rp_state != RPORT_ST_READY) { + FNIC_SCSI_DBG(KERN_DEBUG, fnic->lport->host, "returning DID_NO_CONNECT for IO as rport is removed\n"); - atomic64_inc(&fnic_stats->misc_stats.rport_not_ready); - sc->result = DID_NO_CONNECT<<16; - done(sc); - return 0; - } + atomic64_inc(&fnic_stats->misc_stats.rport_not_ready); + sc->result = DID_NO_CONNECT<<16; + done(sc); + return 0; } if (lp->state != LPORT_ST_READY || !(lp->link_up)) @@ -2543,7 +2551,7 @@ int fnic_reset(struct Scsi_Host *shost) * Reset local port, this will clean up libFC exchanges, * reset remote port sessions, and if link is up, begin flogi */ - ret = lp->tt.lport_reset(lp); + ret = fc_lport_reset(lp); FNIC_SCSI_DBG(KERN_DEBUG, fnic->lport->host, "Returning from fnic reset %s\n", diff --git a/drivers/scsi/fnic/fnic_trace.c b/drivers/scsi/fnic/fnic_trace.c index 4e15c4b..5a5fa01 100644 --- a/drivers/scsi/fnic/fnic_trace.c +++ b/drivers/scsi/fnic/fnic_trace.c @@ -613,7 +613,7 @@ int fnic_fc_trace_set_data(u32 host_no, u8 frame_type, fc_trace_entries.rd_idx = 0; } - fc_buf->time_stamp = CURRENT_TIME; + ktime_get_real_ts64(&fc_buf->time_stamp); fc_buf->host_no = host_no; fc_buf->frame_type = frame_type; @@ -740,7 +740,7 @@ void copy_and_format_trace_data(struct fc_trace_hdr *tdata, len = *orig_len; - time_to_tm(tdata->time_stamp.tv_sec, 0, &tm); + time64_to_tm(tdata->time_stamp.tv_sec, 0, &tm); fmt = "%02d:%02d:%04ld %02d:%02d:%02d.%09lu ns%8x %c%8x\t"; len += snprintf(fnic_dbgfs_prt->buffer + len, diff --git a/drivers/scsi/fnic/fnic_trace.h b/drivers/scsi/fnic/fnic_trace.h index a8aa057..e375d0c 100644 --- a/drivers/scsi/fnic/fnic_trace.h +++ b/drivers/scsi/fnic/fnic_trace.h @@ -72,7 +72,7 @@ struct fnic_trace_data { typedef struct fnic_trace_data fnic_trace_data_t; struct fc_trace_hdr { - struct timespec time_stamp; + struct timespec64 time_stamp; u32 host_no; u8 frame_type; u8 frame_len; diff --git a/drivers/scsi/fnic/vnic_dev.c b/drivers/scsi/fnic/vnic_dev.c index 9795d6f..ba69d61 100644 --- a/drivers/scsi/fnic/vnic_dev.c +++ b/drivers/scsi/fnic/vnic_dev.c @@ -499,10 +499,7 @@ void vnic_dev_add_addr(struct vnic_dev *vdev, u8 *addr) err = vnic_dev_cmd(vdev, CMD_ADDR_ADD, &a0, &a1, wait); if (err) - printk(KERN_ERR - "Can't add addr [%02x:%02x:%02x:%02x:%02x:%02x], %d\n", - addr[0], addr[1], addr[2], addr[3], addr[4], addr[5], - err); + pr_err("Can't add addr [%pM], %d\n", addr, err); } void vnic_dev_del_addr(struct vnic_dev *vdev, u8 *addr) @@ -517,10 +514,7 @@ void vnic_dev_del_addr(struct vnic_dev *vdev, u8 *addr) err = vnic_dev_cmd(vdev, CMD_ADDR_DEL, &a0, &a1, wait); if (err) - printk(KERN_ERR - "Can't del addr [%02x:%02x:%02x:%02x:%02x:%02x], %d\n", - addr[0], addr[1], addr[2], addr[3], addr[4], addr[5], - err); + pr_err("Can't del addr [%pM], %d\n", addr, err); } int vnic_dev_notify_set(struct vnic_dev *vdev, u16 intr) diff --git a/drivers/scsi/g_NCR5380.c b/drivers/scsi/g_NCR5380.c index cbf0103..de5147a 100644 --- a/drivers/scsi/g_NCR5380.c +++ b/drivers/scsi/g_NCR5380.c @@ -64,9 +64,9 @@ static int card[] = { -1, -1, -1, -1, -1, -1, -1, -1 }; module_param_array(card, int, NULL, 0); MODULE_PARM_DESC(card, "card type (0=NCR5380, 1=NCR53C400, 2=NCR53C400A, 3=DTC3181E, 4=HP C2502)"); +MODULE_ALIAS("g_NCR5380_mmio"); MODULE_LICENSE("GPL"); -#ifndef SCSI_G_NCR5380_MEM /* * Configure I/O address of 53C400A or DTC436 by writing magic numbers * to ports 0x779 and 0x379. @@ -88,40 +88,35 @@ static void magic_configure(int idx, u8 irq, u8 magic[]) cfg = 0x80 | idx | (irq << 4); outb(cfg, 0x379); } -#endif + +static unsigned int ncr_53c400a_ports[] = { + 0x280, 0x290, 0x300, 0x310, 0x330, 0x340, 0x348, 0x350, 0 +}; +static unsigned int dtc_3181e_ports[] = { + 0x220, 0x240, 0x280, 0x2a0, 0x2c0, 0x300, 0x320, 0x340, 0 +}; +static u8 ncr_53c400a_magic[] = { /* 53C400A & DTC436 */ + 0x59, 0xb9, 0xc5, 0xae, 0xa6 +}; +static u8 hp_c2502_magic[] = { /* HP C2502 */ + 0x0f, 0x22, 0xf0, 0x20, 0x80 +}; static int generic_NCR5380_init_one(struct scsi_host_template *tpnt, struct device *pdev, int base, int irq, int board) { - unsigned int *ports; + bool is_pmio = base <= 0xffff; + int ret; + int flags = 0; + unsigned int *ports = NULL; u8 *magic = NULL; -#ifndef SCSI_G_NCR5380_MEM int i; int port_idx = -1; unsigned long region_size; -#endif - static unsigned int ncr_53c400a_ports[] = { - 0x280, 0x290, 0x300, 0x310, 0x330, 0x340, 0x348, 0x350, 0 - }; - static unsigned int dtc_3181e_ports[] = { - 0x220, 0x240, 0x280, 0x2a0, 0x2c0, 0x300, 0x320, 0x340, 0 - }; - static u8 ncr_53c400a_magic[] = { /* 53C400A & DTC436 */ - 0x59, 0xb9, 0xc5, 0xae, 0xa6 - }; - static u8 hp_c2502_magic[] = { /* HP C2502 */ - 0x0f, 0x22, 0xf0, 0x20, 0x80 - }; - int flags, ret; struct Scsi_Host *instance; struct NCR5380_hostdata *hostdata; -#ifdef SCSI_G_NCR5380_MEM - void __iomem *iomem; - resource_size_t iomem_size; -#endif + u8 __iomem *iomem; - ports = NULL; - flags = 0; switch (board) { case BOARD_NCR5380: flags = FLAG_NO_PSEUDO_DMA | FLAG_DMA_FIXUP; @@ -140,8 +135,7 @@ static int generic_NCR5380_init_one(struct scsi_host_template *tpnt, break; } -#ifndef SCSI_G_NCR5380_MEM - if (ports && magic) { + if (is_pmio && ports && magic) { /* wakeup sequence for the NCR53C400A and DTC3181E */ /* Disable the adapter and look for a free io port */ @@ -170,84 +164,89 @@ static int generic_NCR5380_init_one(struct scsi_host_template *tpnt, if (ports[i]) { /* At this point we have our region reserved */ magic_configure(i, 0, magic); /* no IRQ yet */ - outb(0xc0, ports[i] + 9); - if (inb(ports[i] + 9) != 0x80) { + base = ports[i]; + outb(0xc0, base + 9); + if (inb(base + 9) != 0x80) { ret = -ENODEV; goto out_release; } - base = ports[i]; port_idx = i; } else return -EINVAL; - } - else - { + } else if (is_pmio) { /* NCR5380 - no configuration, just grab */ region_size = 8; if (!base || !request_region(base, region_size, "ncr5380")) return -EBUSY; + } else { /* MMIO */ + region_size = NCR53C400_region_size; + if (!request_mem_region(base, region_size, "ncr5380")) + return -EBUSY; } -#else - iomem_size = NCR53C400_region_size; - if (!request_mem_region(base, iomem_size, "ncr5380")) - return -EBUSY; - iomem = ioremap(base, iomem_size); + + if (is_pmio) + iomem = ioport_map(base, region_size); + else + iomem = ioremap(base, region_size); + if (!iomem) { - release_mem_region(base, iomem_size); - return -ENOMEM; + ret = -ENOMEM; + goto out_release; } -#endif + instance = scsi_host_alloc(tpnt, sizeof(struct NCR5380_hostdata)); if (instance == NULL) { ret = -ENOMEM; - goto out_release; + goto out_unmap; } hostdata = shost_priv(instance); -#ifndef SCSI_G_NCR5380_MEM - instance->io_port = base; - instance->n_io_port = region_size; - hostdata->io_width = 1; /* 8-bit PDMA by default */ - - /* - * On NCR53C400 boards, NCR5380 registers are mapped 8 past - * the base address. - */ - switch (board) { - case BOARD_NCR53C400: - instance->io_port += 8; - hostdata->c400_ctl_status = 0; - hostdata->c400_blk_cnt = 1; - hostdata->c400_host_buf = 4; - break; - case BOARD_DTC3181E: - hostdata->io_width = 2; /* 16-bit PDMA */ - /* fall through */ - case BOARD_NCR53C400A: - case BOARD_HP_C2502: - hostdata->c400_ctl_status = 9; - hostdata->c400_blk_cnt = 10; - hostdata->c400_host_buf = 8; - break; - } -#else - instance->base = base; - hostdata->iomem = iomem; - hostdata->iomem_size = iomem_size; - switch (board) { - case BOARD_NCR53C400: - hostdata->c400_ctl_status = 0x100; - hostdata->c400_blk_cnt = 0x101; - hostdata->c400_host_buf = 0x104; - break; - case BOARD_DTC3181E: - case BOARD_NCR53C400A: - case BOARD_HP_C2502: - pr_err(DRV_MODULE_NAME ": unknown register offsets\n"); - ret = -EINVAL; - goto out_unregister; + hostdata->io = iomem; + hostdata->region_size = region_size; + + if (is_pmio) { + hostdata->io_port = base; + hostdata->io_width = 1; /* 8-bit PDMA by default */ + hostdata->offset = 0; + + /* + * On NCR53C400 boards, NCR5380 registers are mapped 8 past + * the base address. + */ + switch (board) { + case BOARD_NCR53C400: + hostdata->io_port += 8; + hostdata->c400_ctl_status = 0; + hostdata->c400_blk_cnt = 1; + hostdata->c400_host_buf = 4; + break; + case BOARD_DTC3181E: + hostdata->io_width = 2; /* 16-bit PDMA */ + /* fall through */ + case BOARD_NCR53C400A: + case BOARD_HP_C2502: + hostdata->c400_ctl_status = 9; + hostdata->c400_blk_cnt = 10; + hostdata->c400_host_buf = 8; + break; + } + } else { + hostdata->base = base; + hostdata->offset = NCR53C400_mem_base; + switch (board) { + case BOARD_NCR53C400: + hostdata->c400_ctl_status = 0x100; + hostdata->c400_blk_cnt = 0x101; + hostdata->c400_host_buf = 0x104; + break; + case BOARD_DTC3181E: + case BOARD_NCR53C400A: + case BOARD_HP_C2502: + pr_err(DRV_MODULE_NAME ": unknown register offsets\n"); + ret = -EINVAL; + goto out_unregister; + } } -#endif ret = NCR5380_init(instance, flags | FLAG_LATE_DMA_SETUP); if (ret) @@ -273,11 +272,9 @@ static int generic_NCR5380_init_one(struct scsi_host_template *tpnt, instance->irq = NO_IRQ; if (instance->irq != NO_IRQ) { -#ifndef SCSI_G_NCR5380_MEM /* set IRQ for HP C2502 */ if (board == BOARD_HP_C2502) magic_configure(port_idx, instance->irq, magic); -#endif if (request_irq(instance->irq, generic_NCR5380_intr, 0, "NCR5380", instance)) { printk(KERN_WARNING "scsi%d : IRQ%d not free, interrupts disabled\n", instance->host_no, instance->irq); @@ -303,38 +300,39 @@ out_free_irq: NCR5380_exit(instance); out_unregister: scsi_host_put(instance); -out_release: -#ifndef SCSI_G_NCR5380_MEM - release_region(base, region_size); -#else +out_unmap: iounmap(iomem); - release_mem_region(base, iomem_size); -#endif +out_release: + if (is_pmio) + release_region(base, region_size); + else + release_mem_region(base, region_size); return ret; } static void generic_NCR5380_release_resources(struct Scsi_Host *instance) { + struct NCR5380_hostdata *hostdata = shost_priv(instance); + void __iomem *iomem = hostdata->io; + unsigned long io_port = hostdata->io_port; + unsigned long base = hostdata->base; + unsigned long region_size = hostdata->region_size; + scsi_remove_host(instance); if (instance->irq != NO_IRQ) free_irq(instance->irq, instance); NCR5380_exit(instance); -#ifndef SCSI_G_NCR5380_MEM - release_region(instance->io_port, instance->n_io_port); -#else - { - struct NCR5380_hostdata *hostdata = shost_priv(instance); - - iounmap(hostdata->iomem); - release_mem_region(instance->base, hostdata->iomem_size); - } -#endif scsi_host_put(instance); + iounmap(iomem); + if (io_port) + release_region(io_port, region_size); + else + release_mem_region(base, region_size); } /** * generic_NCR5380_pread - pseudo DMA read - * @instance: adapter to read from + * @hostdata: scsi host private data * @dst: buffer to read into * @len: buffer length * @@ -342,10 +340,9 @@ static void generic_NCR5380_release_resources(struct Scsi_Host *instance) * controller */ -static inline int generic_NCR5380_pread(struct Scsi_Host *instance, +static inline int generic_NCR5380_pread(struct NCR5380_hostdata *hostdata, unsigned char *dst, int len) { - struct NCR5380_hostdata *hostdata = shost_priv(instance); int blocks = len / 128; int start = 0; @@ -361,18 +358,16 @@ static inline int generic_NCR5380_pread(struct Scsi_Host *instance, while (NCR5380_read(hostdata->c400_ctl_status) & CSR_HOST_BUF_NOT_RDY) ; /* FIXME - no timeout */ -#ifndef SCSI_G_NCR5380_MEM - if (hostdata->io_width == 2) - insw(instance->io_port + hostdata->c400_host_buf, + if (hostdata->io_port && hostdata->io_width == 2) + insw(hostdata->io_port + hostdata->c400_host_buf, dst + start, 64); - else - insb(instance->io_port + hostdata->c400_host_buf, + else if (hostdata->io_port) + insb(hostdata->io_port + hostdata->c400_host_buf, dst + start, 128); -#else - /* implies SCSI_G_NCR5380_MEM */ - memcpy_fromio(dst + start, - hostdata->iomem + NCR53C400_host_buffer, 128); -#endif + else + memcpy_fromio(dst + start, + hostdata->io + NCR53C400_host_buffer, 128); + start += 128; blocks--; } @@ -381,18 +376,16 @@ static inline int generic_NCR5380_pread(struct Scsi_Host *instance, while (NCR5380_read(hostdata->c400_ctl_status) & CSR_HOST_BUF_NOT_RDY) ; /* FIXME - no timeout */ -#ifndef SCSI_G_NCR5380_MEM - if (hostdata->io_width == 2) - insw(instance->io_port + hostdata->c400_host_buf, + if (hostdata->io_port && hostdata->io_width == 2) + insw(hostdata->io_port + hostdata->c400_host_buf, dst + start, 64); - else - insb(instance->io_port + hostdata->c400_host_buf, + else if (hostdata->io_port) + insb(hostdata->io_port + hostdata->c400_host_buf, dst + start, 128); -#else - /* implies SCSI_G_NCR5380_MEM */ - memcpy_fromio(dst + start, - hostdata->iomem + NCR53C400_host_buffer, 128); -#endif + else + memcpy_fromio(dst + start, + hostdata->io + NCR53C400_host_buffer, 128); + start += 128; blocks--; } @@ -412,7 +405,7 @@ static inline int generic_NCR5380_pread(struct Scsi_Host *instance, /** * generic_NCR5380_pwrite - pseudo DMA write - * @instance: adapter to read from + * @hostdata: scsi host private data * @dst: buffer to read into * @len: buffer length * @@ -420,10 +413,9 @@ static inline int generic_NCR5380_pread(struct Scsi_Host *instance, * controller */ -static inline int generic_NCR5380_pwrite(struct Scsi_Host *instance, +static inline int generic_NCR5380_pwrite(struct NCR5380_hostdata *hostdata, unsigned char *src, int len) { - struct NCR5380_hostdata *hostdata = shost_priv(instance); int blocks = len / 128; int start = 0; @@ -439,18 +431,17 @@ static inline int generic_NCR5380_pwrite(struct Scsi_Host *instance, break; while (NCR5380_read(hostdata->c400_ctl_status) & CSR_HOST_BUF_NOT_RDY) ; // FIXME - timeout -#ifndef SCSI_G_NCR5380_MEM - if (hostdata->io_width == 2) - outsw(instance->io_port + hostdata->c400_host_buf, + + if (hostdata->io_port && hostdata->io_width == 2) + outsw(hostdata->io_port + hostdata->c400_host_buf, src + start, 64); - else - outsb(instance->io_port + hostdata->c400_host_buf, + else if (hostdata->io_port) + outsb(hostdata->io_port + hostdata->c400_host_buf, src + start, 128); -#else - /* implies SCSI_G_NCR5380_MEM */ - memcpy_toio(hostdata->iomem + NCR53C400_host_buffer, - src + start, 128); -#endif + else + memcpy_toio(hostdata->io + NCR53C400_host_buffer, + src + start, 128); + start += 128; blocks--; } @@ -458,18 +449,16 @@ static inline int generic_NCR5380_pwrite(struct Scsi_Host *instance, while (NCR5380_read(hostdata->c400_ctl_status) & CSR_HOST_BUF_NOT_RDY) ; // FIXME - no timeout -#ifndef SCSI_G_NCR5380_MEM - if (hostdata->io_width == 2) - outsw(instance->io_port + hostdata->c400_host_buf, + if (hostdata->io_port && hostdata->io_width == 2) + outsw(hostdata->io_port + hostdata->c400_host_buf, src + start, 64); - else - outsb(instance->io_port + hostdata->c400_host_buf, + else if (hostdata->io_port) + outsb(hostdata->io_port + hostdata->c400_host_buf, src + start, 128); -#else - /* implies SCSI_G_NCR5380_MEM */ - memcpy_toio(hostdata->iomem + NCR53C400_host_buffer, - src + start, 128); -#endif + else + memcpy_toio(hostdata->io + NCR53C400_host_buffer, + src + start, 128); + start += 128; blocks--; } @@ -489,10 +478,9 @@ static inline int generic_NCR5380_pwrite(struct Scsi_Host *instance, return 0; } -static int generic_NCR5380_dma_xfer_len(struct Scsi_Host *instance, +static int generic_NCR5380_dma_xfer_len(struct NCR5380_hostdata *hostdata, struct scsi_cmnd *cmd) { - struct NCR5380_hostdata *hostdata = shost_priv(instance); int transfersize = cmd->transfersize; if (hostdata->flags & FLAG_NO_PSEUDO_DMA) @@ -566,7 +554,7 @@ static struct isa_driver generic_NCR5380_isa_driver = { }, }; -#if !defined(SCSI_G_NCR5380_MEM) && defined(CONFIG_PNP) +#ifdef CONFIG_PNP static struct pnp_device_id generic_NCR5380_pnp_ids[] = { { .id = "DTC436e", .driver_data = BOARD_DTC3181E }, { .id = "" } @@ -600,7 +588,7 @@ static struct pnp_driver generic_NCR5380_pnp_driver = { .probe = generic_NCR5380_pnp_probe, .remove = generic_NCR5380_pnp_remove, }; -#endif /* !defined(SCSI_G_NCR5380_MEM) && defined(CONFIG_PNP) */ +#endif /* defined(CONFIG_PNP) */ static int pnp_registered, isa_registered; @@ -624,7 +612,7 @@ static int __init generic_NCR5380_init(void) card[0] = BOARD_HP_C2502; } -#if !defined(SCSI_G_NCR5380_MEM) && defined(CONFIG_PNP) +#ifdef CONFIG_PNP if (!pnp_register_driver(&generic_NCR5380_pnp_driver)) pnp_registered = 1; #endif @@ -637,7 +625,7 @@ static int __init generic_NCR5380_init(void) static void __exit generic_NCR5380_exit(void) { -#if !defined(SCSI_G_NCR5380_MEM) && defined(CONFIG_PNP) +#ifdef CONFIG_PNP if (pnp_registered) pnp_unregister_driver(&generic_NCR5380_pnp_driver); #endif diff --git a/drivers/scsi/g_NCR5380.h b/drivers/scsi/g_NCR5380.h index b175b92..3ce5b65 100644 --- a/drivers/scsi/g_NCR5380.h +++ b/drivers/scsi/g_NCR5380.h @@ -14,49 +14,28 @@ #ifndef GENERIC_NCR5380_H #define GENERIC_NCR5380_H -#ifndef SCSI_G_NCR5380_MEM #define DRV_MODULE_NAME "g_NCR5380" #define NCR5380_read(reg) \ - inb(instance->io_port + (reg)) + ioread8(hostdata->io + hostdata->offset + (reg)) #define NCR5380_write(reg, value) \ - outb(value, instance->io_port + (reg)) + iowrite8(value, hostdata->io + hostdata->offset + (reg)) #define NCR5380_implementation_fields \ + int offset; \ int c400_ctl_status; \ int c400_blk_cnt; \ int c400_host_buf; \ int io_width; -#else -/* therefore SCSI_G_NCR5380_MEM */ -#define DRV_MODULE_NAME "g_NCR5380_mmio" - #define NCR53C400_mem_base 0x3880 #define NCR53C400_host_buffer 0x3900 #define NCR53C400_region_size 0x3a00 -#define NCR5380_read(reg) \ - readb(((struct NCR5380_hostdata *)shost_priv(instance))->iomem + \ - NCR53C400_mem_base + (reg)) -#define NCR5380_write(reg, value) \ - writeb(value, ((struct NCR5380_hostdata *)shost_priv(instance))->iomem + \ - NCR53C400_mem_base + (reg)) - -#define NCR5380_implementation_fields \ - void __iomem *iomem; \ - resource_size_t iomem_size; \ - int c400_ctl_status; \ - int c400_blk_cnt; \ - int c400_host_buf; - -#endif - -#define NCR5380_dma_xfer_len(instance, cmd, phase) \ - generic_NCR5380_dma_xfer_len(instance, cmd) +#define NCR5380_dma_xfer_len generic_NCR5380_dma_xfer_len #define NCR5380_dma_recv_setup generic_NCR5380_pread #define NCR5380_dma_send_setup generic_NCR5380_pwrite -#define NCR5380_dma_residual(instance) (0) +#define NCR5380_dma_residual NCR5380_dma_residual_none #define NCR5380_intr generic_NCR5380_intr #define NCR5380_queue_command generic_NCR5380_queue_command @@ -73,4 +52,3 @@ #define BOARD_HP_C2502 4 #endif /* GENERIC_NCR5380_H */ - diff --git a/drivers/scsi/g_NCR5380_mmio.c b/drivers/scsi/g_NCR5380_mmio.c deleted file mode 100644 index 8cdde71..0000000 --- a/drivers/scsi/g_NCR5380_mmio.c +++ /dev/null @@ -1,10 +0,0 @@ -/* - * There is probably a nicer way to do this but this one makes - * pretty obvious what is happening. We rebuild the same file with - * different options for mmio versus pio. - */ - -#define SCSI_G_NCR5380_MEM - -#include "g_NCR5380.c" - diff --git a/drivers/scsi/hisi_sas/hisi_sas.h b/drivers/scsi/hisi_sas/hisi_sas.h index 72c9852..c0cd505 100644 --- a/drivers/scsi/hisi_sas/hisi_sas.h +++ b/drivers/scsi/hisi_sas/hisi_sas.h @@ -13,6 +13,7 @@ #define _HISI_SAS_H_ #include <linux/acpi.h> +#include <linux/clk.h> #include <linux/dmapool.h> #include <linux/mfd/syscon.h> #include <linux/module.h> @@ -110,7 +111,7 @@ struct hisi_sas_device { struct domain_device *sas_device; u64 attached_phy; u64 device_id; - u64 running_req; + atomic64_t running_req; u8 dev_status; }; @@ -149,7 +150,8 @@ struct hisi_sas_hw { struct domain_device *device); struct hisi_sas_device *(*alloc_dev)(struct domain_device *device); void (*sl_notify)(struct hisi_hba *hisi_hba, int phy_no); - int (*get_free_slot)(struct hisi_hba *hisi_hba, int *q, int *s); + int (*get_free_slot)(struct hisi_hba *hisi_hba, u32 dev_id, + int *q, int *s); void (*start_delivery)(struct hisi_hba *hisi_hba); int (*prep_ssp)(struct hisi_hba *hisi_hba, struct hisi_sas_slot *slot, int is_tmf, @@ -166,6 +168,9 @@ struct hisi_sas_hw { void (*phy_enable)(struct hisi_hba *hisi_hba, int phy_no); void (*phy_disable)(struct hisi_hba *hisi_hba, int phy_no); void (*phy_hard_reset)(struct hisi_hba *hisi_hba, int phy_no); + void (*phy_set_linkrate)(struct hisi_hba *hisi_hba, int phy_no, + struct sas_phy_linkrates *linkrates); + enum sas_linkrate (*phy_get_max_linkrate)(void); void (*free_device)(struct hisi_hba *hisi_hba, struct hisi_sas_device *dev); int (*get_wideport_bitmap)(struct hisi_hba *hisi_hba, int port_id); @@ -183,6 +188,7 @@ struct hisi_hba { u32 ctrl_reset_reg; u32 ctrl_reset_sts_reg; u32 ctrl_clock_ena_reg; + u32 refclk_frequency_mhz; u8 sas_addr[SAS_ADDR_SIZE]; int n_phy; @@ -205,7 +211,6 @@ struct hisi_hba { struct hisi_sas_port port[HISI_SAS_MAX_PHYS]; int queue_count; - int queue; struct hisi_sas_slot *slot_prep; struct dma_pool *sge_page_pool; diff --git a/drivers/scsi/hisi_sas/hisi_sas_main.c b/drivers/scsi/hisi_sas/hisi_sas_main.c index 2f872f7..d50e9cf 100644 --- a/drivers/scsi/hisi_sas/hisi_sas_main.c +++ b/drivers/scsi/hisi_sas/hisi_sas_main.c @@ -162,8 +162,8 @@ out: hisi_sas_slot_task_free(hisi_hba, task, abort_slot); if (task->task_done) task->task_done(task); - if (sas_dev && sas_dev->running_req) - sas_dev->running_req--; + if (sas_dev) + atomic64_dec(&sas_dev->running_req); } static int hisi_sas_task_prep(struct sas_task *task, struct hisi_hba *hisi_hba, @@ -232,8 +232,8 @@ static int hisi_sas_task_prep(struct sas_task *task, struct hisi_hba *hisi_hba, rc = hisi_sas_slot_index_alloc(hisi_hba, &slot_idx); if (rc) goto err_out; - rc = hisi_hba->hw->get_free_slot(hisi_hba, &dlvry_queue, - &dlvry_queue_slot); + rc = hisi_hba->hw->get_free_slot(hisi_hba, sas_dev->device_id, + &dlvry_queue, &dlvry_queue_slot); if (rc) goto err_out_tag; @@ -303,7 +303,7 @@ static int hisi_sas_task_prep(struct sas_task *task, struct hisi_hba *hisi_hba, hisi_hba->slot_prep = slot; - sas_dev->running_req++; + atomic64_inc(&sas_dev->running_req); ++(*pass); return 0; @@ -369,9 +369,14 @@ static void hisi_sas_bytes_dmaed(struct hisi_hba *hisi_hba, int phy_no) struct sas_phy *sphy = sas_phy->phy; sphy->negotiated_linkrate = sas_phy->linkrate; - sphy->minimum_linkrate = phy->minimum_linkrate; sphy->minimum_linkrate_hw = SAS_LINK_RATE_1_5_GBPS; - sphy->maximum_linkrate = phy->maximum_linkrate; + sphy->maximum_linkrate_hw = + hisi_hba->hw->phy_get_max_linkrate(); + if (sphy->minimum_linkrate == SAS_LINK_RATE_UNKNOWN) + sphy->minimum_linkrate = phy->minimum_linkrate; + + if (sphy->maximum_linkrate == SAS_LINK_RATE_UNKNOWN) + sphy->maximum_linkrate = phy->maximum_linkrate; } if (phy->phy_type & PORT_TYPE_SAS) { @@ -537,7 +542,7 @@ static void hisi_sas_port_notify_formed(struct asd_sas_phy *sas_phy) struct hisi_hba *hisi_hba = sas_ha->lldd_ha; struct hisi_sas_phy *phy = sas_phy->lldd_phy; struct asd_sas_port *sas_port = sas_phy->port; - struct hisi_sas_port *port = &hisi_hba->port[sas_phy->id]; + struct hisi_sas_port *port = &hisi_hba->port[phy->port_id]; unsigned long flags; if (!sas_port) @@ -645,6 +650,9 @@ static int hisi_sas_control_phy(struct asd_sas_phy *sas_phy, enum phy_func func, break; case PHY_FUNC_SET_LINK_RATE: + hisi_hba->hw->phy_set_linkrate(hisi_hba, phy_no, funcdata); + break; + case PHY_FUNC_RELEASE_SPINUP_HOLD: default: return -EOPNOTSUPP; @@ -764,7 +772,8 @@ static int hisi_sas_exec_internal_tmf_task(struct domain_device *device, task = NULL; } ex_err: - WARN_ON(retry == TASK_RETRY); + if (retry == TASK_RETRY) + dev_warn(dev, "abort tmf: executing internal task failed!\n"); sas_free_task(task); return res; } @@ -960,6 +969,9 @@ static int hisi_sas_query_task(struct sas_task *task) case TMF_RESP_FUNC_FAILED: case TMF_RESP_FUNC_COMPLETE: break; + default: + rc = TMF_RESP_FUNC_FAILED; + break; } } return rc; @@ -987,8 +999,8 @@ hisi_sas_internal_abort_task_exec(struct hisi_hba *hisi_hba, u64 device_id, rc = hisi_sas_slot_index_alloc(hisi_hba, &slot_idx); if (rc) goto err_out; - rc = hisi_hba->hw->get_free_slot(hisi_hba, &dlvry_queue, - &dlvry_queue_slot); + rc = hisi_hba->hw->get_free_slot(hisi_hba, sas_dev->device_id, + &dlvry_queue, &dlvry_queue_slot); if (rc) goto err_out_tag; @@ -1023,7 +1035,8 @@ hisi_sas_internal_abort_task_exec(struct hisi_hba *hisi_hba, u64 device_id, hisi_hba->slot_prep = slot; - sas_dev->running_req++; + atomic64_inc(&sas_dev->running_req); + /* send abort command to our chip */ hisi_hba->hw->start_delivery(hisi_hba); @@ -1396,10 +1409,13 @@ static struct Scsi_Host *hisi_sas_shost_alloc(struct platform_device *pdev, struct hisi_hba *hisi_hba; struct device *dev = &pdev->dev; struct device_node *np = pdev->dev.of_node; + struct clk *refclk; shost = scsi_host_alloc(&hisi_sas_sht, sizeof(*hisi_hba)); - if (!shost) - goto err_out; + if (!shost) { + dev_err(dev, "scsi host alloc failed\n"); + return NULL; + } hisi_hba = shost_priv(shost); hisi_hba->hw = hw; @@ -1432,6 +1448,12 @@ static struct Scsi_Host *hisi_sas_shost_alloc(struct platform_device *pdev, goto err_out; } + refclk = devm_clk_get(&pdev->dev, NULL); + if (IS_ERR(refclk)) + dev_info(dev, "no ref clk property\n"); + else + hisi_hba->refclk_frequency_mhz = clk_get_rate(refclk) / 1000000; + if (device_property_read_u32(dev, "phy-count", &hisi_hba->n_phy)) goto err_out; @@ -1457,6 +1479,7 @@ static struct Scsi_Host *hisi_sas_shost_alloc(struct platform_device *pdev, return shost; err_out: + kfree(shost); dev_err(dev, "shost alloc failed\n"); return NULL; } @@ -1483,10 +1506,8 @@ int hisi_sas_probe(struct platform_device *pdev, int rc, phy_nr, port_nr, i; shost = hisi_sas_shost_alloc(pdev, hw); - if (!shost) { - rc = -ENOMEM; - goto err_out_ha; - } + if (!shost) + return -ENOMEM; sha = SHOST_TO_SAS_HA(shost); hisi_hba = shost_priv(shost); @@ -1496,12 +1517,13 @@ int hisi_sas_probe(struct platform_device *pdev, arr_phy = devm_kcalloc(dev, phy_nr, sizeof(void *), GFP_KERNEL); arr_port = devm_kcalloc(dev, port_nr, sizeof(void *), GFP_KERNEL); - if (!arr_phy || !arr_port) - return -ENOMEM; + if (!arr_phy || !arr_port) { + rc = -ENOMEM; + goto err_out_ha; + } sha->sas_phy = arr_phy; sha->sas_port = arr_port; - sha->core.shost = shost; sha->lldd_ha = hisi_hba; shost->transportt = hisi_sas_stt; @@ -1546,6 +1568,7 @@ int hisi_sas_probe(struct platform_device *pdev, err_out_register_ha: scsi_remove_host(shost); err_out_ha: + hisi_sas_free(hisi_hba); kfree(shost); return rc; } @@ -1555,12 +1578,14 @@ int hisi_sas_remove(struct platform_device *pdev) { struct sas_ha_struct *sha = platform_get_drvdata(pdev); struct hisi_hba *hisi_hba = sha->lldd_ha; + struct Scsi_Host *shost = sha->core.shost; scsi_remove_host(sha->core.shost); sas_unregister_ha(sha); sas_remove_host(sha->core.shost); hisi_sas_free(hisi_hba); + kfree(shost); return 0; } EXPORT_SYMBOL_GPL(hisi_sas_remove); diff --git a/drivers/scsi/hisi_sas/hisi_sas_v1_hw.c b/drivers/scsi/hisi_sas/hisi_sas_v1_hw.c index c0ac49d..8a1be0b 100644 --- a/drivers/scsi/hisi_sas/hisi_sas_v1_hw.c +++ b/drivers/scsi/hisi_sas/hisi_sas_v1_hw.c @@ -843,6 +843,49 @@ static void sl_notify_v1_hw(struct hisi_hba *hisi_hba, int phy_no) hisi_sas_phy_write32(hisi_hba, phy_no, SL_CONTROL, sl_control); } +static enum sas_linkrate phy_get_max_linkrate_v1_hw(void) +{ + return SAS_LINK_RATE_6_0_GBPS; +} + +static void phy_set_linkrate_v1_hw(struct hisi_hba *hisi_hba, int phy_no, + struct sas_phy_linkrates *r) +{ + u32 prog_phy_link_rate = + hisi_sas_phy_read32(hisi_hba, phy_no, PROG_PHY_LINK_RATE); + struct hisi_sas_phy *phy = &hisi_hba->phy[phy_no]; + struct asd_sas_phy *sas_phy = &phy->sas_phy; + int i; + enum sas_linkrate min, max; + u32 rate_mask = 0; + + if (r->maximum_linkrate == SAS_LINK_RATE_UNKNOWN) { + max = sas_phy->phy->maximum_linkrate; + min = r->minimum_linkrate; + } else if (r->minimum_linkrate == SAS_LINK_RATE_UNKNOWN) { + max = r->maximum_linkrate; + min = sas_phy->phy->minimum_linkrate; + } else + return; + + sas_phy->phy->maximum_linkrate = max; + sas_phy->phy->minimum_linkrate = min; + + min -= SAS_LINK_RATE_1_5_GBPS; + max -= SAS_LINK_RATE_1_5_GBPS; + + for (i = 0; i <= max; i++) + rate_mask |= 1 << (i * 2); + + prog_phy_link_rate &= ~0xff; + prog_phy_link_rate |= rate_mask; + + hisi_sas_phy_write32(hisi_hba, phy_no, PROG_PHY_LINK_RATE, + prog_phy_link_rate); + + phy_hard_reset_v1_hw(hisi_hba, phy_no); +} + static int get_wideport_bitmap_v1_hw(struct hisi_hba *hisi_hba, int port_id) { int i, bitmap = 0; @@ -862,29 +905,23 @@ static int get_wideport_bitmap_v1_hw(struct hisi_hba *hisi_hba, int port_id) * The callpath to this function and upto writing the write * queue pointer should be safe from interruption. */ -static int get_free_slot_v1_hw(struct hisi_hba *hisi_hba, int *q, int *s) +static int get_free_slot_v1_hw(struct hisi_hba *hisi_hba, u32 dev_id, + int *q, int *s) { struct device *dev = &hisi_hba->pdev->dev; struct hisi_sas_dq *dq; u32 r, w; - int queue = hisi_hba->queue; - - while (1) { - dq = &hisi_hba->dq[queue]; - w = dq->wr_point; - r = hisi_sas_read32_relaxed(hisi_hba, - DLVRY_Q_0_RD_PTR + (queue * 0x14)); - if (r == (w+1) % HISI_SAS_QUEUE_SLOTS) { - queue = (queue + 1) % hisi_hba->queue_count; - if (queue == hisi_hba->queue) { - dev_warn(dev, "could not find free slot\n"); - return -EAGAIN; - } - continue; - } - break; + int queue = dev_id % hisi_hba->queue_count; + + dq = &hisi_hba->dq[queue]; + w = dq->wr_point; + r = hisi_sas_read32_relaxed(hisi_hba, + DLVRY_Q_0_RD_PTR + (queue * 0x14)); + if (r == (w+1) % HISI_SAS_QUEUE_SLOTS) { + dev_warn(dev, "could not find free slot\n"); + return -EAGAIN; } - hisi_hba->queue = (queue + 1) % hisi_hba->queue_count; + *q = queue; *s = w; return 0; @@ -1372,8 +1409,8 @@ static int slot_complete_v1_hw(struct hisi_hba *hisi_hba, } out: - if (sas_dev && sas_dev->running_req) - sas_dev->running_req--; + if (sas_dev) + atomic64_dec(&sas_dev->running_req); hisi_sas_slot_task_free(hisi_hba, task, slot); sts = ts->stat; @@ -1824,6 +1861,8 @@ static const struct hisi_sas_hw hisi_sas_v1_hw = { .phy_enable = enable_phy_v1_hw, .phy_disable = disable_phy_v1_hw, .phy_hard_reset = phy_hard_reset_v1_hw, + .phy_set_linkrate = phy_set_linkrate_v1_hw, + .phy_get_max_linkrate = phy_get_max_linkrate_v1_hw, .get_wideport_bitmap = get_wideport_bitmap_v1_hw, .max_command_entries = HISI_SAS_COMMAND_ENTRIES_V1_HW, .complete_hdr_size = sizeof(struct hisi_sas_complete_v1_hdr), diff --git a/drivers/scsi/hisi_sas/hisi_sas_v2_hw.c b/drivers/scsi/hisi_sas/hisi_sas_v2_hw.c index 9825a3f..b934aec 100644 --- a/drivers/scsi/hisi_sas/hisi_sas_v2_hw.c +++ b/drivers/scsi/hisi_sas/hisi_sas_v2_hw.c @@ -55,10 +55,44 @@ #define HGC_DFX_CFG2 0xc0 #define HGC_IOMB_PROC1_STATUS 0x104 #define CFG_1US_TIMER_TRSH 0xcc +#define HGC_LM_DFX_STATUS2 0x128 +#define HGC_LM_DFX_STATUS2_IOSTLIST_OFF 0 +#define HGC_LM_DFX_STATUS2_IOSTLIST_MSK (0xfff << \ + HGC_LM_DFX_STATUS2_IOSTLIST_OFF) +#define HGC_LM_DFX_STATUS2_ITCTLIST_OFF 12 +#define HGC_LM_DFX_STATUS2_ITCTLIST_MSK (0x7ff << \ + HGC_LM_DFX_STATUS2_ITCTLIST_OFF) +#define HGC_CQE_ECC_ADDR 0x13c +#define HGC_CQE_ECC_1B_ADDR_OFF 0 +#define HGC_CQE_ECC_1B_ADDR_MSK (0x3f << HGC_CQE_ECC_1B_ADDR_OFF) +#define HGC_CQE_ECC_MB_ADDR_OFF 8 +#define HGC_CQE_ECC_MB_ADDR_MSK (0x3f << HGC_CQE_ECC_MB_ADDR_OFF) +#define HGC_IOST_ECC_ADDR 0x140 +#define HGC_IOST_ECC_1B_ADDR_OFF 0 +#define HGC_IOST_ECC_1B_ADDR_MSK (0x3ff << HGC_IOST_ECC_1B_ADDR_OFF) +#define HGC_IOST_ECC_MB_ADDR_OFF 16 +#define HGC_IOST_ECC_MB_ADDR_MSK (0x3ff << HGC_IOST_ECC_MB_ADDR_OFF) +#define HGC_DQE_ECC_ADDR 0x144 +#define HGC_DQE_ECC_1B_ADDR_OFF 0 +#define HGC_DQE_ECC_1B_ADDR_MSK (0xfff << HGC_DQE_ECC_1B_ADDR_OFF) +#define HGC_DQE_ECC_MB_ADDR_OFF 16 +#define HGC_DQE_ECC_MB_ADDR_MSK (0xfff << HGC_DQE_ECC_MB_ADDR_OFF) #define HGC_INVLD_DQE_INFO 0x148 #define HGC_INVLD_DQE_INFO_FB_CH0_OFF 9 #define HGC_INVLD_DQE_INFO_FB_CH0_MSK (0x1 << HGC_INVLD_DQE_INFO_FB_CH0_OFF) #define HGC_INVLD_DQE_INFO_FB_CH3_OFF 18 +#define HGC_ITCT_ECC_ADDR 0x150 +#define HGC_ITCT_ECC_1B_ADDR_OFF 0 +#define HGC_ITCT_ECC_1B_ADDR_MSK (0x3ff << \ + HGC_ITCT_ECC_1B_ADDR_OFF) +#define HGC_ITCT_ECC_MB_ADDR_OFF 16 +#define HGC_ITCT_ECC_MB_ADDR_MSK (0x3ff << \ + HGC_ITCT_ECC_MB_ADDR_OFF) +#define HGC_AXI_FIFO_ERR_INFO 0x154 +#define AXI_ERR_INFO_OFF 0 +#define AXI_ERR_INFO_MSK (0xff << AXI_ERR_INFO_OFF) +#define FIFO_ERR_INFO_OFF 8 +#define FIFO_ERR_INFO_MSK (0xff << FIFO_ERR_INFO_OFF) #define INT_COAL_EN 0x19c #define OQ_INT_COAL_TIME 0x1a0 #define OQ_INT_COAL_CNT 0x1a4 @@ -73,13 +107,41 @@ #define ENT_INT_SRC1_D2H_FIS_CH1_MSK (0x1 << ENT_INT_SRC1_D2H_FIS_CH1_OFF) #define ENT_INT_SRC2 0x1bc #define ENT_INT_SRC3 0x1c0 +#define ENT_INT_SRC3_WP_DEPTH_OFF 8 +#define ENT_INT_SRC3_IPTT_SLOT_NOMATCH_OFF 9 +#define ENT_INT_SRC3_RP_DEPTH_OFF 10 +#define ENT_INT_SRC3_AXI_OFF 11 +#define ENT_INT_SRC3_FIFO_OFF 12 +#define ENT_INT_SRC3_LM_OFF 14 #define ENT_INT_SRC3_ITC_INT_OFF 15 #define ENT_INT_SRC3_ITC_INT_MSK (0x1 << ENT_INT_SRC3_ITC_INT_OFF) +#define ENT_INT_SRC3_ABT_OFF 16 #define ENT_INT_SRC_MSK1 0x1c4 #define ENT_INT_SRC_MSK2 0x1c8 #define ENT_INT_SRC_MSK3 0x1cc #define ENT_INT_SRC_MSK3_ENT95_MSK_OFF 31 #define ENT_INT_SRC_MSK3_ENT95_MSK_MSK (0x1 << ENT_INT_SRC_MSK3_ENT95_MSK_OFF) +#define SAS_ECC_INTR 0x1e8 +#define SAS_ECC_INTR_DQE_ECC_1B_OFF 0 +#define SAS_ECC_INTR_DQE_ECC_MB_OFF 1 +#define SAS_ECC_INTR_IOST_ECC_1B_OFF 2 +#define SAS_ECC_INTR_IOST_ECC_MB_OFF 3 +#define SAS_ECC_INTR_ITCT_ECC_MB_OFF 4 +#define SAS_ECC_INTR_ITCT_ECC_1B_OFF 5 +#define SAS_ECC_INTR_IOSTLIST_ECC_MB_OFF 6 +#define SAS_ECC_INTR_IOSTLIST_ECC_1B_OFF 7 +#define SAS_ECC_INTR_ITCTLIST_ECC_1B_OFF 8 +#define SAS_ECC_INTR_ITCTLIST_ECC_MB_OFF 9 +#define SAS_ECC_INTR_CQE_ECC_1B_OFF 10 +#define SAS_ECC_INTR_CQE_ECC_MB_OFF 11 +#define SAS_ECC_INTR_NCQ_MEM0_ECC_MB_OFF 12 +#define SAS_ECC_INTR_NCQ_MEM0_ECC_1B_OFF 13 +#define SAS_ECC_INTR_NCQ_MEM1_ECC_MB_OFF 14 +#define SAS_ECC_INTR_NCQ_MEM1_ECC_1B_OFF 15 +#define SAS_ECC_INTR_NCQ_MEM2_ECC_MB_OFF 16 +#define SAS_ECC_INTR_NCQ_MEM2_ECC_1B_OFF 17 +#define SAS_ECC_INTR_NCQ_MEM3_ECC_MB_OFF 18 +#define SAS_ECC_INTR_NCQ_MEM3_ECC_1B_OFF 19 #define SAS_ECC_INTR_MSK 0x1ec #define HGC_ERR_STAT_EN 0x238 #define DLVRY_Q_0_BASE_ADDR_LO 0x260 @@ -94,7 +156,20 @@ #define COMPL_Q_0_DEPTH 0x4e8 #define COMPL_Q_0_WR_PTR 0x4ec #define COMPL_Q_0_RD_PTR 0x4f0 - +#define HGC_RXM_DFX_STATUS14 0xae8 +#define HGC_RXM_DFX_STATUS14_MEM0_OFF 0 +#define HGC_RXM_DFX_STATUS14_MEM0_MSK (0x1ff << \ + HGC_RXM_DFX_STATUS14_MEM0_OFF) +#define HGC_RXM_DFX_STATUS14_MEM1_OFF 9 +#define HGC_RXM_DFX_STATUS14_MEM1_MSK (0x1ff << \ + HGC_RXM_DFX_STATUS14_MEM1_OFF) +#define HGC_RXM_DFX_STATUS14_MEM2_OFF 18 +#define HGC_RXM_DFX_STATUS14_MEM2_MSK (0x1ff << \ + HGC_RXM_DFX_STATUS14_MEM2_OFF) +#define HGC_RXM_DFX_STATUS15 0xaec +#define HGC_RXM_DFX_STATUS15_MEM3_OFF 0 +#define HGC_RXM_DFX_STATUS15_MEM3_MSK (0x1ff << \ + HGC_RXM_DFX_STATUS15_MEM3_OFF) /* phy registers need init */ #define PORT_BASE (0x2000) @@ -119,6 +194,9 @@ #define SL_CONTROL_NOTIFY_EN_MSK (0x1 << SL_CONTROL_NOTIFY_EN_OFF) #define SL_CONTROL_CTA_OFF 17 #define SL_CONTROL_CTA_MSK (0x1 << SL_CONTROL_CTA_OFF) +#define RX_PRIMS_STATUS (PORT_BASE + 0x98) +#define RX_BCAST_CHG_OFF 1 +#define RX_BCAST_CHG_MSK (0x1 << RX_BCAST_CHG_OFF) #define TX_ID_DWORD0 (PORT_BASE + 0x9c) #define TX_ID_DWORD1 (PORT_BASE + 0xa0) #define TX_ID_DWORD2 (PORT_BASE + 0xa4) @@ -267,6 +345,8 @@ #define ITCT_HDR_RTOLT_OFF 48 #define ITCT_HDR_RTOLT_MSK (0xffffULL << ITCT_HDR_RTOLT_OFF) +#define HISI_SAS_FATAL_INT_NR 2 + struct hisi_sas_complete_v2_hdr { __le32 dw0; __le32 dw1; @@ -659,8 +739,6 @@ static void free_device_v2_hw(struct hisi_hba *hisi_hba, qw0 &= ~(1 << ITCT_HDR_VALID_OFF); hisi_sas_write32(hisi_hba, ENT_INT_SRC3, ENT_INT_SRC3_ITC_INT_MSK); - hisi_hba->devices[dev_id].dev_type = SAS_PHY_UNUSED; - hisi_hba->devices[dev_id].dev_status = HISI_SAS_DEV_NORMAL; /* clear the itct */ hisi_sas_write32(hisi_hba, ITCT_CLR, 0); @@ -808,7 +886,7 @@ static void init_reg_v2_hw(struct hisi_hba *hisi_hba) hisi_sas_write32(hisi_hba, ENT_INT_SRC_MSK1, 0x7efefefe); hisi_sas_write32(hisi_hba, ENT_INT_SRC_MSK2, 0x7efefefe); hisi_sas_write32(hisi_hba, ENT_INT_SRC_MSK3, 0x7ffffffe); - hisi_sas_write32(hisi_hba, SAS_ECC_INTR_MSK, 0xfffff3c0); + hisi_sas_write32(hisi_hba, SAS_ECC_INTR_MSK, 0xfff00c30); for (i = 0; i < hisi_hba->queue_count; i++) hisi_sas_write32(hisi_hba, OQ0_INT_SRC_MSK+0x4*i, 0); @@ -824,7 +902,7 @@ static void init_reg_v2_hw(struct hisi_hba *hisi_hba) hisi_sas_phy_write32(hisi_hba, i, DONE_RECEIVED_TIME, 0x10); hisi_sas_phy_write32(hisi_hba, i, CHL_INT0, 0xffffffff); hisi_sas_phy_write32(hisi_hba, i, CHL_INT1, 0xffffffff); - hisi_sas_phy_write32(hisi_hba, i, CHL_INT2, 0xffffffff); + hisi_sas_phy_write32(hisi_hba, i, CHL_INT2, 0xfff87fff); hisi_sas_phy_write32(hisi_hba, i, RXOP_CHECK_CFG_H, 0x1000); hisi_sas_phy_write32(hisi_hba, i, CHL_INT1_MSK, 0xffffffff); hisi_sas_phy_write32(hisi_hba, i, CHL_INT2_MSK, 0x8ffffbff); @@ -836,7 +914,9 @@ static void init_reg_v2_hw(struct hisi_hba *hisi_hba) hisi_sas_phy_write32(hisi_hba, i, SL_RX_BCAST_CHK_MSK, 0x0); hisi_sas_phy_write32(hisi_hba, i, CHL_INT_COAL_EN, 0x0); hisi_sas_phy_write32(hisi_hba, i, PHYCTRL_OOB_RESTART_MSK, 0x0); - hisi_sas_phy_write32(hisi_hba, i, PHY_CTRL, 0x199B694); + if (hisi_hba->refclk_frequency_mhz == 66) + hisi_sas_phy_write32(hisi_hba, i, PHY_CTRL, 0x199B694); + /* else, do nothing -> leave it how you found it */ } for (i = 0; i < hisi_hba->queue_count; i++) { @@ -980,6 +1060,49 @@ static void sl_notify_v2_hw(struct hisi_hba *hisi_hba, int phy_no) hisi_sas_phy_write32(hisi_hba, phy_no, SL_CONTROL, sl_control); } +static enum sas_linkrate phy_get_max_linkrate_v2_hw(void) +{ + return SAS_LINK_RATE_12_0_GBPS; +} + +static void phy_set_linkrate_v2_hw(struct hisi_hba *hisi_hba, int phy_no, + struct sas_phy_linkrates *r) +{ + u32 prog_phy_link_rate = + hisi_sas_phy_read32(hisi_hba, phy_no, PROG_PHY_LINK_RATE); + struct hisi_sas_phy *phy = &hisi_hba->phy[phy_no]; + struct asd_sas_phy *sas_phy = &phy->sas_phy; + int i; + enum sas_linkrate min, max; + u32 rate_mask = 0; + + if (r->maximum_linkrate == SAS_LINK_RATE_UNKNOWN) { + max = sas_phy->phy->maximum_linkrate; + min = r->minimum_linkrate; + } else if (r->minimum_linkrate == SAS_LINK_RATE_UNKNOWN) { + max = r->maximum_linkrate; + min = sas_phy->phy->minimum_linkrate; + } else + return; + + sas_phy->phy->maximum_linkrate = max; + sas_phy->phy->minimum_linkrate = min; + + min -= SAS_LINK_RATE_1_5_GBPS; + max -= SAS_LINK_RATE_1_5_GBPS; + + for (i = 0; i <= max; i++) + rate_mask |= 1 << (i * 2); + + prog_phy_link_rate &= ~0xff; + prog_phy_link_rate |= rate_mask; + + hisi_sas_phy_write32(hisi_hba, phy_no, PROG_PHY_LINK_RATE, + prog_phy_link_rate); + + phy_hard_reset_v2_hw(hisi_hba, phy_no); +} + static int get_wideport_bitmap_v2_hw(struct hisi_hba *hisi_hba, int port_id) { int i, bitmap = 0; @@ -1010,29 +1133,24 @@ static int get_wideport_bitmap_v2_hw(struct hisi_hba *hisi_hba, int port_id) * The callpath to this function and upto writing the write * queue pointer should be safe from interruption. */ -static int get_free_slot_v2_hw(struct hisi_hba *hisi_hba, int *q, int *s) +static int get_free_slot_v2_hw(struct hisi_hba *hisi_hba, u32 dev_id, + int *q, int *s) { struct device *dev = &hisi_hba->pdev->dev; struct hisi_sas_dq *dq; u32 r, w; - int queue = hisi_hba->queue; - - while (1) { - dq = &hisi_hba->dq[queue]; - w = dq->wr_point; - r = hisi_sas_read32_relaxed(hisi_hba, - DLVRY_Q_0_RD_PTR + (queue * 0x14)); - if (r == (w+1) % HISI_SAS_QUEUE_SLOTS) { - queue = (queue + 1) % hisi_hba->queue_count; - if (queue == hisi_hba->queue) { - dev_warn(dev, "could not find free slot\n"); - return -EAGAIN; - } - continue; - } - break; + int queue = dev_id % hisi_hba->queue_count; + + dq = &hisi_hba->dq[queue]; + w = dq->wr_point; + r = hisi_sas_read32_relaxed(hisi_hba, + DLVRY_Q_0_RD_PTR + (queue * 0x14)); + if (r == (w+1) % HISI_SAS_QUEUE_SLOTS) { + dev_warn(dev, "full queue=%d r=%d w=%d\n\n", + queue, r, w); + return -EAGAIN; } - hisi_hba->queue = (queue + 1) % hisi_hba->queue_count; + *q = queue; *s = w; return 0; @@ -1653,8 +1771,8 @@ slot_complete_v2_hw(struct hisi_hba *hisi_hba, struct hisi_sas_slot *slot, } out: - if (sas_dev && sas_dev->running_req) - sas_dev->running_req--; + if (sas_dev) + atomic64_dec(&sas_dev->running_req); hisi_sas_slot_task_free(hisi_hba, task, slot); sts = ts->stat; @@ -1675,6 +1793,7 @@ static u8 get_ata_protocol(u8 cmd, int direction) case ATA_CMD_NCQ_NON_DATA: return SATA_PROTOCOL_FPDMA; + case ATA_CMD_DOWNLOAD_MICRO: case ATA_CMD_ID_ATA: case ATA_CMD_PMP_READ: case ATA_CMD_READ_LOG_EXT: @@ -1686,18 +1805,27 @@ static u8 get_ata_protocol(u8 cmd, int direction) case ATA_CMD_PIO_WRITE_EXT: return SATA_PROTOCOL_PIO; + case ATA_CMD_DSM: + case ATA_CMD_DOWNLOAD_MICRO_DMA: + case ATA_CMD_PMP_READ_DMA: + case ATA_CMD_PMP_WRITE_DMA: case ATA_CMD_READ: case ATA_CMD_READ_EXT: case ATA_CMD_READ_LOG_DMA_EXT: + case ATA_CMD_READ_STREAM_DMA_EXT: + case ATA_CMD_TRUSTED_RCV_DMA: + case ATA_CMD_TRUSTED_SND_DMA: case ATA_CMD_WRITE: case ATA_CMD_WRITE_EXT: + case ATA_CMD_WRITE_FUA_EXT: case ATA_CMD_WRITE_QUEUED: case ATA_CMD_WRITE_LOG_DMA_EXT: + case ATA_CMD_WRITE_STREAM_DMA_EXT: return SATA_PROTOCOL_DMA; - case ATA_CMD_DOWNLOAD_MICRO: - case ATA_CMD_DEV_RESET: case ATA_CMD_CHK_POWER: + case ATA_CMD_DEV_RESET: + case ATA_CMD_EDD: case ATA_CMD_FLUSH: case ATA_CMD_FLUSH_EXT: case ATA_CMD_VERIFY: @@ -1970,9 +2098,12 @@ static void phy_bcast_v2_hw(int phy_no, struct hisi_hba *hisi_hba) struct hisi_sas_phy *phy = &hisi_hba->phy[phy_no]; struct asd_sas_phy *sas_phy = &phy->sas_phy; struct sas_ha_struct *sas_ha = &hisi_hba->sha; + u32 bcast_status; hisi_sas_phy_write32(hisi_hba, phy_no, SL_RX_BCAST_CHK_MSK, 1); - sas_ha->notify_port_event(sas_phy, PORTE_BROADCAST_RCVD); + bcast_status = hisi_sas_phy_read32(hisi_hba, phy_no, RX_PRIMS_STATUS); + if (bcast_status & RX_BCAST_CHG_MSK) + sas_ha->notify_port_event(sas_phy, PORTE_BROADCAST_RCVD); hisi_sas_phy_write32(hisi_hba, phy_no, CHL_INT0, CHL_INT0_SL_RX_BCST_ACK_MSK); hisi_sas_phy_write32(hisi_hba, phy_no, SL_RX_BCAST_CHK_MSK, 0); @@ -2005,8 +2136,9 @@ static irqreturn_t int_chnl_int_v2_hw(int irq_no, void *p) if (irq_value1) { if (irq_value1 & (CHL_INT1_DMAC_RX_ECC_ERR_MSK | CHL_INT1_DMAC_TX_ECC_ERR_MSK)) - panic("%s: DMAC RX/TX ecc bad error! (0x%x)", - dev_name(dev), irq_value1); + panic("%s: DMAC RX/TX ecc bad error!\ + (0x%x)", + dev_name(dev), irq_value1); hisi_sas_phy_write32(hisi_hba, phy_no, CHL_INT1, irq_value1); @@ -2037,6 +2169,318 @@ static irqreturn_t int_chnl_int_v2_hw(int irq_no, void *p) return IRQ_HANDLED; } +static void +one_bit_ecc_error_process_v2_hw(struct hisi_hba *hisi_hba, u32 irq_value) +{ + struct device *dev = &hisi_hba->pdev->dev; + u32 reg_val; + + if (irq_value & BIT(SAS_ECC_INTR_DQE_ECC_1B_OFF)) { + reg_val = hisi_sas_read32(hisi_hba, HGC_DQE_ECC_ADDR); + dev_warn(dev, "hgc_dqe_acc1b_intr found: \ + Ram address is 0x%08X\n", + (reg_val & HGC_DQE_ECC_1B_ADDR_MSK) >> + HGC_DQE_ECC_1B_ADDR_OFF); + } + + if (irq_value & BIT(SAS_ECC_INTR_IOST_ECC_1B_OFF)) { + reg_val = hisi_sas_read32(hisi_hba, HGC_IOST_ECC_ADDR); + dev_warn(dev, "hgc_iost_acc1b_intr found: \ + Ram address is 0x%08X\n", + (reg_val & HGC_IOST_ECC_1B_ADDR_MSK) >> + HGC_IOST_ECC_1B_ADDR_OFF); + } + + if (irq_value & BIT(SAS_ECC_INTR_ITCT_ECC_1B_OFF)) { + reg_val = hisi_sas_read32(hisi_hba, HGC_ITCT_ECC_ADDR); + dev_warn(dev, "hgc_itct_acc1b_intr found: \ + Ram address is 0x%08X\n", + (reg_val & HGC_ITCT_ECC_1B_ADDR_MSK) >> + HGC_ITCT_ECC_1B_ADDR_OFF); + } + + if (irq_value & BIT(SAS_ECC_INTR_IOSTLIST_ECC_1B_OFF)) { + reg_val = hisi_sas_read32(hisi_hba, HGC_LM_DFX_STATUS2); + dev_warn(dev, "hgc_iostl_acc1b_intr found: \ + memory address is 0x%08X\n", + (reg_val & HGC_LM_DFX_STATUS2_IOSTLIST_MSK) >> + HGC_LM_DFX_STATUS2_IOSTLIST_OFF); + } + + if (irq_value & BIT(SAS_ECC_INTR_ITCTLIST_ECC_1B_OFF)) { + reg_val = hisi_sas_read32(hisi_hba, HGC_LM_DFX_STATUS2); + dev_warn(dev, "hgc_itctl_acc1b_intr found: \ + memory address is 0x%08X\n", + (reg_val & HGC_LM_DFX_STATUS2_ITCTLIST_MSK) >> + HGC_LM_DFX_STATUS2_ITCTLIST_OFF); + } + + if (irq_value & BIT(SAS_ECC_INTR_CQE_ECC_1B_OFF)) { + reg_val = hisi_sas_read32(hisi_hba, HGC_CQE_ECC_ADDR); + dev_warn(dev, "hgc_cqe_acc1b_intr found: \ + Ram address is 0x%08X\n", + (reg_val & HGC_CQE_ECC_1B_ADDR_MSK) >> + HGC_CQE_ECC_1B_ADDR_OFF); + } + + if (irq_value & BIT(SAS_ECC_INTR_NCQ_MEM0_ECC_1B_OFF)) { + reg_val = hisi_sas_read32(hisi_hba, HGC_RXM_DFX_STATUS14); + dev_warn(dev, "rxm_mem0_acc1b_intr found: \ + memory address is 0x%08X\n", + (reg_val & HGC_RXM_DFX_STATUS14_MEM0_MSK) >> + HGC_RXM_DFX_STATUS14_MEM0_OFF); + } + + if (irq_value & BIT(SAS_ECC_INTR_NCQ_MEM1_ECC_1B_OFF)) { + reg_val = hisi_sas_read32(hisi_hba, HGC_RXM_DFX_STATUS14); + dev_warn(dev, "rxm_mem1_acc1b_intr found: \ + memory address is 0x%08X\n", + (reg_val & HGC_RXM_DFX_STATUS14_MEM1_MSK) >> + HGC_RXM_DFX_STATUS14_MEM1_OFF); + } + + if (irq_value & BIT(SAS_ECC_INTR_NCQ_MEM2_ECC_1B_OFF)) { + reg_val = hisi_sas_read32(hisi_hba, HGC_RXM_DFX_STATUS14); + dev_warn(dev, "rxm_mem2_acc1b_intr found: \ + memory address is 0x%08X\n", + (reg_val & HGC_RXM_DFX_STATUS14_MEM2_MSK) >> + HGC_RXM_DFX_STATUS14_MEM2_OFF); + } + + if (irq_value & BIT(SAS_ECC_INTR_NCQ_MEM3_ECC_1B_OFF)) { + reg_val = hisi_sas_read32(hisi_hba, HGC_RXM_DFX_STATUS15); + dev_warn(dev, "rxm_mem3_acc1b_intr found: \ + memory address is 0x%08X\n", + (reg_val & HGC_RXM_DFX_STATUS15_MEM3_MSK) >> + HGC_RXM_DFX_STATUS15_MEM3_OFF); + } + +} + +static void multi_bit_ecc_error_process_v2_hw(struct hisi_hba *hisi_hba, + u32 irq_value) +{ + u32 reg_val; + struct device *dev = &hisi_hba->pdev->dev; + + if (irq_value & BIT(SAS_ECC_INTR_DQE_ECC_MB_OFF)) { + reg_val = hisi_sas_read32(hisi_hba, HGC_DQE_ECC_ADDR); + panic("%s: hgc_dqe_accbad_intr (0x%x) found: \ + Ram address is 0x%08X\n", + dev_name(dev), irq_value, + (reg_val & HGC_DQE_ECC_MB_ADDR_MSK) >> + HGC_DQE_ECC_MB_ADDR_OFF); + } + + if (irq_value & BIT(SAS_ECC_INTR_IOST_ECC_MB_OFF)) { + reg_val = hisi_sas_read32(hisi_hba, HGC_IOST_ECC_ADDR); + panic("%s: hgc_iost_accbad_intr (0x%x) found: \ + Ram address is 0x%08X\n", + dev_name(dev), irq_value, + (reg_val & HGC_IOST_ECC_MB_ADDR_MSK) >> + HGC_IOST_ECC_MB_ADDR_OFF); + } + + if (irq_value & BIT(SAS_ECC_INTR_ITCT_ECC_MB_OFF)) { + reg_val = hisi_sas_read32(hisi_hba, HGC_ITCT_ECC_ADDR); + panic("%s: hgc_itct_accbad_intr (0x%x) found: \ + Ram address is 0x%08X\n", + dev_name(dev), irq_value, + (reg_val & HGC_ITCT_ECC_MB_ADDR_MSK) >> + HGC_ITCT_ECC_MB_ADDR_OFF); + } + + if (irq_value & BIT(SAS_ECC_INTR_IOSTLIST_ECC_MB_OFF)) { + reg_val = hisi_sas_read32(hisi_hba, HGC_LM_DFX_STATUS2); + panic("%s: hgc_iostl_accbad_intr (0x%x) found: \ + memory address is 0x%08X\n", + dev_name(dev), irq_value, + (reg_val & HGC_LM_DFX_STATUS2_IOSTLIST_MSK) >> + HGC_LM_DFX_STATUS2_IOSTLIST_OFF); + } + + if (irq_value & BIT(SAS_ECC_INTR_ITCTLIST_ECC_MB_OFF)) { + reg_val = hisi_sas_read32(hisi_hba, HGC_LM_DFX_STATUS2); + panic("%s: hgc_itctl_accbad_intr (0x%x) found: \ + memory address is 0x%08X\n", + dev_name(dev), irq_value, + (reg_val & HGC_LM_DFX_STATUS2_ITCTLIST_MSK) >> + HGC_LM_DFX_STATUS2_ITCTLIST_OFF); + } + + if (irq_value & BIT(SAS_ECC_INTR_CQE_ECC_MB_OFF)) { + reg_val = hisi_sas_read32(hisi_hba, HGC_CQE_ECC_ADDR); + panic("%s: hgc_cqe_accbad_intr (0x%x) found: \ + Ram address is 0x%08X\n", + dev_name(dev), irq_value, + (reg_val & HGC_CQE_ECC_MB_ADDR_MSK) >> + HGC_CQE_ECC_MB_ADDR_OFF); + } + + if (irq_value & BIT(SAS_ECC_INTR_NCQ_MEM0_ECC_MB_OFF)) { + reg_val = hisi_sas_read32(hisi_hba, HGC_RXM_DFX_STATUS14); + panic("%s: rxm_mem0_accbad_intr (0x%x) found: \ + memory address is 0x%08X\n", + dev_name(dev), irq_value, + (reg_val & HGC_RXM_DFX_STATUS14_MEM0_MSK) >> + HGC_RXM_DFX_STATUS14_MEM0_OFF); + } + + if (irq_value & BIT(SAS_ECC_INTR_NCQ_MEM1_ECC_MB_OFF)) { + reg_val = hisi_sas_read32(hisi_hba, HGC_RXM_DFX_STATUS14); + panic("%s: rxm_mem1_accbad_intr (0x%x) found: \ + memory address is 0x%08X\n", + dev_name(dev), irq_value, + (reg_val & HGC_RXM_DFX_STATUS14_MEM1_MSK) >> + HGC_RXM_DFX_STATUS14_MEM1_OFF); + } + + if (irq_value & BIT(SAS_ECC_INTR_NCQ_MEM2_ECC_MB_OFF)) { + reg_val = hisi_sas_read32(hisi_hba, HGC_RXM_DFX_STATUS14); + panic("%s: rxm_mem2_accbad_intr (0x%x) found: \ + memory address is 0x%08X\n", + dev_name(dev), irq_value, + (reg_val & HGC_RXM_DFX_STATUS14_MEM2_MSK) >> + HGC_RXM_DFX_STATUS14_MEM2_OFF); + } + + if (irq_value & BIT(SAS_ECC_INTR_NCQ_MEM3_ECC_MB_OFF)) { + reg_val = hisi_sas_read32(hisi_hba, HGC_RXM_DFX_STATUS15); + panic("%s: rxm_mem3_accbad_intr (0x%x) found: \ + memory address is 0x%08X\n", + dev_name(dev), irq_value, + (reg_val & HGC_RXM_DFX_STATUS15_MEM3_MSK) >> + HGC_RXM_DFX_STATUS15_MEM3_OFF); + } + +} + +static irqreturn_t fatal_ecc_int_v2_hw(int irq_no, void *p) +{ + struct hisi_hba *hisi_hba = p; + u32 irq_value, irq_msk; + + irq_msk = hisi_sas_read32(hisi_hba, SAS_ECC_INTR_MSK); + hisi_sas_write32(hisi_hba, SAS_ECC_INTR_MSK, irq_msk | 0xffffffff); + + irq_value = hisi_sas_read32(hisi_hba, SAS_ECC_INTR); + if (irq_value) { + one_bit_ecc_error_process_v2_hw(hisi_hba, irq_value); + multi_bit_ecc_error_process_v2_hw(hisi_hba, irq_value); + } + + hisi_sas_write32(hisi_hba, SAS_ECC_INTR, irq_value); + hisi_sas_write32(hisi_hba, SAS_ECC_INTR_MSK, irq_msk); + + return IRQ_HANDLED; +} + +#define AXI_ERR_NR 8 +static const char axi_err_info[AXI_ERR_NR][32] = { + "IOST_AXI_W_ERR", + "IOST_AXI_R_ERR", + "ITCT_AXI_W_ERR", + "ITCT_AXI_R_ERR", + "SATA_AXI_W_ERR", + "SATA_AXI_R_ERR", + "DQE_AXI_R_ERR", + "CQE_AXI_W_ERR" +}; + +#define FIFO_ERR_NR 5 +static const char fifo_err_info[FIFO_ERR_NR][32] = { + "CQE_WINFO_FIFO", + "CQE_MSG_FIFIO", + "GETDQE_FIFO", + "CMDP_FIFO", + "AWTCTRL_FIFO" +}; + +static irqreturn_t fatal_axi_int_v2_hw(int irq_no, void *p) +{ + struct hisi_hba *hisi_hba = p; + u32 irq_value, irq_msk, err_value; + struct device *dev = &hisi_hba->pdev->dev; + + irq_msk = hisi_sas_read32(hisi_hba, ENT_INT_SRC_MSK3); + hisi_sas_write32(hisi_hba, ENT_INT_SRC_MSK3, irq_msk | 0xfffffffe); + + irq_value = hisi_sas_read32(hisi_hba, ENT_INT_SRC3); + if (irq_value) { + if (irq_value & BIT(ENT_INT_SRC3_WP_DEPTH_OFF)) { + hisi_sas_write32(hisi_hba, ENT_INT_SRC3, + 1 << ENT_INT_SRC3_WP_DEPTH_OFF); + panic("%s: write pointer and depth error (0x%x) \ + found!\n", + dev_name(dev), irq_value); + } + + if (irq_value & BIT(ENT_INT_SRC3_IPTT_SLOT_NOMATCH_OFF)) { + hisi_sas_write32(hisi_hba, ENT_INT_SRC3, + 1 << + ENT_INT_SRC3_IPTT_SLOT_NOMATCH_OFF); + panic("%s: iptt no match slot error (0x%x) found!\n", + dev_name(dev), irq_value); + } + + if (irq_value & BIT(ENT_INT_SRC3_RP_DEPTH_OFF)) + panic("%s: read pointer and depth error (0x%x) \ + found!\n", + dev_name(dev), irq_value); + + if (irq_value & BIT(ENT_INT_SRC3_AXI_OFF)) { + int i; + + hisi_sas_write32(hisi_hba, ENT_INT_SRC3, + 1 << ENT_INT_SRC3_AXI_OFF); + err_value = hisi_sas_read32(hisi_hba, + HGC_AXI_FIFO_ERR_INFO); + + for (i = 0; i < AXI_ERR_NR; i++) { + if (err_value & BIT(i)) + panic("%s: %s (0x%x) found!\n", + dev_name(dev), + axi_err_info[i], irq_value); + } + } + + if (irq_value & BIT(ENT_INT_SRC3_FIFO_OFF)) { + int i; + + hisi_sas_write32(hisi_hba, ENT_INT_SRC3, + 1 << ENT_INT_SRC3_FIFO_OFF); + err_value = hisi_sas_read32(hisi_hba, + HGC_AXI_FIFO_ERR_INFO); + + for (i = 0; i < FIFO_ERR_NR; i++) { + if (err_value & BIT(AXI_ERR_NR + i)) + panic("%s: %s (0x%x) found!\n", + dev_name(dev), + fifo_err_info[i], irq_value); + } + + } + + if (irq_value & BIT(ENT_INT_SRC3_LM_OFF)) { + hisi_sas_write32(hisi_hba, ENT_INT_SRC3, + 1 << ENT_INT_SRC3_LM_OFF); + panic("%s: LM add/fetch list error (0x%x) found!\n", + dev_name(dev), irq_value); + } + + if (irq_value & BIT(ENT_INT_SRC3_ABT_OFF)) { + hisi_sas_write32(hisi_hba, ENT_INT_SRC3, + 1 << ENT_INT_SRC3_ABT_OFF); + panic("%s: SAS_HGC_ABT fetch LM list error (0x%x) found!\n", + dev_name(dev), irq_value); + } + } + + hisi_sas_write32(hisi_hba, ENT_INT_SRC_MSK3, irq_msk); + + return IRQ_HANDLED; +} + static irqreturn_t cq_interrupt_v2_hw(int irq_no, void *p) { struct hisi_sas_cq *cq = p; @@ -2136,6 +2580,16 @@ static irqreturn_t sata_int_v2_hw(int irq_no, void *p) goto end; } + /* check ERR bit of Status Register */ + if (fis->status & ATA_ERR) { + dev_warn(dev, "sata int: phy%d FIS status: 0x%x\n", phy_no, + fis->status); + disable_phy_v2_hw(hisi_hba, phy_no); + enable_phy_v2_hw(hisi_hba, phy_no); + res = IRQ_NONE; + goto end; + } + if (unlikely(phy_no == 8)) { u32 port_state = hisi_sas_read32(hisi_hba, PORT_STATE); @@ -2190,6 +2644,11 @@ static irq_handler_t phy_interrupts[HISI_SAS_PHY_INT_NR] = { int_chnl_int_v2_hw, }; +static irq_handler_t fatal_interrupts[HISI_SAS_FATAL_INT_NR] = { + fatal_ecc_int_v2_hw, + fatal_axi_int_v2_hw +}; + /** * There is a limitation in the hip06 chipset that we need * to map in all mbigen interrupts, even if they are not used. @@ -2245,6 +2704,26 @@ static int interrupt_init_v2_hw(struct hisi_hba *hisi_hba) } } + for (i = 0; i < HISI_SAS_FATAL_INT_NR; i++) { + int idx = i; + + irq = irq_map[idx + 81]; + if (!irq) { + dev_err(dev, "irq init: fail map fatal interrupt %d\n", + idx); + return -ENOENT; + } + + rc = devm_request_irq(dev, irq, fatal_interrupts[i], 0, + DRV_NAME " fatal", hisi_hba); + if (rc) { + dev_err(dev, + "irq init: could not request fatal interrupt %d, rc=%d\n", + irq, rc); + return -ENOENT; + } + } + for (i = 0; i < hisi_hba->queue_count; i++) { int idx = i + 96; /* First cq interrupt is irq96 */ @@ -2303,12 +2782,26 @@ static const struct hisi_sas_hw hisi_sas_v2_hw = { .phy_enable = enable_phy_v2_hw, .phy_disable = disable_phy_v2_hw, .phy_hard_reset = phy_hard_reset_v2_hw, + .phy_set_linkrate = phy_set_linkrate_v2_hw, + .phy_get_max_linkrate = phy_get_max_linkrate_v2_hw, .max_command_entries = HISI_SAS_COMMAND_ENTRIES_V2_HW, .complete_hdr_size = sizeof(struct hisi_sas_complete_v2_hdr), }; static int hisi_sas_v2_probe(struct platform_device *pdev) { + /* + * Check if we should defer the probe before we probe the + * upper layer, as it's hard to defer later on. + */ + int ret = platform_get_irq(pdev, 0); + + if (ret < 0) { + if (ret != -EPROBE_DEFER) + dev_err(&pdev->dev, "cannot obtain irq\n"); + return ret; + } + return hisi_sas_probe(pdev, &hisi_sas_v2_hw); } @@ -2319,6 +2812,7 @@ static int hisi_sas_v2_remove(struct platform_device *pdev) static const struct of_device_id sas_v2_of_match[] = { { .compatible = "hisilicon,hip06-sas-v2",}, + { .compatible = "hisilicon,hip07-sas-v2",}, {}, }; MODULE_DEVICE_TABLE(of, sas_v2_of_match); diff --git a/drivers/scsi/hpsa.c b/drivers/scsi/hpsa.c index a1d6ab7..691a093 100644 --- a/drivers/scsi/hpsa.c +++ b/drivers/scsi/hpsa.c @@ -276,6 +276,9 @@ static int hpsa_find_cfg_addrs(struct pci_dev *pdev, void __iomem *vaddr, static int hpsa_pci_find_memory_BAR(struct pci_dev *pdev, unsigned long *memory_bar); static int hpsa_lookup_board_id(struct pci_dev *pdev, u32 *board_id); +static int wait_for_device_to_become_ready(struct ctlr_info *h, + unsigned char lunaddr[], + int reply_queue); static int hpsa_wait_for_board_state(struct pci_dev *pdev, void __iomem *vaddr, int wait_for_ready); static inline void finish_cmd(struct CommandList *c); @@ -700,9 +703,7 @@ static ssize_t lunid_show(struct device *dev, } memcpy(lunid, hdev->scsi3addr, sizeof(lunid)); spin_unlock_irqrestore(&h->lock, flags); - return snprintf(buf, 20, "0x%02x%02x%02x%02x%02x%02x%02x%02x\n", - lunid[0], lunid[1], lunid[2], lunid[3], - lunid[4], lunid[5], lunid[6], lunid[7]); + return snprintf(buf, 20, "0x%8phN\n", lunid); } static ssize_t unique_id_show(struct device *dev, @@ -864,6 +865,16 @@ static ssize_t path_info_show(struct device *dev, return output_len; } +static ssize_t host_show_ctlr_num(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct ctlr_info *h; + struct Scsi_Host *shost = class_to_shost(dev); + + h = shost_to_hba(shost); + return snprintf(buf, 20, "%d\n", h->ctlr); +} + static DEVICE_ATTR(raid_level, S_IRUGO, raid_level_show, NULL); static DEVICE_ATTR(lunid, S_IRUGO, lunid_show, NULL); static DEVICE_ATTR(unique_id, S_IRUGO, unique_id_show, NULL); @@ -887,6 +898,8 @@ static DEVICE_ATTR(resettable, S_IRUGO, host_show_resettable, NULL); static DEVICE_ATTR(lockup_detected, S_IRUGO, host_show_lockup_detected, NULL); +static DEVICE_ATTR(ctlr_num, S_IRUGO, + host_show_ctlr_num, NULL); static struct device_attribute *hpsa_sdev_attrs[] = { &dev_attr_raid_level, @@ -907,6 +920,7 @@ static struct device_attribute *hpsa_shost_attrs[] = { &dev_attr_hp_ssd_smart_path_status, &dev_attr_raid_offload_debug, &dev_attr_lockup_detected, + &dev_attr_ctlr_num, NULL, }; @@ -1001,7 +1015,7 @@ static void set_performant_mode(struct ctlr_info *h, struct CommandList *c, { if (likely(h->transMethod & CFGTBL_Trans_Performant)) { c->busaddr |= 1 | (h->blockFetchTable[c->Header.SGList] << 1); - if (unlikely(!h->msix_vector)) + if (unlikely(!h->msix_vectors)) return; if (likely(reply_queue == DEFAULT_REPLY_QUEUE)) c->Header.ReplyQueue = @@ -2541,7 +2555,7 @@ static void complete_scsi_command(struct CommandList *cp) if ((unlikely(hpsa_is_pending_event(cp)))) { if (cp->reset_pending) - return hpsa_cmd_resolve_and_free(h, cp); + return hpsa_cmd_free_and_done(h, cp, cmd); if (cp->abort_pending) return hpsa_cmd_abort_and_free(h, cp, cmd); } @@ -2824,14 +2838,8 @@ static void hpsa_print_cmd(struct ctlr_info *h, char *txt, const u8 *cdb = c->Request.CDB; const u8 *lun = c->Header.LUN.LunAddrBytes; - dev_warn(&h->pdev->dev, "%s: LUN:%02x%02x%02x%02x%02x%02x%02x%02x" - " CDB:%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x\n", - txt, lun[0], lun[1], lun[2], lun[3], - lun[4], lun[5], lun[6], lun[7], - cdb[0], cdb[1], cdb[2], cdb[3], - cdb[4], cdb[5], cdb[6], cdb[7], - cdb[8], cdb[9], cdb[10], cdb[11], - cdb[12], cdb[13], cdb[14], cdb[15]); + dev_warn(&h->pdev->dev, "%s: LUN:%8phN CDB:%16phN\n", + txt, lun, cdb); } static void hpsa_scsi_interpret_error(struct ctlr_info *h, @@ -3080,6 +3088,8 @@ static int hpsa_do_reset(struct ctlr_info *h, struct hpsa_scsi_dev_t *dev, if (unlikely(rc)) atomic_set(&dev->reset_cmds_out, 0); + else + wait_for_device_to_become_ready(h, scsi3addr, 0); mutex_unlock(&h->reset_mutex); return rc; @@ -3623,8 +3633,32 @@ out: static inline int hpsa_scsi_do_report_phys_luns(struct ctlr_info *h, struct ReportExtendedLUNdata *buf, int bufsize) { - return hpsa_scsi_do_report_luns(h, 0, buf, bufsize, - HPSA_REPORT_PHYS_EXTENDED); + int rc; + struct ReportLUNdata *lbuf; + + rc = hpsa_scsi_do_report_luns(h, 0, buf, bufsize, + HPSA_REPORT_PHYS_EXTENDED); + if (!rc || !hpsa_allow_any) + return rc; + + /* REPORT PHYS EXTENDED is not supported */ + lbuf = kzalloc(sizeof(*lbuf), GFP_KERNEL); + if (!lbuf) + return -ENOMEM; + + rc = hpsa_scsi_do_report_luns(h, 0, lbuf, sizeof(*lbuf), 0); + if (!rc) { + int i; + u32 nphys; + + /* Copy ReportLUNdata header */ + memcpy(buf, lbuf, 8); + nphys = be32_to_cpu(*((__be32 *)lbuf->LUNListLength)) / 8; + for (i = 0; i < nphys; i++) + memcpy(buf->LUN[i].lunid, lbuf->LUN[i], 8); + } + kfree(lbuf); + return rc; } static inline int hpsa_scsi_do_report_log_luns(struct ctlr_info *h, @@ -5488,7 +5522,7 @@ static int hpsa_scsi_queue_command(struct Scsi_Host *sh, struct scsi_cmnd *cmd) dev = cmd->device->hostdata; if (!dev) { - cmd->result = NOT_READY << 16; /* host byte */ + cmd->result = DID_NO_CONNECT << 16; cmd->scsi_done(cmd); return 0; } @@ -5569,6 +5603,14 @@ static void hpsa_scan_start(struct Scsi_Host *sh) if (unlikely(lockup_detected(h))) return hpsa_scan_complete(h); + /* + * Do the scan after a reset completion + */ + if (h->reset_in_progress) { + h->drv_req_rescan = 1; + return; + } + hpsa_update_scsi_devices(h); hpsa_scan_complete(h); @@ -5624,7 +5666,7 @@ static int hpsa_scsi_host_alloc(struct ctlr_info *h) sh->sg_tablesize = h->maxsgentries; sh->transportt = hpsa_sas_transport_template; sh->hostdata[0] = (unsigned long) h; - sh->irq = h->intr[h->intr_mode]; + sh->irq = pci_irq_vector(h->pdev, 0); sh->unique_id = sh->irq; h->scsi_host = sh; @@ -5999,11 +6041,9 @@ static int hpsa_send_reset_as_abort_ioaccel2(struct ctlr_info *h, if (h->raid_offload_debug > 0) dev_info(&h->pdev->dev, - "scsi %d:%d:%d:%d %s scsi3addr 0x%02x%02x%02x%02x%02x%02x%02x%02x\n", + "scsi %d:%d:%d:%d %s scsi3addr 0x%8phN\n", h->scsi_host->host_no, dev->bus, dev->target, dev->lun, - "Reset as abort", - scsi3addr[0], scsi3addr[1], scsi3addr[2], scsi3addr[3], - scsi3addr[4], scsi3addr[5], scsi3addr[6], scsi3addr[7]); + "Reset as abort", scsi3addr); if (!dev->offload_enabled) { dev_warn(&h->pdev->dev, @@ -6020,32 +6060,28 @@ static int hpsa_send_reset_as_abort_ioaccel2(struct ctlr_info *h, /* send the reset */ if (h->raid_offload_debug > 0) dev_info(&h->pdev->dev, - "Reset as abort: Resetting physical device at scsi3addr 0x%02x%02x%02x%02x%02x%02x%02x%02x\n", - psa[0], psa[1], psa[2], psa[3], - psa[4], psa[5], psa[6], psa[7]); + "Reset as abort: Resetting physical device at scsi3addr 0x%8phN\n", + psa); rc = hpsa_do_reset(h, dev, psa, HPSA_PHYS_TARGET_RESET, reply_queue); if (rc != 0) { dev_warn(&h->pdev->dev, - "Reset as abort: Failed on physical device at scsi3addr 0x%02x%02x%02x%02x%02x%02x%02x%02x\n", - psa[0], psa[1], psa[2], psa[3], - psa[4], psa[5], psa[6], psa[7]); + "Reset as abort: Failed on physical device at scsi3addr 0x%8phN\n", + psa); return rc; /* failed to reset */ } /* wait for device to recover */ if (wait_for_device_to_become_ready(h, psa, reply_queue) != 0) { dev_warn(&h->pdev->dev, - "Reset as abort: Failed: Device never recovered from reset: 0x%02x%02x%02x%02x%02x%02x%02x%02x\n", - psa[0], psa[1], psa[2], psa[3], - psa[4], psa[5], psa[6], psa[7]); + "Reset as abort: Failed: Device never recovered from reset: 0x%8phN\n", + psa); return -1; /* failed to recover */ } /* device recovered */ dev_info(&h->pdev->dev, - "Reset as abort: Device recovered from reset: scsi3addr 0x%02x%02x%02x%02x%02x%02x%02x%02x\n", - psa[0], psa[1], psa[2], psa[3], - psa[4], psa[5], psa[6], psa[7]); + "Reset as abort: Device recovered from reset: scsi3addr 0x%8phN\n", + psa); return rc; /* success */ } @@ -6663,8 +6699,7 @@ static int hpsa_big_passthru_ioctl(struct ctlr_info *h, void __user *argp) return -EINVAL; if (!capable(CAP_SYS_RAWIO)) return -EPERM; - ioc = (BIG_IOCTL_Command_struct *) - kmalloc(sizeof(*ioc), GFP_KERNEL); + ioc = kmalloc(sizeof(*ioc), GFP_KERNEL); if (!ioc) { status = -ENOMEM; goto cleanup1; @@ -7658,67 +7693,41 @@ static int find_PCI_BAR_index(struct pci_dev *pdev, unsigned long pci_bar_addr) static void hpsa_disable_interrupt_mode(struct ctlr_info *h) { - if (h->msix_vector) { - if (h->pdev->msix_enabled) - pci_disable_msix(h->pdev); - h->msix_vector = 0; - } else if (h->msi_vector) { - if (h->pdev->msi_enabled) - pci_disable_msi(h->pdev); - h->msi_vector = 0; - } + pci_free_irq_vectors(h->pdev); + h->msix_vectors = 0; } /* If MSI/MSI-X is supported by the kernel we will try to enable it on * controllers that are capable. If not, we use legacy INTx mode. */ -static void hpsa_interrupt_mode(struct ctlr_info *h) +static int hpsa_interrupt_mode(struct ctlr_info *h) { -#ifdef CONFIG_PCI_MSI - int err, i; - struct msix_entry hpsa_msix_entries[MAX_REPLY_QUEUES]; - - for (i = 0; i < MAX_REPLY_QUEUES; i++) { - hpsa_msix_entries[i].vector = 0; - hpsa_msix_entries[i].entry = i; - } + unsigned int flags = PCI_IRQ_LEGACY; + int ret; /* Some boards advertise MSI but don't really support it */ - if ((h->board_id == 0x40700E11) || (h->board_id == 0x40800E11) || - (h->board_id == 0x40820E11) || (h->board_id == 0x40830E11)) - goto default_int_mode; - if (pci_find_capability(h->pdev, PCI_CAP_ID_MSIX)) { - dev_info(&h->pdev->dev, "MSI-X capable controller\n"); - h->msix_vector = MAX_REPLY_QUEUES; - if (h->msix_vector > num_online_cpus()) - h->msix_vector = num_online_cpus(); - err = pci_enable_msix_range(h->pdev, hpsa_msix_entries, - 1, h->msix_vector); - if (err < 0) { - dev_warn(&h->pdev->dev, "MSI-X init failed %d\n", err); - h->msix_vector = 0; - goto single_msi_mode; - } else if (err < h->msix_vector) { - dev_warn(&h->pdev->dev, "only %d MSI-X vectors " - "available\n", err); + switch (h->board_id) { + case 0x40700E11: + case 0x40800E11: + case 0x40820E11: + case 0x40830E11: + break; + default: + ret = pci_alloc_irq_vectors(h->pdev, 1, MAX_REPLY_QUEUES, + PCI_IRQ_MSIX | PCI_IRQ_AFFINITY); + if (ret > 0) { + h->msix_vectors = ret; + return 0; } - h->msix_vector = err; - for (i = 0; i < h->msix_vector; i++) - h->intr[i] = hpsa_msix_entries[i].vector; - return; - } -single_msi_mode: - if (pci_find_capability(h->pdev, PCI_CAP_ID_MSI)) { - dev_info(&h->pdev->dev, "MSI capable controller\n"); - if (!pci_enable_msi(h->pdev)) - h->msi_vector = 1; - else - dev_warn(&h->pdev->dev, "MSI init failed\n"); + + flags |= PCI_IRQ_MSI; + break; } -default_int_mode: -#endif /* CONFIG_PCI_MSI */ - /* if we get here we're going to use the default interrupt mode */ - h->intr[h->intr_mode] = h->pdev->irq; + + ret = pci_alloc_irq_vectors(h->pdev, 1, 1, flags); + if (ret < 0) + return ret; + return 0; } static int hpsa_lookup_board_id(struct pci_dev *pdev, u32 *board_id) @@ -8074,7 +8083,9 @@ static int hpsa_pci_init(struct ctlr_info *h) pci_set_master(h->pdev); - hpsa_interrupt_mode(h); + err = hpsa_interrupt_mode(h); + if (err) + goto clean1; err = hpsa_pci_find_memory_BAR(h->pdev, &h->paddr); if (err) goto clean2; /* intmode+region, pci */ @@ -8110,6 +8121,7 @@ clean3: /* vaddr, intmode+region, pci */ h->vaddr = NULL; clean2: /* intmode+region, pci */ hpsa_disable_interrupt_mode(h); +clean1: /* * call pci_disable_device before pci_release_regions per * Documentation/PCI/pci.txt @@ -8243,34 +8255,20 @@ clean_up: return -ENOMEM; } -static void hpsa_irq_affinity_hints(struct ctlr_info *h) -{ - int i, cpu; - - cpu = cpumask_first(cpu_online_mask); - for (i = 0; i < h->msix_vector; i++) { - irq_set_affinity_hint(h->intr[i], get_cpu_mask(cpu)); - cpu = cpumask_next(cpu, cpu_online_mask); - } -} - /* clear affinity hints and free MSI-X, MSI, or legacy INTx vectors */ static void hpsa_free_irqs(struct ctlr_info *h) { int i; - if (!h->msix_vector || h->intr_mode != PERF_MODE_INT) { + if (!h->msix_vectors || h->intr_mode != PERF_MODE_INT) { /* Single reply queue, only one irq to free */ - i = h->intr_mode; - irq_set_affinity_hint(h->intr[i], NULL); - free_irq(h->intr[i], &h->q[i]); - h->q[i] = 0; + free_irq(pci_irq_vector(h->pdev, 0), &h->q[h->intr_mode]); + h->q[h->intr_mode] = 0; return; } - for (i = 0; i < h->msix_vector; i++) { - irq_set_affinity_hint(h->intr[i], NULL); - free_irq(h->intr[i], &h->q[i]); + for (i = 0; i < h->msix_vectors; i++) { + free_irq(pci_irq_vector(h->pdev, i), &h->q[i]); h->q[i] = 0; } for (; i < MAX_REPLY_QUEUES; i++) @@ -8291,11 +8289,11 @@ static int hpsa_request_irqs(struct ctlr_info *h, for (i = 0; i < MAX_REPLY_QUEUES; i++) h->q[i] = (u8) i; - if (h->intr_mode == PERF_MODE_INT && h->msix_vector > 0) { + if (h->intr_mode == PERF_MODE_INT && h->msix_vectors > 0) { /* If performant mode and MSI-X, use multiple reply queues */ - for (i = 0; i < h->msix_vector; i++) { + for (i = 0; i < h->msix_vectors; i++) { sprintf(h->intrname[i], "%s-msix%d", h->devname, i); - rc = request_irq(h->intr[i], msixhandler, + rc = request_irq(pci_irq_vector(h->pdev, i), msixhandler, 0, h->intrname[i], &h->q[i]); if (rc) { @@ -8303,9 +8301,9 @@ static int hpsa_request_irqs(struct ctlr_info *h, dev_err(&h->pdev->dev, "failed to get irq %d for %s\n", - h->intr[i], h->devname); + pci_irq_vector(h->pdev, i), h->devname); for (j = 0; j < i; j++) { - free_irq(h->intr[j], &h->q[j]); + free_irq(pci_irq_vector(h->pdev, j), &h->q[j]); h->q[j] = 0; } for (; j < MAX_REPLY_QUEUES; j++) @@ -8313,33 +8311,27 @@ static int hpsa_request_irqs(struct ctlr_info *h, return rc; } } - hpsa_irq_affinity_hints(h); } else { /* Use single reply pool */ - if (h->msix_vector > 0 || h->msi_vector) { - if (h->msix_vector) - sprintf(h->intrname[h->intr_mode], - "%s-msix", h->devname); - else - sprintf(h->intrname[h->intr_mode], - "%s-msi", h->devname); - rc = request_irq(h->intr[h->intr_mode], + if (h->msix_vectors > 0 || h->pdev->msi_enabled) { + sprintf(h->intrname[0], "%s-msi%s", h->devname, + h->msix_vectors ? "x" : ""); + rc = request_irq(pci_irq_vector(h->pdev, 0), msixhandler, 0, - h->intrname[h->intr_mode], + h->intrname[0], &h->q[h->intr_mode]); } else { sprintf(h->intrname[h->intr_mode], "%s-intx", h->devname); - rc = request_irq(h->intr[h->intr_mode], + rc = request_irq(pci_irq_vector(h->pdev, 0), intxhandler, IRQF_SHARED, - h->intrname[h->intr_mode], + h->intrname[0], &h->q[h->intr_mode]); } - irq_set_affinity_hint(h->intr[h->intr_mode], NULL); } if (rc) { dev_err(&h->pdev->dev, "failed to get irq %d for %s\n", - h->intr[h->intr_mode], h->devname); + pci_irq_vector(h->pdev, 0), h->devname); hpsa_free_irqs(h); return -ENODEV; } @@ -8640,6 +8632,14 @@ static void hpsa_rescan_ctlr_worker(struct work_struct *work) if (h->remove_in_progress) return; + /* + * Do the scan after the reset + */ + if (h->reset_in_progress) { + h->drv_req_rescan = 1; + return; + } + if (hpsa_ctlr_needs_rescan(h) || hpsa_offline_devices_ready(h)) { scsi_host_get(h->scsi_host); hpsa_ack_ctlr_events(h); @@ -9525,7 +9525,7 @@ static int hpsa_put_ctlr_into_performant_mode(struct ctlr_info *h) return rc; } - h->nreply_queues = h->msix_vector > 0 ? h->msix_vector : 1; + h->nreply_queues = h->msix_vectors > 0 ? h->msix_vectors : 1; hpsa_get_max_perf_mode_cmds(h); /* Performant mode ring buffer and supporting data structures */ h->reply_queue_size = h->max_commands * sizeof(u64); diff --git a/drivers/scsi/hpsa.h b/drivers/scsi/hpsa.h index 9ea162d..64e9829 100644 --- a/drivers/scsi/hpsa.h +++ b/drivers/scsi/hpsa.h @@ -176,9 +176,7 @@ struct ctlr_info { # define DOORBELL_INT 1 # define SIMPLE_MODE_INT 2 # define MEMQ_MODE_INT 3 - unsigned int intr[MAX_REPLY_QUEUES]; - unsigned int msix_vector; - unsigned int msi_vector; + unsigned int msix_vectors; int intr_mode; /* either PERF_MODE_INT or SIMPLE_MODE_INT */ struct access_method access; @@ -466,7 +464,7 @@ static unsigned long SA5_performant_completed(struct ctlr_info *h, u8 q) unsigned long register_value = FIFO_EMPTY; /* msi auto clears the interrupt pending bit. */ - if (unlikely(!(h->msi_vector || h->msix_vector))) { + if (unlikely(!(h->pdev->msi_enabled || h->msix_vectors))) { /* flush the controller write of the reply queue by reading * outbound doorbell status register. */ diff --git a/drivers/scsi/ibmvscsi/ibmvfc.c b/drivers/scsi/ibmvscsi/ibmvfc.c index 7e487c7..78b72c2 100644 --- a/drivers/scsi/ibmvscsi/ibmvfc.c +++ b/drivers/scsi/ibmvscsi/ibmvfc.c @@ -32,6 +32,7 @@ #include <linux/of.h> #include <linux/pm.h> #include <linux/stringify.h> +#include <linux/bsg-lib.h> #include <asm/firmware.h> #include <asm/irq.h> #include <asm/vio.h> @@ -1701,14 +1702,14 @@ static void ibmvfc_bsg_timeout_done(struct ibmvfc_event *evt) /** * ibmvfc_bsg_timeout - Handle a BSG timeout - * @job: struct fc_bsg_job that timed out + * @job: struct bsg_job that timed out * * Returns: * 0 on success / other on failure **/ -static int ibmvfc_bsg_timeout(struct fc_bsg_job *job) +static int ibmvfc_bsg_timeout(struct bsg_job *job) { - struct ibmvfc_host *vhost = shost_priv(job->shost); + struct ibmvfc_host *vhost = shost_priv(fc_bsg_to_shost(job)); unsigned long port_id = (unsigned long)job->dd_data; struct ibmvfc_event *evt; struct ibmvfc_tmf *tmf; @@ -1814,41 +1815,43 @@ unlock_out: /** * ibmvfc_bsg_request - Handle a BSG request - * @job: struct fc_bsg_job to be executed + * @job: struct bsg_job to be executed * * Returns: * 0 on success / other on failure **/ -static int ibmvfc_bsg_request(struct fc_bsg_job *job) +static int ibmvfc_bsg_request(struct bsg_job *job) { - struct ibmvfc_host *vhost = shost_priv(job->shost); - struct fc_rport *rport = job->rport; + struct ibmvfc_host *vhost = shost_priv(fc_bsg_to_shost(job)); + struct fc_rport *rport = fc_bsg_to_rport(job); struct ibmvfc_passthru_mad *mad; struct ibmvfc_event *evt; union ibmvfc_iu rsp_iu; unsigned long flags, port_id = -1; - unsigned int code = job->request->msgcode; + struct fc_bsg_request *bsg_request = job->request; + struct fc_bsg_reply *bsg_reply = job->reply; + unsigned int code = bsg_request->msgcode; int rc = 0, req_seg, rsp_seg, issue_login = 0; u32 fc_flags, rsp_len; ENTER; - job->reply->reply_payload_rcv_len = 0; + bsg_reply->reply_payload_rcv_len = 0; if (rport) port_id = rport->port_id; switch (code) { case FC_BSG_HST_ELS_NOLOGIN: - port_id = (job->request->rqst_data.h_els.port_id[0] << 16) | - (job->request->rqst_data.h_els.port_id[1] << 8) | - job->request->rqst_data.h_els.port_id[2]; + port_id = (bsg_request->rqst_data.h_els.port_id[0] << 16) | + (bsg_request->rqst_data.h_els.port_id[1] << 8) | + bsg_request->rqst_data.h_els.port_id[2]; case FC_BSG_RPT_ELS: fc_flags = IBMVFC_FC_ELS; break; case FC_BSG_HST_CT: issue_login = 1; - port_id = (job->request->rqst_data.h_ct.port_id[0] << 16) | - (job->request->rqst_data.h_ct.port_id[1] << 8) | - job->request->rqst_data.h_ct.port_id[2]; + port_id = (bsg_request->rqst_data.h_ct.port_id[0] << 16) | + (bsg_request->rqst_data.h_ct.port_id[1] << 8) | + bsg_request->rqst_data.h_ct.port_id[2]; case FC_BSG_RPT_CT: fc_flags = IBMVFC_FC_CT_IU; break; @@ -1937,13 +1940,14 @@ static int ibmvfc_bsg_request(struct fc_bsg_job *job) if (rsp_iu.passthru.common.status) rc = -EIO; else - job->reply->reply_payload_rcv_len = rsp_len; + bsg_reply->reply_payload_rcv_len = rsp_len; spin_lock_irqsave(vhost->host->host_lock, flags); ibmvfc_free_event(evt); spin_unlock_irqrestore(vhost->host->host_lock, flags); - job->reply->result = rc; - job->job_done(job); + bsg_reply->result = rc; + bsg_job_done(job, bsg_reply->result, + bsg_reply->reply_payload_rcv_len); rc = 0; out: dma_unmap_sg(vhost->dev, job->request_payload.sg_list, diff --git a/drivers/scsi/ibmvscsi_tgt/ibmvscsi_tgt.c b/drivers/scsi/ibmvscsi_tgt/ibmvscsi_tgt.c index 642b739..c9fa356 100644 --- a/drivers/scsi/ibmvscsi_tgt/ibmvscsi_tgt.c +++ b/drivers/scsi/ibmvscsi_tgt/ibmvscsi_tgt.c @@ -22,7 +22,7 @@ * ****************************************************************************/ -#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt #include <linux/module.h> #include <linux/kernel.h> @@ -81,7 +81,7 @@ static void ibmvscsis_determine_resid(struct se_cmd *se_cmd, } } else if (se_cmd->se_cmd_flags & SCF_OVERFLOW_BIT) { if (se_cmd->data_direction == DMA_TO_DEVICE) { - /* residual data from an overflow write */ + /* residual data from an overflow write */ rsp->flags = SRP_RSP_FLAG_DOOVER; rsp->data_out_res_cnt = cpu_to_be32(residual_count); } else if (se_cmd->data_direction == DMA_FROM_DEVICE) { @@ -101,7 +101,7 @@ static void ibmvscsis_determine_resid(struct se_cmd *se_cmd, * and the function returns TRUE. * * EXECUTION ENVIRONMENT: - * Interrupt or Process environment + * Interrupt or Process environment */ static bool connection_broken(struct scsi_info *vscsi) { @@ -324,7 +324,7 @@ static struct viosrp_crq *ibmvscsis_cmd_q_dequeue(uint mask, } /** - * ibmvscsis_send_init_message() - send initialize message to the client + * ibmvscsis_send_init_message() - send initialize message to the client * @vscsi: Pointer to our adapter structure * @format: Which Init Message format to send * @@ -382,13 +382,13 @@ static long ibmvscsis_check_init_msg(struct scsi_info *vscsi, uint *format) vscsi->cmd_q.base_addr); if (crq) { *format = (uint)(crq->format); - rc = ERROR; + rc = ERROR; crq->valid = INVALIDATE_CMD_RESP_EL; dma_rmb(); } } else { *format = (uint)(crq->format); - rc = ERROR; + rc = ERROR; crq->valid = INVALIDATE_CMD_RESP_EL; dma_rmb(); } @@ -397,166 +397,6 @@ static long ibmvscsis_check_init_msg(struct scsi_info *vscsi, uint *format) } /** - * ibmvscsis_establish_new_q() - Establish new CRQ queue - * @vscsi: Pointer to our adapter structure - * @new_state: New state being established after resetting the queue - * - * Must be called with interrupt lock held. - */ -static long ibmvscsis_establish_new_q(struct scsi_info *vscsi, uint new_state) -{ - long rc = ADAPT_SUCCESS; - uint format; - - vscsi->flags &= PRESERVE_FLAG_FIELDS; - vscsi->rsp_q_timer.timer_pops = 0; - vscsi->debit = 0; - vscsi->credit = 0; - - rc = vio_enable_interrupts(vscsi->dma_dev); - if (rc) { - pr_warn("reset_queue: failed to enable interrupts, rc %ld\n", - rc); - return rc; - } - - rc = ibmvscsis_check_init_msg(vscsi, &format); - if (rc) { - dev_err(&vscsi->dev, "reset_queue: check_init_msg failed, rc %ld\n", - rc); - return rc; - } - - if (format == UNUSED_FORMAT && new_state == WAIT_CONNECTION) { - rc = ibmvscsis_send_init_message(vscsi, INIT_MSG); - switch (rc) { - case H_SUCCESS: - case H_DROPPED: - case H_CLOSED: - rc = ADAPT_SUCCESS; - break; - - case H_PARAMETER: - case H_HARDWARE: - break; - - default: - vscsi->state = UNDEFINED; - rc = H_HARDWARE; - break; - } - } - - return rc; -} - -/** - * ibmvscsis_reset_queue() - Reset CRQ Queue - * @vscsi: Pointer to our adapter structure - * @new_state: New state to establish after resetting the queue - * - * This function calls h_free_q and then calls h_reg_q and does all - * of the bookkeeping to get us back to where we can communicate. - * - * Actually, we don't always call h_free_crq. A problem was discovered - * where one partition would close and reopen his queue, which would - * cause his partner to get a transport event, which would cause him to - * close and reopen his queue, which would cause the original partition - * to get a transport event, etc., etc. To prevent this, we don't - * actually close our queue if the client initiated the reset, (i.e. - * either we got a transport event or we have detected that the client's - * queue is gone) - * - * EXECUTION ENVIRONMENT: - * Process environment, called with interrupt lock held - */ -static void ibmvscsis_reset_queue(struct scsi_info *vscsi, uint new_state) -{ - int bytes; - long rc = ADAPT_SUCCESS; - - pr_debug("reset_queue: flags 0x%x\n", vscsi->flags); - - /* don't reset, the client did it for us */ - if (vscsi->flags & (CLIENT_FAILED | TRANS_EVENT)) { - vscsi->flags &= PRESERVE_FLAG_FIELDS; - vscsi->rsp_q_timer.timer_pops = 0; - vscsi->debit = 0; - vscsi->credit = 0; - vscsi->state = new_state; - vio_enable_interrupts(vscsi->dma_dev); - } else { - rc = ibmvscsis_free_command_q(vscsi); - if (rc == ADAPT_SUCCESS) { - vscsi->state = new_state; - - bytes = vscsi->cmd_q.size * PAGE_SIZE; - rc = h_reg_crq(vscsi->dds.unit_id, - vscsi->cmd_q.crq_token, bytes); - if (rc == H_CLOSED || rc == H_SUCCESS) { - rc = ibmvscsis_establish_new_q(vscsi, - new_state); - } - - if (rc != ADAPT_SUCCESS) { - pr_debug("reset_queue: reg_crq rc %ld\n", rc); - - vscsi->state = ERR_DISCONNECTED; - vscsi->flags |= RESPONSE_Q_DOWN; - ibmvscsis_free_command_q(vscsi); - } - } else { - vscsi->state = ERR_DISCONNECTED; - vscsi->flags |= RESPONSE_Q_DOWN; - } - } -} - -/** - * ibmvscsis_free_cmd_resources() - Free command resources - * @vscsi: Pointer to our adapter structure - * @cmd: Command which is not longer in use - * - * Must be called with interrupt lock held. - */ -static void ibmvscsis_free_cmd_resources(struct scsi_info *vscsi, - struct ibmvscsis_cmd *cmd) -{ - struct iu_entry *iue = cmd->iue; - - switch (cmd->type) { - case TASK_MANAGEMENT: - case SCSI_CDB: - /* - * When the queue goes down this value is cleared, so it - * cannot be cleared in this general purpose function. - */ - if (vscsi->debit) - vscsi->debit -= 1; - break; - case ADAPTER_MAD: - vscsi->flags &= ~PROCESSING_MAD; - break; - case UNSET_TYPE: - break; - default: - dev_err(&vscsi->dev, "free_cmd_resources unknown type %d\n", - cmd->type); - break; - } - - cmd->iue = NULL; - list_add_tail(&cmd->list, &vscsi->free_cmd); - srp_iu_put(iue); - - if (list_empty(&vscsi->active_q) && list_empty(&vscsi->schedule_q) && - list_empty(&vscsi->waiting_rsp) && (vscsi->flags & WAIT_FOR_IDLE)) { - vscsi->flags &= ~WAIT_FOR_IDLE; - complete(&vscsi->wait_idle); - } -} - -/** * ibmvscsis_disconnect() - Helper function to disconnect * @work: Pointer to work_struct, gives access to our adapter structure * @@ -575,7 +415,6 @@ static void ibmvscsis_disconnect(struct work_struct *work) proc_work); u16 new_state; bool wait_idle = false; - long rc = ADAPT_SUCCESS; spin_lock_bh(&vscsi->intr_lock); new_state = vscsi->new_state; @@ -589,7 +428,7 @@ static void ibmvscsis_disconnect(struct work_struct *work) * should transitition to the new state */ switch (vscsi->state) { - /* Should never be called while in this state. */ + /* Should never be called while in this state. */ case NO_QUEUE: /* * Can never transition from this state; @@ -628,30 +467,24 @@ static void ibmvscsis_disconnect(struct work_struct *work) vscsi->state = new_state; break; - /* - * If this is a transition into an error state. - * a client is attempting to establish a connection - * and has violated the RPA protocol. - * There can be nothing pending on the adapter although - * there can be requests in the command queue. - */ case WAIT_ENABLED: - case PART_UP_WAIT_ENAB: switch (new_state) { - case ERR_DISCONNECT: - vscsi->flags |= RESPONSE_Q_DOWN; + case UNCONFIGURING: vscsi->state = new_state; + vscsi->flags |= RESPONSE_Q_DOWN; vscsi->flags &= ~(SCHEDULE_DISCONNECT | DISCONNECT_SCHEDULED); - ibmvscsis_free_command_q(vscsi); - break; - case ERR_DISCONNECT_RECONNECT: - ibmvscsis_reset_queue(vscsi, WAIT_ENABLED); + dma_rmb(); + if (vscsi->flags & CFG_SLEEPING) { + vscsi->flags &= ~CFG_SLEEPING; + complete(&vscsi->unconfig); + } break; /* should never happen */ + case ERR_DISCONNECT: + case ERR_DISCONNECT_RECONNECT: case WAIT_IDLE: - rc = ERROR; dev_err(&vscsi->dev, "disconnect: invalid state %d for WAIT_IDLE\n", vscsi->state); break; @@ -660,6 +493,13 @@ static void ibmvscsis_disconnect(struct work_struct *work) case WAIT_IDLE: switch (new_state) { + case UNCONFIGURING: + vscsi->flags |= RESPONSE_Q_DOWN; + vscsi->state = new_state; + vscsi->flags &= ~(SCHEDULE_DISCONNECT | + DISCONNECT_SCHEDULED); + ibmvscsis_free_command_q(vscsi); + break; case ERR_DISCONNECT: case ERR_DISCONNECT_RECONNECT: vscsi->state = new_state; @@ -788,7 +628,6 @@ static void ibmvscsis_post_disconnect(struct scsi_info *vscsi, uint new_state, break; case WAIT_ENABLED: - case PART_UP_WAIT_ENAB: case WAIT_IDLE: case WAIT_CONNECTION: case CONNECTED: @@ -806,6 +645,310 @@ static void ibmvscsis_post_disconnect(struct scsi_info *vscsi, uint new_state, } /** + * ibmvscsis_handle_init_compl_msg() - Respond to an Init Complete Message + * @vscsi: Pointer to our adapter structure + * + * Must be called with interrupt lock held. + */ +static long ibmvscsis_handle_init_compl_msg(struct scsi_info *vscsi) +{ + long rc = ADAPT_SUCCESS; + + switch (vscsi->state) { + case NO_QUEUE: + case ERR_DISCONNECT: + case ERR_DISCONNECT_RECONNECT: + case ERR_DISCONNECTED: + case UNCONFIGURING: + case UNDEFINED: + rc = ERROR; + break; + + case WAIT_CONNECTION: + vscsi->state = CONNECTED; + break; + + case WAIT_IDLE: + case SRP_PROCESSING: + case CONNECTED: + case WAIT_ENABLED: + default: + rc = ERROR; + dev_err(&vscsi->dev, "init_msg: invalid state %d to get init compl msg\n", + vscsi->state); + ibmvscsis_post_disconnect(vscsi, ERR_DISCONNECT_RECONNECT, 0); + break; + } + + return rc; +} + +/** + * ibmvscsis_handle_init_msg() - Respond to an Init Message + * @vscsi: Pointer to our adapter structure + * + * Must be called with interrupt lock held. + */ +static long ibmvscsis_handle_init_msg(struct scsi_info *vscsi) +{ + long rc = ADAPT_SUCCESS; + + switch (vscsi->state) { + case WAIT_CONNECTION: + rc = ibmvscsis_send_init_message(vscsi, INIT_COMPLETE_MSG); + switch (rc) { + case H_SUCCESS: + vscsi->state = CONNECTED; + break; + + case H_PARAMETER: + dev_err(&vscsi->dev, "init_msg: failed to send, rc %ld\n", + rc); + ibmvscsis_post_disconnect(vscsi, ERR_DISCONNECT, 0); + break; + + case H_DROPPED: + dev_err(&vscsi->dev, "init_msg: failed to send, rc %ld\n", + rc); + rc = ERROR; + ibmvscsis_post_disconnect(vscsi, + ERR_DISCONNECT_RECONNECT, 0); + break; + + case H_CLOSED: + pr_warn("init_msg: failed to send, rc %ld\n", rc); + rc = 0; + break; + } + break; + + case UNDEFINED: + rc = ERROR; + break; + + case UNCONFIGURING: + break; + + case WAIT_ENABLED: + case CONNECTED: + case SRP_PROCESSING: + case WAIT_IDLE: + case NO_QUEUE: + case ERR_DISCONNECT: + case ERR_DISCONNECT_RECONNECT: + case ERR_DISCONNECTED: + default: + rc = ERROR; + dev_err(&vscsi->dev, "init_msg: invalid state %d to get init msg\n", + vscsi->state); + ibmvscsis_post_disconnect(vscsi, ERR_DISCONNECT_RECONNECT, 0); + break; + } + + return rc; +} + +/** + * ibmvscsis_init_msg() - Respond to an init message + * @vscsi: Pointer to our adapter structure + * @crq: Pointer to CRQ element containing the Init Message + * + * EXECUTION ENVIRONMENT: + * Interrupt, interrupt lock held + */ +static long ibmvscsis_init_msg(struct scsi_info *vscsi, struct viosrp_crq *crq) +{ + long rc = ADAPT_SUCCESS; + + pr_debug("init_msg: state 0x%hx\n", vscsi->state); + + rc = h_vioctl(vscsi->dds.unit_id, H_GET_PARTNER_INFO, + (u64)vscsi->map_ioba | ((u64)PAGE_SIZE << 32), 0, 0, 0, + 0); + if (rc == H_SUCCESS) { + vscsi->client_data.partition_number = + be64_to_cpu(*(u64 *)vscsi->map_buf); + pr_debug("init_msg, part num %d\n", + vscsi->client_data.partition_number); + } else { + pr_debug("init_msg h_vioctl rc %ld\n", rc); + rc = ADAPT_SUCCESS; + } + + if (crq->format == INIT_MSG) { + rc = ibmvscsis_handle_init_msg(vscsi); + } else if (crq->format == INIT_COMPLETE_MSG) { + rc = ibmvscsis_handle_init_compl_msg(vscsi); + } else { + rc = ERROR; + dev_err(&vscsi->dev, "init_msg: invalid format %d\n", + (uint)crq->format); + ibmvscsis_post_disconnect(vscsi, ERR_DISCONNECT_RECONNECT, 0); + } + + return rc; +} + +/** + * ibmvscsis_establish_new_q() - Establish new CRQ queue + * @vscsi: Pointer to our adapter structure + * + * Must be called with interrupt lock held. + */ +static long ibmvscsis_establish_new_q(struct scsi_info *vscsi) +{ + long rc = ADAPT_SUCCESS; + uint format; + + vscsi->flags &= PRESERVE_FLAG_FIELDS; + vscsi->rsp_q_timer.timer_pops = 0; + vscsi->debit = 0; + vscsi->credit = 0; + + rc = vio_enable_interrupts(vscsi->dma_dev); + if (rc) { + pr_warn("establish_new_q: failed to enable interrupts, rc %ld\n", + rc); + return rc; + } + + rc = ibmvscsis_check_init_msg(vscsi, &format); + if (rc) { + dev_err(&vscsi->dev, "establish_new_q: check_init_msg failed, rc %ld\n", + rc); + return rc; + } + + if (format == UNUSED_FORMAT) { + rc = ibmvscsis_send_init_message(vscsi, INIT_MSG); + switch (rc) { + case H_SUCCESS: + case H_DROPPED: + case H_CLOSED: + rc = ADAPT_SUCCESS; + break; + + case H_PARAMETER: + case H_HARDWARE: + break; + + default: + vscsi->state = UNDEFINED; + rc = H_HARDWARE; + break; + } + } else if (format == INIT_MSG) { + rc = ibmvscsis_handle_init_msg(vscsi); + } + + return rc; +} + +/** + * ibmvscsis_reset_queue() - Reset CRQ Queue + * @vscsi: Pointer to our adapter structure + * + * This function calls h_free_q and then calls h_reg_q and does all + * of the bookkeeping to get us back to where we can communicate. + * + * Actually, we don't always call h_free_crq. A problem was discovered + * where one partition would close and reopen his queue, which would + * cause his partner to get a transport event, which would cause him to + * close and reopen his queue, which would cause the original partition + * to get a transport event, etc., etc. To prevent this, we don't + * actually close our queue if the client initiated the reset, (i.e. + * either we got a transport event or we have detected that the client's + * queue is gone) + * + * EXECUTION ENVIRONMENT: + * Process environment, called with interrupt lock held + */ +static void ibmvscsis_reset_queue(struct scsi_info *vscsi) +{ + int bytes; + long rc = ADAPT_SUCCESS; + + pr_debug("reset_queue: flags 0x%x\n", vscsi->flags); + + /* don't reset, the client did it for us */ + if (vscsi->flags & (CLIENT_FAILED | TRANS_EVENT)) { + vscsi->flags &= PRESERVE_FLAG_FIELDS; + vscsi->rsp_q_timer.timer_pops = 0; + vscsi->debit = 0; + vscsi->credit = 0; + vscsi->state = WAIT_CONNECTION; + vio_enable_interrupts(vscsi->dma_dev); + } else { + rc = ibmvscsis_free_command_q(vscsi); + if (rc == ADAPT_SUCCESS) { + vscsi->state = WAIT_CONNECTION; + + bytes = vscsi->cmd_q.size * PAGE_SIZE; + rc = h_reg_crq(vscsi->dds.unit_id, + vscsi->cmd_q.crq_token, bytes); + if (rc == H_CLOSED || rc == H_SUCCESS) { + rc = ibmvscsis_establish_new_q(vscsi); + } + + if (rc != ADAPT_SUCCESS) { + pr_debug("reset_queue: reg_crq rc %ld\n", rc); + + vscsi->state = ERR_DISCONNECTED; + vscsi->flags |= RESPONSE_Q_DOWN; + ibmvscsis_free_command_q(vscsi); + } + } else { + vscsi->state = ERR_DISCONNECTED; + vscsi->flags |= RESPONSE_Q_DOWN; + } + } +} + +/** + * ibmvscsis_free_cmd_resources() - Free command resources + * @vscsi: Pointer to our adapter structure + * @cmd: Command which is not longer in use + * + * Must be called with interrupt lock held. + */ +static void ibmvscsis_free_cmd_resources(struct scsi_info *vscsi, + struct ibmvscsis_cmd *cmd) +{ + struct iu_entry *iue = cmd->iue; + + switch (cmd->type) { + case TASK_MANAGEMENT: + case SCSI_CDB: + /* + * When the queue goes down this value is cleared, so it + * cannot be cleared in this general purpose function. + */ + if (vscsi->debit) + vscsi->debit -= 1; + break; + case ADAPTER_MAD: + vscsi->flags &= ~PROCESSING_MAD; + break; + case UNSET_TYPE: + break; + default: + dev_err(&vscsi->dev, "free_cmd_resources unknown type %d\n", + cmd->type); + break; + } + + cmd->iue = NULL; + list_add_tail(&cmd->list, &vscsi->free_cmd); + srp_iu_put(iue); + + if (list_empty(&vscsi->active_q) && list_empty(&vscsi->schedule_q) && + list_empty(&vscsi->waiting_rsp) && (vscsi->flags & WAIT_FOR_IDLE)) { + vscsi->flags &= ~WAIT_FOR_IDLE; + complete(&vscsi->wait_idle); + } +} + +/** * ibmvscsis_trans_event() - Handle a Transport Event * @vscsi: Pointer to our adapter structure * @crq: Pointer to CRQ entry containing the Transport Event @@ -863,10 +1006,6 @@ static long ibmvscsis_trans_event(struct scsi_info *vscsi, TRANS_EVENT)); break; - case PART_UP_WAIT_ENAB: - vscsi->state = WAIT_ENABLED; - break; - case SRP_PROCESSING: if ((vscsi->debit > 0) || !list_empty(&vscsi->schedule_q) || @@ -895,7 +1034,7 @@ static long ibmvscsis_trans_event(struct scsi_info *vscsi, } } - rc = vscsi->flags & SCHEDULE_DISCONNECT; + rc = vscsi->flags & SCHEDULE_DISCONNECT; pr_debug("Leaving trans_event: flags 0x%x, state 0x%hx, rc %ld\n", vscsi->flags, vscsi->state, rc); @@ -1066,16 +1205,28 @@ static void ibmvscsis_adapter_idle(struct scsi_info *vscsi) free_qs = true; switch (vscsi->state) { + case UNCONFIGURING: + ibmvscsis_free_command_q(vscsi); + dma_rmb(); + isync(); + if (vscsi->flags & CFG_SLEEPING) { + vscsi->flags &= ~CFG_SLEEPING; + complete(&vscsi->unconfig); + } + break; case ERR_DISCONNECT_RECONNECT: - ibmvscsis_reset_queue(vscsi, WAIT_CONNECTION); + ibmvscsis_reset_queue(vscsi); pr_debug("adapter_idle, disc_rec: flags 0x%x\n", vscsi->flags); break; case ERR_DISCONNECT: ibmvscsis_free_command_q(vscsi); - vscsi->flags &= ~DISCONNECT_SCHEDULED; + vscsi->flags &= ~(SCHEDULE_DISCONNECT | DISCONNECT_SCHEDULED); vscsi->flags |= RESPONSE_Q_DOWN; - vscsi->state = ERR_DISCONNECTED; + if (vscsi->tport.enabled) + vscsi->state = ERR_DISCONNECTED; + else + vscsi->state = WAIT_ENABLED; pr_debug("adapter_idle, disc: flags 0x%x, state 0x%hx\n", vscsi->flags, vscsi->state); break; @@ -1220,7 +1371,7 @@ static long ibmvscsis_copy_crq_packet(struct scsi_info *vscsi, * @iue: Information Unit containing the Adapter Info MAD request * * EXECUTION ENVIRONMENT: - * Interrupt adpater lock is held + * Interrupt adapter lock is held */ static long ibmvscsis_adapter_info(struct scsi_info *vscsi, struct iu_entry *iue) @@ -1620,8 +1771,8 @@ static void ibmvscsis_send_messages(struct scsi_info *vscsi) be64_to_cpu(msg_hi), be64_to_cpu(cmd->rsp.tag)); - pr_debug("send_messages: tag 0x%llx, rc %ld\n", - be64_to_cpu(cmd->rsp.tag), rc); + pr_debug("send_messages: cmd %p, tag 0x%llx, rc %ld\n", + cmd, be64_to_cpu(cmd->rsp.tag), rc); /* if all ok free up the command element resources */ if (rc == H_SUCCESS) { @@ -1691,7 +1842,7 @@ static void ibmvscsis_send_mad_resp(struct scsi_info *vscsi, * @crq: Pointer to the CRQ entry containing the MAD request * * EXECUTION ENVIRONMENT: - * Interrupt called with adapter lock held + * Interrupt, called with adapter lock held */ static long ibmvscsis_mad(struct scsi_info *vscsi, struct viosrp_crq *crq) { @@ -1745,14 +1896,7 @@ static long ibmvscsis_mad(struct scsi_info *vscsi, struct viosrp_crq *crq) pr_debug("mad: type %d\n", be32_to_cpu(mad->type)); - if (be16_to_cpu(mad->length) < 0) { - dev_err(&vscsi->dev, "mad: length is < 0\n"); - ibmvscsis_post_disconnect(vscsi, - ERR_DISCONNECT_RECONNECT, 0); - rc = SRP_VIOLATION; - } else { - rc = ibmvscsis_process_mad(vscsi, iue); - } + rc = ibmvscsis_process_mad(vscsi, iue); pr_debug("mad: status %hd, rc %ld\n", be16_to_cpu(mad->status), rc); @@ -1864,7 +2008,7 @@ static long ibmvscsis_srp_login_rej(struct scsi_info *vscsi, break; case H_PERMISSION: if (connection_broken(vscsi)) - flag_bits = RESPONSE_Q_DOWN | CLIENT_FAILED; + flag_bits = RESPONSE_Q_DOWN | CLIENT_FAILED; dev_err(&vscsi->dev, "login_rej: error copying to client, rc %ld\n", rc); ibmvscsis_post_disconnect(vscsi, ERR_DISCONNECT_RECONNECT, @@ -2187,156 +2331,6 @@ static long ibmvscsis_ping_response(struct scsi_info *vscsi) } /** - * ibmvscsis_handle_init_compl_msg() - Respond to an Init Complete Message - * @vscsi: Pointer to our adapter structure - * - * Must be called with interrupt lock held. - */ -static long ibmvscsis_handle_init_compl_msg(struct scsi_info *vscsi) -{ - long rc = ADAPT_SUCCESS; - - switch (vscsi->state) { - case NO_QUEUE: - case ERR_DISCONNECT: - case ERR_DISCONNECT_RECONNECT: - case ERR_DISCONNECTED: - case UNCONFIGURING: - case UNDEFINED: - rc = ERROR; - break; - - case WAIT_CONNECTION: - vscsi->state = CONNECTED; - break; - - case WAIT_IDLE: - case SRP_PROCESSING: - case CONNECTED: - case WAIT_ENABLED: - case PART_UP_WAIT_ENAB: - default: - rc = ERROR; - dev_err(&vscsi->dev, "init_msg: invalid state %d to get init compl msg\n", - vscsi->state); - ibmvscsis_post_disconnect(vscsi, ERR_DISCONNECT_RECONNECT, 0); - break; - } - - return rc; -} - -/** - * ibmvscsis_handle_init_msg() - Respond to an Init Message - * @vscsi: Pointer to our adapter structure - * - * Must be called with interrupt lock held. - */ -static long ibmvscsis_handle_init_msg(struct scsi_info *vscsi) -{ - long rc = ADAPT_SUCCESS; - - switch (vscsi->state) { - case WAIT_ENABLED: - vscsi->state = PART_UP_WAIT_ENAB; - break; - - case WAIT_CONNECTION: - rc = ibmvscsis_send_init_message(vscsi, INIT_COMPLETE_MSG); - switch (rc) { - case H_SUCCESS: - vscsi->state = CONNECTED; - break; - - case H_PARAMETER: - dev_err(&vscsi->dev, "init_msg: failed to send, rc %ld\n", - rc); - ibmvscsis_post_disconnect(vscsi, ERR_DISCONNECT, 0); - break; - - case H_DROPPED: - dev_err(&vscsi->dev, "init_msg: failed to send, rc %ld\n", - rc); - rc = ERROR; - ibmvscsis_post_disconnect(vscsi, - ERR_DISCONNECT_RECONNECT, 0); - break; - - case H_CLOSED: - pr_warn("init_msg: failed to send, rc %ld\n", rc); - rc = 0; - break; - } - break; - - case UNDEFINED: - rc = ERROR; - break; - - case UNCONFIGURING: - break; - - case PART_UP_WAIT_ENAB: - case CONNECTED: - case SRP_PROCESSING: - case WAIT_IDLE: - case NO_QUEUE: - case ERR_DISCONNECT: - case ERR_DISCONNECT_RECONNECT: - case ERR_DISCONNECTED: - default: - rc = ERROR; - dev_err(&vscsi->dev, "init_msg: invalid state %d to get init msg\n", - vscsi->state); - ibmvscsis_post_disconnect(vscsi, ERR_DISCONNECT_RECONNECT, 0); - break; - } - - return rc; -} - -/** - * ibmvscsis_init_msg() - Respond to an init message - * @vscsi: Pointer to our adapter structure - * @crq: Pointer to CRQ element containing the Init Message - * - * EXECUTION ENVIRONMENT: - * Interrupt, interrupt lock held - */ -static long ibmvscsis_init_msg(struct scsi_info *vscsi, struct viosrp_crq *crq) -{ - long rc = ADAPT_SUCCESS; - - pr_debug("init_msg: state 0x%hx\n", vscsi->state); - - rc = h_vioctl(vscsi->dds.unit_id, H_GET_PARTNER_INFO, - (u64)vscsi->map_ioba | ((u64)PAGE_SIZE << 32), 0, 0, 0, - 0); - if (rc == H_SUCCESS) { - vscsi->client_data.partition_number = - be64_to_cpu(*(u64 *)vscsi->map_buf); - pr_debug("init_msg, part num %d\n", - vscsi->client_data.partition_number); - } else { - pr_debug("init_msg h_vioctl rc %ld\n", rc); - rc = ADAPT_SUCCESS; - } - - if (crq->format == INIT_MSG) { - rc = ibmvscsis_handle_init_msg(vscsi); - } else if (crq->format == INIT_COMPLETE_MSG) { - rc = ibmvscsis_handle_init_compl_msg(vscsi); - } else { - rc = ERROR; - dev_err(&vscsi->dev, "init_msg: invalid format %d\n", - (uint)crq->format); - ibmvscsis_post_disconnect(vscsi, ERR_DISCONNECT_RECONNECT, 0); - } - - return rc; -} - -/** * ibmvscsis_parse_command() - Parse an element taken from the cmd rsp queue. * @vscsi: Pointer to our adapter structure * @crq: Pointer to CRQ element containing the SRP request @@ -2391,7 +2385,7 @@ static long ibmvscsis_parse_command(struct scsi_info *vscsi, break; case VALID_TRANS_EVENT: - rc = ibmvscsis_trans_event(vscsi, crq); + rc = ibmvscsis_trans_event(vscsi, crq); break; case VALID_INIT_MSG: @@ -2522,7 +2516,6 @@ static void ibmvscsis_parse_cmd(struct scsi_info *vscsi, dev_err(&vscsi->dev, "0x%llx: parsing SRP descriptor table failed.\n", srp->tag); goto fail; - return; } cmd->rsp.sol_not = srp->sol_not; @@ -2559,6 +2552,10 @@ static void ibmvscsis_parse_cmd(struct scsi_info *vscsi, data_len, attr, dir, 0); if (rc) { dev_err(&vscsi->dev, "target_submit_cmd failed, rc %d\n", rc); + spin_lock_bh(&vscsi->intr_lock); + list_del(&cmd->list); + ibmvscsis_free_cmd_resources(vscsi, cmd); + spin_unlock_bh(&vscsi->intr_lock); goto fail; } return; @@ -2638,6 +2635,9 @@ static void ibmvscsis_parse_task(struct scsi_info *vscsi, if (rc) { dev_err(&vscsi->dev, "target_submit_tmr failed, rc %d\n", rc); + spin_lock_bh(&vscsi->intr_lock); + list_del(&cmd->list); + spin_unlock_bh(&vscsi->intr_lock); cmd->se_cmd.se_tmr_req->response = TMR_FUNCTION_REJECTED; } @@ -2786,36 +2786,6 @@ static irqreturn_t ibmvscsis_interrupt(int dummy, void *data) } /** - * ibmvscsis_check_q() - Helper function to Check Init Message Valid - * @vscsi: Pointer to our adapter structure - * - * Checks if a initialize message was queued by the initiatior - * while the timing window was open. This function is called from - * probe after the CRQ is created and interrupts are enabled. - * It would only be used by adapters who wait for some event before - * completing the init handshake with the client. For ibmvscsi, this - * event is waiting for the port to be enabled. - * - * EXECUTION ENVIRONMENT: - * Process level only, interrupt lock held - */ -static long ibmvscsis_check_q(struct scsi_info *vscsi) -{ - uint format; - long rc; - - rc = ibmvscsis_check_init_msg(vscsi, &format); - if (rc) - ibmvscsis_post_disconnect(vscsi, ERR_DISCONNECT_RECONNECT, 0); - else if (format == UNUSED_FORMAT) - vscsi->state = WAIT_ENABLED; - else - vscsi->state = PART_UP_WAIT_ENAB; - - return rc; -} - -/** * ibmvscsis_enable_change_state() - Set new state based on enabled status * @vscsi: Pointer to our adapter structure * @@ -2826,77 +2796,19 @@ static long ibmvscsis_check_q(struct scsi_info *vscsi) */ static long ibmvscsis_enable_change_state(struct scsi_info *vscsi) { + int bytes; long rc = ADAPT_SUCCESS; -handle_state_change: - switch (vscsi->state) { - case WAIT_ENABLED: - rc = ibmvscsis_send_init_message(vscsi, INIT_MSG); - switch (rc) { - case H_SUCCESS: - case H_DROPPED: - case H_CLOSED: - vscsi->state = WAIT_CONNECTION; - rc = ADAPT_SUCCESS; - break; - - case H_PARAMETER: - break; - - case H_HARDWARE: - break; - - default: - vscsi->state = UNDEFINED; - rc = H_HARDWARE; - break; - } - break; - case PART_UP_WAIT_ENAB: - rc = ibmvscsis_send_init_message(vscsi, INIT_COMPLETE_MSG); - switch (rc) { - case H_SUCCESS: - vscsi->state = CONNECTED; - rc = ADAPT_SUCCESS; - break; - - case H_DROPPED: - case H_CLOSED: - vscsi->state = WAIT_ENABLED; - goto handle_state_change; - - case H_PARAMETER: - break; - - case H_HARDWARE: - break; - - default: - rc = H_HARDWARE; - break; - } - break; - - case WAIT_CONNECTION: - case WAIT_IDLE: - case SRP_PROCESSING: - case CONNECTED: - rc = ADAPT_SUCCESS; - break; - /* should not be able to get here */ - case UNCONFIGURING: - rc = ERROR; - vscsi->state = UNDEFINED; - break; + bytes = vscsi->cmd_q.size * PAGE_SIZE; + rc = h_reg_crq(vscsi->dds.unit_id, vscsi->cmd_q.crq_token, bytes); + if (rc == H_CLOSED || rc == H_SUCCESS) { + vscsi->state = WAIT_CONNECTION; + rc = ibmvscsis_establish_new_q(vscsi); + } - /* driver should never allow this to happen */ - case ERR_DISCONNECT: - case ERR_DISCONNECT_RECONNECT: - default: - dev_err(&vscsi->dev, "in invalid state %d during enable_change_state\n", - vscsi->state); - rc = ADAPT_SUCCESS; - break; + if (rc != ADAPT_SUCCESS) { + vscsi->state = ERR_DISCONNECTED; + vscsi->flags |= RESPONSE_Q_DOWN; } return rc; @@ -2916,7 +2828,6 @@ handle_state_change: */ static long ibmvscsis_create_command_q(struct scsi_info *vscsi, int num_cmds) { - long rc = 0; int pages; struct vio_dev *vdev = vscsi->dma_dev; @@ -2940,22 +2851,7 @@ static long ibmvscsis_create_command_q(struct scsi_info *vscsi, int num_cmds) return -ENOMEM; } - rc = h_reg_crq(vscsi->dds.unit_id, vscsi->cmd_q.crq_token, PAGE_SIZE); - if (rc) { - if (rc == H_CLOSED) { - vscsi->state = WAIT_ENABLED; - rc = 0; - } else { - dma_unmap_single(&vdev->dev, vscsi->cmd_q.crq_token, - PAGE_SIZE, DMA_BIDIRECTIONAL); - free_page((unsigned long)vscsi->cmd_q.base_addr); - rc = -ENODEV; - } - } else { - vscsi->state = WAIT_ENABLED; - } - - return rc; + return 0; } /** @@ -3270,7 +3166,7 @@ static void ibmvscsis_handle_crq(unsigned long data) /* * if we are in a path where we are waiting for all pending commands * to complete because we received a transport event and anything in - * the command queue is for a new connection, do nothing + * the command queue is for a new connection, do nothing */ if (TARGET_STOP(vscsi)) { vio_enable_interrupts(vscsi->dma_dev); @@ -3314,7 +3210,7 @@ cmd_work: * everything but transport events on the queue * * need to decrement the queue index so we can - * look at the elment again + * look at the element again */ if (vscsi->cmd_q.index) vscsi->cmd_q.index -= 1; @@ -3378,7 +3274,8 @@ static int ibmvscsis_probe(struct vio_dev *vdev, INIT_LIST_HEAD(&vscsi->waiting_rsp); INIT_LIST_HEAD(&vscsi->active_q); - snprintf(vscsi->tport.tport_name, 256, "%s", dev_name(&vdev->dev)); + snprintf(vscsi->tport.tport_name, IBMVSCSIS_NAMELEN, "%s", + dev_name(&vdev->dev)); pr_debug("probe tport_name: %s\n", vscsi->tport.tport_name); @@ -3393,6 +3290,9 @@ static int ibmvscsis_probe(struct vio_dev *vdev, strncat(vscsi->eye, vdev->name, MAX_EYE); vscsi->dds.unit_id = vdev->unit_address; + strncpy(vscsi->dds.partition_name, partition_name, + sizeof(vscsi->dds.partition_name)); + vscsi->dds.partition_num = partition_number; spin_lock_bh(&ibmvscsis_dev_lock); list_add_tail(&vscsi->list, &ibmvscsis_dev_list); @@ -3469,6 +3369,7 @@ static int ibmvscsis_probe(struct vio_dev *vdev, (unsigned long)vscsi); init_completion(&vscsi->wait_idle); + init_completion(&vscsi->unconfig); snprintf(wq_name, 24, "ibmvscsis%s", dev_name(&vdev->dev)); vscsi->work_q = create_workqueue(wq_name); @@ -3485,31 +3386,12 @@ static int ibmvscsis_probe(struct vio_dev *vdev, goto destroy_WQ; } - spin_lock_bh(&vscsi->intr_lock); - vio_enable_interrupts(vdev); - if (rc) { - dev_err(&vscsi->dev, "enabling interrupts failed, rc %d\n", rc); - rc = -ENODEV; - spin_unlock_bh(&vscsi->intr_lock); - goto free_irq; - } - - if (ibmvscsis_check_q(vscsi)) { - rc = ERROR; - dev_err(&vscsi->dev, "probe: check_q failed, rc %d\n", rc); - spin_unlock_bh(&vscsi->intr_lock); - goto disable_interrupt; - } - spin_unlock_bh(&vscsi->intr_lock); + vscsi->state = WAIT_ENABLED; dev_set_drvdata(&vdev->dev, vscsi); return 0; -disable_interrupt: - vio_disable_interrupts(vdev); -free_irq: - free_irq(vdev->irq, vscsi); destroy_WQ: destroy_workqueue(vscsi->work_q); unmap_buf: @@ -3543,10 +3425,11 @@ static int ibmvscsis_remove(struct vio_dev *vdev) pr_debug("remove (%s)\n", dev_name(&vscsi->dma_dev->dev)); - /* - * TBD: Need to handle if there are commands on the waiting_rsp q - * Actually, can there still be cmds outstanding to tcm? - */ + spin_lock_bh(&vscsi->intr_lock); + ibmvscsis_post_disconnect(vscsi, UNCONFIGURING, 0); + vscsi->flags |= CFG_SLEEPING; + spin_unlock_bh(&vscsi->intr_lock); + wait_for_completion(&vscsi->unconfig); vio_disable_interrupts(vdev); free_irq(vdev->irq, vscsi); @@ -3555,7 +3438,6 @@ static int ibmvscsis_remove(struct vio_dev *vdev) DMA_BIDIRECTIONAL); kfree(vscsi->map_buf); tasklet_kill(&vscsi->work_task); - ibmvscsis_unregister_command_q(vscsi); ibmvscsis_destroy_command_q(vscsi); ibmvscsis_freetimer(vscsi); ibmvscsis_free_cmds(vscsi); @@ -3609,7 +3491,7 @@ static int ibmvscsis_get_system_info(void) num = of_get_property(rootdn, "ibm,partition-no", NULL); if (num) - partition_number = *num; + partition_number = of_read_number(num, 1); of_node_put(rootdn); @@ -3903,18 +3785,22 @@ static ssize_t ibmvscsis_tpg_enable_store(struct config_item *item, } if (tmp) { - tport->enabled = true; spin_lock_bh(&vscsi->intr_lock); + tport->enabled = true; lrc = ibmvscsis_enable_change_state(vscsi); if (lrc) pr_err("enable_change_state failed, rc %ld state %d\n", lrc, vscsi->state); spin_unlock_bh(&vscsi->intr_lock); } else { + spin_lock_bh(&vscsi->intr_lock); tport->enabled = false; + /* This simulates the server going down */ + ibmvscsis_post_disconnect(vscsi, ERR_DISCONNECT, 0); + spin_unlock_bh(&vscsi->intr_lock); } - pr_debug("tpg_enable_store, state %d\n", vscsi->state); + pr_debug("tpg_enable_store, tmp %ld, state %d\n", tmp, vscsi->state); return count; } @@ -3983,10 +3869,10 @@ static struct attribute *ibmvscsis_dev_attrs[] = { ATTRIBUTE_GROUPS(ibmvscsis_dev); static struct class ibmvscsis_class = { - .name = "ibmvscsis", - .dev_release = ibmvscsis_dev_release, - .class_attrs = ibmvscsis_class_attrs, - .dev_groups = ibmvscsis_dev_groups, + .name = "ibmvscsis", + .dev_release = ibmvscsis_dev_release, + .class_attrs = ibmvscsis_class_attrs, + .dev_groups = ibmvscsis_dev_groups, }; static struct vio_device_id ibmvscsis_device_table[] = { diff --git a/drivers/scsi/ibmvscsi_tgt/ibmvscsi_tgt.h b/drivers/scsi/ibmvscsi_tgt/ibmvscsi_tgt.h index 981a0c9..98b0ca7 100644 --- a/drivers/scsi/ibmvscsi_tgt/ibmvscsi_tgt.h +++ b/drivers/scsi/ibmvscsi_tgt/ibmvscsi_tgt.h @@ -204,8 +204,6 @@ struct scsi_info { struct list_head waiting_rsp; #define NO_QUEUE 0x00 #define WAIT_ENABLED 0X01 - /* driver has received an initialize command */ -#define PART_UP_WAIT_ENAB 0x02 #define WAIT_CONNECTION 0x04 /* have established a connection */ #define CONNECTED 0x08 @@ -259,6 +257,8 @@ struct scsi_info { #define SCHEDULE_DISCONNECT 0x00400 /* disconnect handler is scheduled */ #define DISCONNECT_SCHEDULED 0x00800 + /* remove function is sleeping */ +#define CFG_SLEEPING 0x01000 u32 flags; /* adapter lock */ spinlock_t intr_lock; @@ -287,6 +287,7 @@ struct scsi_info { struct workqueue_struct *work_q; struct completion wait_idle; + struct completion unconfig; struct device dev; struct vio_dev *dma_dev; struct srp_target target; diff --git a/drivers/scsi/ipr.c b/drivers/scsi/ipr.c index 5324741..835c59c 100644 --- a/drivers/scsi/ipr.c +++ b/drivers/scsi/ipr.c @@ -186,16 +186,16 @@ static const struct ipr_chip_cfg_t ipr_chip_cfg[] = { }; static const struct ipr_chip_t ipr_chip[] = { - { PCI_VENDOR_ID_MYLEX, PCI_DEVICE_ID_IBM_GEMSTONE, IPR_USE_LSI, IPR_SIS32, IPR_PCI_CFG, &ipr_chip_cfg[0] }, - { PCI_VENDOR_ID_IBM, PCI_DEVICE_ID_IBM_CITRINE, IPR_USE_LSI, IPR_SIS32, IPR_PCI_CFG, &ipr_chip_cfg[0] }, - { PCI_VENDOR_ID_ADAPTEC2, PCI_DEVICE_ID_ADAPTEC2_OBSIDIAN, IPR_USE_LSI, IPR_SIS32, IPR_PCI_CFG, &ipr_chip_cfg[0] }, - { PCI_VENDOR_ID_IBM, PCI_DEVICE_ID_IBM_OBSIDIAN, IPR_USE_LSI, IPR_SIS32, IPR_PCI_CFG, &ipr_chip_cfg[0] }, - { PCI_VENDOR_ID_IBM, PCI_DEVICE_ID_IBM_OBSIDIAN_E, IPR_USE_MSI, IPR_SIS32, IPR_PCI_CFG, &ipr_chip_cfg[0] }, - { PCI_VENDOR_ID_IBM, PCI_DEVICE_ID_IBM_SNIPE, IPR_USE_LSI, IPR_SIS32, IPR_PCI_CFG, &ipr_chip_cfg[1] }, - { PCI_VENDOR_ID_ADAPTEC2, PCI_DEVICE_ID_ADAPTEC2_SCAMP, IPR_USE_LSI, IPR_SIS32, IPR_PCI_CFG, &ipr_chip_cfg[1] }, - { PCI_VENDOR_ID_IBM, PCI_DEVICE_ID_IBM_CROC_FPGA_E2, IPR_USE_MSI, IPR_SIS64, IPR_MMIO, &ipr_chip_cfg[2] }, - { PCI_VENDOR_ID_IBM, PCI_DEVICE_ID_IBM_CROCODILE, IPR_USE_MSI, IPR_SIS64, IPR_MMIO, &ipr_chip_cfg[2] }, - { PCI_VENDOR_ID_IBM, PCI_DEVICE_ID_IBM_RATTLESNAKE, IPR_USE_MSI, IPR_SIS64, IPR_MMIO, &ipr_chip_cfg[2] } + { PCI_VENDOR_ID_MYLEX, PCI_DEVICE_ID_IBM_GEMSTONE, false, IPR_SIS32, IPR_PCI_CFG, &ipr_chip_cfg[0] }, + { PCI_VENDOR_ID_IBM, PCI_DEVICE_ID_IBM_CITRINE, false, IPR_SIS32, IPR_PCI_CFG, &ipr_chip_cfg[0] }, + { PCI_VENDOR_ID_ADAPTEC2, PCI_DEVICE_ID_ADAPTEC2_OBSIDIAN, false, IPR_SIS32, IPR_PCI_CFG, &ipr_chip_cfg[0] }, + { PCI_VENDOR_ID_IBM, PCI_DEVICE_ID_IBM_OBSIDIAN, false, IPR_SIS32, IPR_PCI_CFG, &ipr_chip_cfg[0] }, + { PCI_VENDOR_ID_IBM, PCI_DEVICE_ID_IBM_OBSIDIAN_E, true, IPR_SIS32, IPR_PCI_CFG, &ipr_chip_cfg[0] }, + { PCI_VENDOR_ID_IBM, PCI_DEVICE_ID_IBM_SNIPE, false, IPR_SIS32, IPR_PCI_CFG, &ipr_chip_cfg[1] }, + { PCI_VENDOR_ID_ADAPTEC2, PCI_DEVICE_ID_ADAPTEC2_SCAMP, false, IPR_SIS32, IPR_PCI_CFG, &ipr_chip_cfg[1] }, + { PCI_VENDOR_ID_IBM, PCI_DEVICE_ID_IBM_CROC_FPGA_E2, true, IPR_SIS64, IPR_MMIO, &ipr_chip_cfg[2] }, + { PCI_VENDOR_ID_IBM, PCI_DEVICE_ID_IBM_CROCODILE, true, IPR_SIS64, IPR_MMIO, &ipr_chip_cfg[2] }, + { PCI_VENDOR_ID_IBM, PCI_DEVICE_ID_IBM_RATTLESNAKE, true, IPR_SIS64, IPR_MMIO, &ipr_chip_cfg[2] } }; static int ipr_max_bus_speeds[] = { @@ -9439,23 +9439,11 @@ static void ipr_free_mem(struct ipr_ioa_cfg *ioa_cfg) static void ipr_free_irqs(struct ipr_ioa_cfg *ioa_cfg) { struct pci_dev *pdev = ioa_cfg->pdev; + int i; - if (ioa_cfg->intr_flag == IPR_USE_MSI || - ioa_cfg->intr_flag == IPR_USE_MSIX) { - int i; - for (i = 0; i < ioa_cfg->nvectors; i++) - free_irq(ioa_cfg->vectors_info[i].vec, - &ioa_cfg->hrrq[i]); - } else - free_irq(pdev->irq, &ioa_cfg->hrrq[0]); - - if (ioa_cfg->intr_flag == IPR_USE_MSI) { - pci_disable_msi(pdev); - ioa_cfg->intr_flag &= ~IPR_USE_MSI; - } else if (ioa_cfg->intr_flag == IPR_USE_MSIX) { - pci_disable_msix(pdev); - ioa_cfg->intr_flag &= ~IPR_USE_MSIX; - } + for (i = 0; i < ioa_cfg->nvectors; i++) + free_irq(pci_irq_vector(pdev, i), &ioa_cfg->hrrq[i]); + pci_free_irq_vectors(pdev); } /** @@ -9883,45 +9871,6 @@ static void ipr_wait_for_pci_err_recovery(struct ipr_ioa_cfg *ioa_cfg) } } -static int ipr_enable_msix(struct ipr_ioa_cfg *ioa_cfg) -{ - struct msix_entry entries[IPR_MAX_MSIX_VECTORS]; - int i, vectors; - - for (i = 0; i < ARRAY_SIZE(entries); ++i) - entries[i].entry = i; - - vectors = pci_enable_msix_range(ioa_cfg->pdev, - entries, 1, ipr_number_of_msix); - if (vectors < 0) { - ipr_wait_for_pci_err_recovery(ioa_cfg); - return vectors; - } - - for (i = 0; i < vectors; i++) - ioa_cfg->vectors_info[i].vec = entries[i].vector; - ioa_cfg->nvectors = vectors; - - return 0; -} - -static int ipr_enable_msi(struct ipr_ioa_cfg *ioa_cfg) -{ - int i, vectors; - - vectors = pci_enable_msi_range(ioa_cfg->pdev, 1, ipr_number_of_msix); - if (vectors < 0) { - ipr_wait_for_pci_err_recovery(ioa_cfg); - return vectors; - } - - for (i = 0; i < vectors; i++) - ioa_cfg->vectors_info[i].vec = ioa_cfg->pdev->irq + i; - ioa_cfg->nvectors = vectors; - - return 0; -} - static void name_msi_vectors(struct ipr_ioa_cfg *ioa_cfg) { int vec_idx, n = sizeof(ioa_cfg->vectors_info[0].desc) - 1; @@ -9934,19 +9883,20 @@ static void name_msi_vectors(struct ipr_ioa_cfg *ioa_cfg) } } -static int ipr_request_other_msi_irqs(struct ipr_ioa_cfg *ioa_cfg) +static int ipr_request_other_msi_irqs(struct ipr_ioa_cfg *ioa_cfg, + struct pci_dev *pdev) { int i, rc; for (i = 1; i < ioa_cfg->nvectors; i++) { - rc = request_irq(ioa_cfg->vectors_info[i].vec, + rc = request_irq(pci_irq_vector(pdev, i), ipr_isr_mhrrq, 0, ioa_cfg->vectors_info[i].desc, &ioa_cfg->hrrq[i]); if (rc) { while (--i >= 0) - free_irq(ioa_cfg->vectors_info[i].vec, + free_irq(pci_irq_vector(pdev, i), &ioa_cfg->hrrq[i]); return rc; } @@ -9984,8 +9934,7 @@ static irqreturn_t ipr_test_intr(int irq, void *devp) * ipr_test_msi - Test for Message Signaled Interrupt (MSI) support. * @pdev: PCI device struct * - * Description: The return value from pci_enable_msi_range() can not always be - * trusted. This routine sets up and initiates a test interrupt to determine + * Description: This routine sets up and initiates a test interrupt to determine * if the interrupt is received via the ipr_test_intr() service routine. * If the tests fails, the driver will fall back to LSI. * @@ -9997,6 +9946,7 @@ static int ipr_test_msi(struct ipr_ioa_cfg *ioa_cfg, struct pci_dev *pdev) int rc; volatile u32 int_reg; unsigned long lock_flags = 0; + int irq = pci_irq_vector(pdev, 0); ENTER; @@ -10008,15 +9958,12 @@ static int ipr_test_msi(struct ipr_ioa_cfg *ioa_cfg, struct pci_dev *pdev) int_reg = readl(ioa_cfg->regs.sense_interrupt_mask_reg); spin_unlock_irqrestore(ioa_cfg->host->host_lock, lock_flags); - if (ioa_cfg->intr_flag == IPR_USE_MSIX) - rc = request_irq(ioa_cfg->vectors_info[0].vec, ipr_test_intr, 0, IPR_NAME, ioa_cfg); - else - rc = request_irq(pdev->irq, ipr_test_intr, 0, IPR_NAME, ioa_cfg); + rc = request_irq(irq, ipr_test_intr, 0, IPR_NAME, ioa_cfg); if (rc) { - dev_err(&pdev->dev, "Can not assign irq %d\n", pdev->irq); + dev_err(&pdev->dev, "Can not assign irq %d\n", irq); return rc; } else if (ipr_debug) - dev_info(&pdev->dev, "IRQ assigned: %d\n", pdev->irq); + dev_info(&pdev->dev, "IRQ assigned: %d\n", irq); writel(IPR_PCII_IO_DEBUG_ACKNOWLEDGE, ioa_cfg->regs.sense_interrupt_reg32); int_reg = readl(ioa_cfg->regs.sense_interrupt_reg); @@ -10033,10 +9980,7 @@ static int ipr_test_msi(struct ipr_ioa_cfg *ioa_cfg, struct pci_dev *pdev) spin_unlock_irqrestore(ioa_cfg->host->host_lock, lock_flags); - if (ioa_cfg->intr_flag == IPR_USE_MSIX) - free_irq(ioa_cfg->vectors_info[0].vec, ioa_cfg); - else - free_irq(pdev->irq, ioa_cfg); + free_irq(irq, ioa_cfg); LEAVE; @@ -10060,6 +10004,7 @@ static int ipr_probe_ioa(struct pci_dev *pdev, int rc = PCIBIOS_SUCCESSFUL; volatile u32 mask, uproc, interrupts; unsigned long lock_flags, driver_lock_flags; + unsigned int irq_flag; ENTER; @@ -10175,18 +10120,18 @@ static int ipr_probe_ioa(struct pci_dev *pdev, ipr_number_of_msix = IPR_MAX_MSIX_VECTORS; } - if (ioa_cfg->ipr_chip->intr_type == IPR_USE_MSI && - ipr_enable_msix(ioa_cfg) == 0) - ioa_cfg->intr_flag = IPR_USE_MSIX; - else if (ioa_cfg->ipr_chip->intr_type == IPR_USE_MSI && - ipr_enable_msi(ioa_cfg) == 0) - ioa_cfg->intr_flag = IPR_USE_MSI; - else { - ioa_cfg->intr_flag = IPR_USE_LSI; - ioa_cfg->clear_isr = 1; - ioa_cfg->nvectors = 1; - dev_info(&pdev->dev, "Cannot enable MSI.\n"); + irq_flag = PCI_IRQ_LEGACY; + if (ioa_cfg->ipr_chip->has_msi) + irq_flag |= PCI_IRQ_MSI | PCI_IRQ_MSIX; + rc = pci_alloc_irq_vectors(pdev, 1, ipr_number_of_msix, irq_flag); + if (rc < 0) { + ipr_wait_for_pci_err_recovery(ioa_cfg); + goto cleanup_nomem; } + ioa_cfg->nvectors = rc; + + if (!pdev->msi_enabled && !pdev->msix_enabled) + ioa_cfg->clear_isr = 1; pci_set_master(pdev); @@ -10199,33 +10144,23 @@ static int ipr_probe_ioa(struct pci_dev *pdev, } } - if (ioa_cfg->intr_flag == IPR_USE_MSI || - ioa_cfg->intr_flag == IPR_USE_MSIX) { + if (pdev->msi_enabled || pdev->msix_enabled) { rc = ipr_test_msi(ioa_cfg, pdev); - if (rc == -EOPNOTSUPP) { + switch (rc) { + case 0: + dev_info(&pdev->dev, + "Request for %d MSI%ss succeeded.", ioa_cfg->nvectors, + pdev->msix_enabled ? "-X" : ""); + break; + case -EOPNOTSUPP: ipr_wait_for_pci_err_recovery(ioa_cfg); - if (ioa_cfg->intr_flag == IPR_USE_MSI) { - ioa_cfg->intr_flag &= ~IPR_USE_MSI; - pci_disable_msi(pdev); - } else if (ioa_cfg->intr_flag == IPR_USE_MSIX) { - ioa_cfg->intr_flag &= ~IPR_USE_MSIX; - pci_disable_msix(pdev); - } + pci_free_irq_vectors(pdev); - ioa_cfg->intr_flag = IPR_USE_LSI; ioa_cfg->nvectors = 1; - } - else if (rc) + ioa_cfg->clear_isr = 1; + break; + default: goto out_msi_disable; - else { - if (ioa_cfg->intr_flag == IPR_USE_MSI) - dev_info(&pdev->dev, - "Request for %d MSIs succeeded with starting IRQ: %d\n", - ioa_cfg->nvectors, pdev->irq); - else if (ioa_cfg->intr_flag == IPR_USE_MSIX) - dev_info(&pdev->dev, - "Request for %d MSIXs succeeded.", - ioa_cfg->nvectors); } } @@ -10273,15 +10208,13 @@ static int ipr_probe_ioa(struct pci_dev *pdev, ipr_mask_and_clear_interrupts(ioa_cfg, ~IPR_PCII_IOA_TRANS_TO_OPER); spin_unlock_irqrestore(ioa_cfg->host->host_lock, lock_flags); - if (ioa_cfg->intr_flag == IPR_USE_MSI - || ioa_cfg->intr_flag == IPR_USE_MSIX) { + if (pdev->msi_enabled || pdev->msix_enabled) { name_msi_vectors(ioa_cfg); - rc = request_irq(ioa_cfg->vectors_info[0].vec, ipr_isr, - 0, + rc = request_irq(pci_irq_vector(pdev, 0), ipr_isr, 0, ioa_cfg->vectors_info[0].desc, &ioa_cfg->hrrq[0]); if (!rc) - rc = ipr_request_other_msi_irqs(ioa_cfg); + rc = ipr_request_other_msi_irqs(ioa_cfg, pdev); } else { rc = request_irq(pdev->irq, ipr_isr, IRQF_SHARED, @@ -10323,10 +10256,7 @@ cleanup_nolog: ipr_free_mem(ioa_cfg); out_msi_disable: ipr_wait_for_pci_err_recovery(ioa_cfg); - if (ioa_cfg->intr_flag == IPR_USE_MSI) - pci_disable_msi(pdev); - else if (ioa_cfg->intr_flag == IPR_USE_MSIX) - pci_disable_msix(pdev); + pci_free_irq_vectors(pdev); cleanup_nomem: iounmap(ipr_regs); out_disable: diff --git a/drivers/scsi/ipr.h b/drivers/scsi/ipr.h index 8995053..b7d2e98 100644 --- a/drivers/scsi/ipr.h +++ b/drivers/scsi/ipr.h @@ -1413,10 +1413,7 @@ struct ipr_chip_cfg_t { struct ipr_chip_t { u16 vendor; u16 device; - u16 intr_type; -#define IPR_USE_LSI 0x00 -#define IPR_USE_MSI 0x01 -#define IPR_USE_MSIX 0x02 + bool has_msi; u16 sis_type; #define IPR_SIS32 0x00 #define IPR_SIS64 0x01 @@ -1593,11 +1590,9 @@ struct ipr_ioa_cfg { struct ipr_cmnd **ipr_cmnd_list; dma_addr_t *ipr_cmnd_list_dma; - u16 intr_flag; unsigned int nvectors; struct { - unsigned short vec; char desc[22]; } vectors_info[IPR_MAX_MSIX_VECTORS]; diff --git a/drivers/scsi/ips.c b/drivers/scsi/ips.c index 02cb76f..3419e1b 100644 --- a/drivers/scsi/ips.c +++ b/drivers/scsi/ips.c @@ -2241,9 +2241,6 @@ ips_get_bios_version(ips_ha_t * ha, int intr) uint8_t minor; uint8_t subminor; uint8_t *buffer; - char hexDigits[] = - { '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'A', 'B', 'C', - 'D', 'E', 'F' }; METHOD_TRACE("ips_get_bios_version", 1); @@ -2374,13 +2371,13 @@ ips_get_bios_version(ips_ha_t * ha, int intr) } } - ha->bios_version[0] = hexDigits[(major & 0xF0) >> 4]; + ha->bios_version[0] = hex_asc_upper_hi(major); ha->bios_version[1] = '.'; - ha->bios_version[2] = hexDigits[major & 0x0F]; - ha->bios_version[3] = hexDigits[subminor]; + ha->bios_version[2] = hex_asc_upper_lo(major); + ha->bios_version[3] = hex_asc_upper_lo(subminor); ha->bios_version[4] = '.'; - ha->bios_version[5] = hexDigits[(minor & 0xF0) >> 4]; - ha->bios_version[6] = hexDigits[minor & 0x0F]; + ha->bios_version[5] = hex_asc_upper_hi(minor); + ha->bios_version[6] = hex_asc_upper_lo(minor); ha->bios_version[7] = 0; } diff --git a/drivers/scsi/isci/host.h b/drivers/scsi/isci/host.h index 22a9bb1..b353992 100644 --- a/drivers/scsi/isci/host.h +++ b/drivers/scsi/isci/host.h @@ -295,7 +295,6 @@ enum sci_controller_states { #define SCI_MAX_MSIX_INT (SCI_NUM_MSI_X_INT*SCI_MAX_CONTROLLERS) struct isci_pci_info { - struct msix_entry msix_entries[SCI_MAX_MSIX_INT]; struct isci_host *hosts[SCI_MAX_CONTROLLERS]; struct isci_orom *orom; }; diff --git a/drivers/scsi/isci/init.c b/drivers/scsi/isci/init.c index 77128d68..0b5b5db 100644 --- a/drivers/scsi/isci/init.c +++ b/drivers/scsi/isci/init.c @@ -350,16 +350,12 @@ static int isci_setup_interrupts(struct pci_dev *pdev) */ num_msix = num_controllers(pdev) * SCI_NUM_MSI_X_INT; - for (i = 0; i < num_msix; i++) - pci_info->msix_entries[i].entry = i; - - err = pci_enable_msix_exact(pdev, pci_info->msix_entries, num_msix); - if (err) + err = pci_alloc_irq_vectors(pdev, num_msix, num_msix, PCI_IRQ_MSIX); + if (err < 0) goto intx; for (i = 0; i < num_msix; i++) { int id = i / SCI_NUM_MSI_X_INT; - struct msix_entry *msix = &pci_info->msix_entries[i]; irq_handler_t isr; ihost = pci_info->hosts[id]; @@ -369,8 +365,8 @@ static int isci_setup_interrupts(struct pci_dev *pdev) else isr = isci_msix_isr; - err = devm_request_irq(&pdev->dev, msix->vector, isr, 0, - DRV_NAME"-msix", ihost); + err = devm_request_irq(&pdev->dev, pci_irq_vector(pdev, i), + isr, 0, DRV_NAME"-msix", ihost); if (!err) continue; @@ -378,18 +374,19 @@ static int isci_setup_interrupts(struct pci_dev *pdev) while (i--) { id = i / SCI_NUM_MSI_X_INT; ihost = pci_info->hosts[id]; - msix = &pci_info->msix_entries[i]; - devm_free_irq(&pdev->dev, msix->vector, ihost); + devm_free_irq(&pdev->dev, pci_irq_vector(pdev, i), + ihost); } - pci_disable_msix(pdev); + pci_free_irq_vectors(pdev); goto intx; } return 0; intx: for_each_isci_host(i, ihost, pdev) { - err = devm_request_irq(&pdev->dev, pdev->irq, isci_intx_isr, - IRQF_SHARED, DRV_NAME"-intx", ihost); + err = devm_request_irq(&pdev->dev, pci_irq_vector(pdev, 0), + isci_intx_isr, IRQF_SHARED, DRV_NAME"-intx", + ihost); if (err) break; } diff --git a/drivers/scsi/isci/probe_roms.c b/drivers/scsi/isci/probe_roms.c index 8ac646e..a2bbe46 100644 --- a/drivers/scsi/isci/probe_roms.c +++ b/drivers/scsi/isci/probe_roms.c @@ -54,6 +54,7 @@ struct isci_orom *isci_request_oprom(struct pci_dev *pdev) len = pci_biosrom_size(pdev); rom = devm_kzalloc(&pdev->dev, sizeof(*rom), GFP_KERNEL); if (!rom) { + pci_unmap_biosrom(oprom); dev_warn(&pdev->dev, "Unable to allocate memory for orom\n"); return NULL; diff --git a/drivers/scsi/isci/remote_node_context.c b/drivers/scsi/isci/remote_node_context.c index 1910100..e3f2a53 100644 --- a/drivers/scsi/isci/remote_node_context.c +++ b/drivers/scsi/isci/remote_node_context.c @@ -66,6 +66,9 @@ const char *rnc_state_name(enum scis_sds_remote_node_context_states state) { static const char * const strings[] = RNC_STATES; + if (state >= ARRAY_SIZE(strings)) + return "UNKNOWN"; + return strings[state]; } #undef C @@ -454,7 +457,7 @@ enum sci_status sci_remote_node_context_event_handler(struct sci_remote_node_con * the device since it's being invalidated anyway */ dev_warn(scirdev_to_dev(rnc_to_dev(sci_rnc)), "%s: SCIC Remote Node Context 0x%p was " - "suspeneded by hardware while being " + "suspended by hardware while being " "invalidated.\n", __func__, sci_rnc); break; default: @@ -473,7 +476,7 @@ enum sci_status sci_remote_node_context_event_handler(struct sci_remote_node_con * the device since it's being resumed anyway */ dev_warn(scirdev_to_dev(rnc_to_dev(sci_rnc)), "%s: SCIC Remote Node Context 0x%p was " - "suspeneded by hardware while being resumed.\n", + "suspended by hardware while being resumed.\n", __func__, sci_rnc); break; default: diff --git a/drivers/scsi/isci/request.c b/drivers/scsi/isci/request.c index b709d2b..47f66e9 100644 --- a/drivers/scsi/isci/request.c +++ b/drivers/scsi/isci/request.c @@ -2473,7 +2473,7 @@ static void isci_request_process_response_iu( "%s: resp_iu = %p " "resp_iu->status = 0x%x,\nresp_iu->datapres = %d " "resp_iu->response_data_len = %x, " - "resp_iu->sense_data_len = %x\nrepsonse data: ", + "resp_iu->sense_data_len = %x\nresponse data: ", __func__, resp_iu, resp_iu->status, diff --git a/drivers/scsi/libfc/fc_disc.c b/drivers/scsi/libfc/fc_disc.c index 880a906..6103231 100644 --- a/drivers/scsi/libfc/fc_disc.c +++ b/drivers/scsi/libfc/fc_disc.c @@ -68,10 +68,14 @@ static void fc_disc_stop_rports(struct fc_disc *disc) lport = fc_disc_lport(disc); - mutex_lock(&disc->disc_mutex); - list_for_each_entry_rcu(rdata, &disc->rports, peers) - lport->tt.rport_logoff(rdata); - mutex_unlock(&disc->disc_mutex); + rcu_read_lock(); + list_for_each_entry_rcu(rdata, &disc->rports, peers) { + if (kref_get_unless_zero(&rdata->kref)) { + fc_rport_logoff(rdata); + kref_put(&rdata->kref, fc_rport_destroy); + } + } + rcu_read_unlock(); } /** @@ -150,7 +154,7 @@ static void fc_disc_recv_rscn_req(struct fc_disc *disc, struct fc_frame *fp) break; } } - lport->tt.seq_els_rsp_send(fp, ELS_LS_ACC, NULL); + fc_seq_els_rsp_send(fp, ELS_LS_ACC, NULL); /* * If not doing a complete rediscovery, do GPN_ID on @@ -178,7 +182,7 @@ reject: FC_DISC_DBG(disc, "Received a bad RSCN frame\n"); rjt_data.reason = ELS_RJT_LOGIC; rjt_data.explan = ELS_EXPL_NONE; - lport->tt.seq_els_rsp_send(fp, ELS_LS_RJT, &rjt_data); + fc_seq_els_rsp_send(fp, ELS_LS_RJT, &rjt_data); fc_frame_free(fp); } @@ -289,15 +293,19 @@ static void fc_disc_done(struct fc_disc *disc, enum fc_disc_event event) * Skip ports which were never discovered. These are the dNS port * and ports which were created by PLOGI. */ + rcu_read_lock(); list_for_each_entry_rcu(rdata, &disc->rports, peers) { - if (!rdata->disc_id) + if (!kref_get_unless_zero(&rdata->kref)) continue; - if (rdata->disc_id == disc->disc_id) - lport->tt.rport_login(rdata); - else - lport->tt.rport_logoff(rdata); + if (rdata->disc_id) { + if (rdata->disc_id == disc->disc_id) + fc_rport_login(rdata); + else + fc_rport_logoff(rdata); + } + kref_put(&rdata->kref, fc_rport_destroy); } - + rcu_read_unlock(); mutex_unlock(&disc->disc_mutex); disc->disc_callback(lport, event); mutex_lock(&disc->disc_mutex); @@ -446,7 +454,7 @@ static int fc_disc_gpn_ft_parse(struct fc_disc *disc, void *buf, size_t len) if (ids.port_id != lport->port_id && ids.port_name != lport->wwpn) { - rdata = lport->tt.rport_create(lport, ids.port_id); + rdata = fc_rport_create(lport, ids.port_id); if (rdata) { rdata->ids.port_name = ids.port_name; rdata->disc_id = disc->disc_id; @@ -592,7 +600,6 @@ static void fc_disc_gpn_id_resp(struct fc_seq *sp, struct fc_frame *fp, lport = rdata->local_port; disc = &lport->disc; - mutex_lock(&disc->disc_mutex); if (PTR_ERR(fp) == -FC_EX_CLOSED) goto out; if (IS_ERR(fp)) @@ -607,37 +614,41 @@ static void fc_disc_gpn_id_resp(struct fc_seq *sp, struct fc_frame *fp, goto redisc; pn = (struct fc_ns_gid_pn *)(cp + 1); port_name = get_unaligned_be64(&pn->fn_wwpn); + mutex_lock(&rdata->rp_mutex); if (rdata->ids.port_name == -1) rdata->ids.port_name = port_name; else if (rdata->ids.port_name != port_name) { FC_DISC_DBG(disc, "GPN_ID accepted. WWPN changed. " "Port-id %6.6x wwpn %16.16llx\n", rdata->ids.port_id, port_name); - lport->tt.rport_logoff(rdata); - - new_rdata = lport->tt.rport_create(lport, - rdata->ids.port_id); + mutex_unlock(&rdata->rp_mutex); + fc_rport_logoff(rdata); + mutex_lock(&lport->disc.disc_mutex); + new_rdata = fc_rport_create(lport, rdata->ids.port_id); + mutex_unlock(&lport->disc.disc_mutex); if (new_rdata) { new_rdata->disc_id = disc->disc_id; - lport->tt.rport_login(new_rdata); + fc_rport_login(new_rdata); } goto out; } rdata->disc_id = disc->disc_id; - lport->tt.rport_login(rdata); + mutex_unlock(&rdata->rp_mutex); + fc_rport_login(rdata); } else if (ntohs(cp->ct_cmd) == FC_FS_RJT) { FC_DISC_DBG(disc, "GPN_ID rejected reason %x exp %x\n", cp->ct_reason, cp->ct_explan); - lport->tt.rport_logoff(rdata); + fc_rport_logoff(rdata); } else { FC_DISC_DBG(disc, "GPN_ID unexpected response code %x\n", ntohs(cp->ct_cmd)); redisc: + mutex_lock(&disc->disc_mutex); fc_disc_restart(disc); + mutex_unlock(&disc->disc_mutex); } out: - mutex_unlock(&disc->disc_mutex); - kref_put(&rdata->kref, lport->tt.rport_destroy); + kref_put(&rdata->kref, fc_rport_destroy); } /** @@ -678,7 +689,7 @@ static int fc_disc_single(struct fc_lport *lport, struct fc_disc_port *dp) { struct fc_rport_priv *rdata; - rdata = lport->tt.rport_create(lport, dp->port_id); + rdata = fc_rport_create(lport, dp->port_id); if (!rdata) return -ENOMEM; rdata->disc_id = 0; @@ -708,7 +719,7 @@ static void fc_disc_stop(struct fc_lport *lport) static void fc_disc_stop_final(struct fc_lport *lport) { fc_disc_stop(lport); - lport->tt.rport_flush_queue(); + fc_rport_flush_queue(); } /** diff --git a/drivers/scsi/libfc/fc_elsct.c b/drivers/scsi/libfc/fc_elsct.c index c2384d5..6384a98 100644 --- a/drivers/scsi/libfc/fc_elsct.c +++ b/drivers/scsi/libfc/fc_elsct.c @@ -67,7 +67,7 @@ struct fc_seq *fc_elsct_send(struct fc_lport *lport, u32 did, fc_fill_fc_hdr(fp, r_ctl, did, lport->port_id, fh_type, FC_FCTL_REQ, 0); - return lport->tt.exch_seq_send(lport, fp, resp, NULL, arg, timer_msec); + return fc_exch_seq_send(lport, fp, resp, NULL, arg, timer_msec); } EXPORT_SYMBOL(fc_elsct_send); diff --git a/drivers/scsi/libfc/fc_exch.c b/drivers/scsi/libfc/fc_exch.c index 16ca31a..42bcf7f 100644 --- a/drivers/scsi/libfc/fc_exch.c +++ b/drivers/scsi/libfc/fc_exch.c @@ -94,6 +94,7 @@ struct fc_exch_pool { struct fc_exch_mgr { struct fc_exch_pool __percpu *pool; mempool_t *ep_pool; + struct fc_lport *lport; enum fc_class class; struct kref kref; u16 min_xid; @@ -362,8 +363,10 @@ static inline void fc_exch_timer_set_locked(struct fc_exch *ep, fc_exch_hold(ep); /* hold for timer */ if (!queue_delayed_work(fc_exch_workqueue, &ep->timeout_work, - msecs_to_jiffies(timer_msec))) + msecs_to_jiffies(timer_msec))) { + FC_EXCH_DBG(ep, "Exchange already queued\n"); fc_exch_release(ep); + } } /** @@ -406,6 +409,8 @@ static int fc_exch_done_locked(struct fc_exch *ep) return rc; } +static struct fc_exch fc_quarantine_exch; + /** * fc_exch_ptr_get() - Return an exchange from an exchange pool * @pool: Exchange Pool to get an exchange from @@ -450,14 +455,17 @@ static void fc_exch_delete(struct fc_exch *ep) /* update cache of free slot */ index = (ep->xid - ep->em->min_xid) >> fc_cpu_order; - if (pool->left == FC_XID_UNKNOWN) - pool->left = index; - else if (pool->right == FC_XID_UNKNOWN) - pool->right = index; - else - pool->next_index = index; - - fc_exch_ptr_set(pool, index, NULL); + if (!(ep->state & FC_EX_QUARANTINE)) { + if (pool->left == FC_XID_UNKNOWN) + pool->left = index; + else if (pool->right == FC_XID_UNKNOWN) + pool->right = index; + else + pool->next_index = index; + fc_exch_ptr_set(pool, index, NULL); + } else { + fc_exch_ptr_set(pool, index, &fc_quarantine_exch); + } list_del(&ep->ex_list); spin_unlock_bh(&pool->lock); fc_exch_release(ep); /* drop hold for exch in mp */ @@ -525,8 +533,7 @@ out: * Note: The frame will be freed either by a direct call to fc_frame_free(fp) * or indirectly by calling libfc_function_template.frame_send(). */ -static int fc_seq_send(struct fc_lport *lport, struct fc_seq *sp, - struct fc_frame *fp) +int fc_seq_send(struct fc_lport *lport, struct fc_seq *sp, struct fc_frame *fp) { struct fc_exch *ep; int error; @@ -536,6 +543,7 @@ static int fc_seq_send(struct fc_lport *lport, struct fc_seq *sp, spin_unlock_bh(&ep->ex_lock); return error; } +EXPORT_SYMBOL(fc_seq_send); /** * fc_seq_alloc() - Allocate a sequence for a given exchange @@ -577,7 +585,7 @@ static struct fc_seq *fc_seq_start_next_locked(struct fc_seq *sp) * for a given sequence/exchange pair * @sp: The sequence/exchange to get a new exchange for */ -static struct fc_seq *fc_seq_start_next(struct fc_seq *sp) +struct fc_seq *fc_seq_start_next(struct fc_seq *sp) { struct fc_exch *ep = fc_seq_exch(sp); @@ -587,16 +595,16 @@ static struct fc_seq *fc_seq_start_next(struct fc_seq *sp) return sp; } +EXPORT_SYMBOL(fc_seq_start_next); /* * Set the response handler for the exchange associated with a sequence. * * Note: May sleep if invoked from outside a response handler. */ -static void fc_seq_set_resp(struct fc_seq *sp, - void (*resp)(struct fc_seq *, struct fc_frame *, - void *), - void *arg) +void fc_seq_set_resp(struct fc_seq *sp, + void (*resp)(struct fc_seq *, struct fc_frame *, void *), + void *arg) { struct fc_exch *ep = fc_seq_exch(sp); DEFINE_WAIT(wait); @@ -615,12 +623,20 @@ static void fc_seq_set_resp(struct fc_seq *sp, ep->arg = arg; spin_unlock_bh(&ep->ex_lock); } +EXPORT_SYMBOL(fc_seq_set_resp); /** * fc_exch_abort_locked() - Abort an exchange * @ep: The exchange to be aborted * @timer_msec: The period of time to wait before aborting * + * Abort an exchange and sequence. Generally called because of a + * exchange timeout or an abort from the upper layer. + * + * A timer_msec can be specified for abort timeout, if non-zero + * timer_msec value is specified then exchange resp handler + * will be called with timeout error if no response to abort. + * * Locking notes: Called with exch lock held * * Return value: 0 on success else error code @@ -632,9 +648,13 @@ static int fc_exch_abort_locked(struct fc_exch *ep, struct fc_frame *fp; int error; + FC_EXCH_DBG(ep, "exch: abort, time %d msecs\n", timer_msec); if (ep->esb_stat & (ESB_ST_COMPLETE | ESB_ST_ABNORMAL) || - ep->state & (FC_EX_DONE | FC_EX_RST_CLEANUP)) + ep->state & (FC_EX_DONE | FC_EX_RST_CLEANUP)) { + FC_EXCH_DBG(ep, "exch: already completed esb %x state %x\n", + ep->esb_stat, ep->state); return -ENXIO; + } /* * Send the abort on a new sequence if possible. @@ -680,8 +700,7 @@ static int fc_exch_abort_locked(struct fc_exch *ep, * * Return value: 0 on success else error code */ -static int fc_seq_exch_abort(const struct fc_seq *req_sp, - unsigned int timer_msec) +int fc_seq_exch_abort(const struct fc_seq *req_sp, unsigned int timer_msec) { struct fc_exch *ep; int error; @@ -758,7 +777,7 @@ static void fc_exch_timeout(struct work_struct *work) u32 e_stat; int rc = 1; - FC_EXCH_DBG(ep, "Exchange timed out\n"); + FC_EXCH_DBG(ep, "Exchange timed out state %x\n", ep->state); spin_lock_bh(&ep->ex_lock); if (ep->state & (FC_EX_RST_CLEANUP | FC_EX_DONE)) @@ -821,14 +840,18 @@ static struct fc_exch *fc_exch_em_alloc(struct fc_lport *lport, /* peek cache of free slot */ if (pool->left != FC_XID_UNKNOWN) { - index = pool->left; - pool->left = FC_XID_UNKNOWN; - goto hit; + if (!WARN_ON(fc_exch_ptr_get(pool, pool->left))) { + index = pool->left; + pool->left = FC_XID_UNKNOWN; + goto hit; + } } if (pool->right != FC_XID_UNKNOWN) { - index = pool->right; - pool->right = FC_XID_UNKNOWN; - goto hit; + if (!WARN_ON(fc_exch_ptr_get(pool, pool->right))) { + index = pool->right; + pool->right = FC_XID_UNKNOWN; + goto hit; + } } index = pool->next_index; @@ -888,14 +911,19 @@ err: * EM is selected when a NULL match function pointer is encountered * or when a call to a match function returns true. */ -static inline struct fc_exch *fc_exch_alloc(struct fc_lport *lport, - struct fc_frame *fp) +static struct fc_exch *fc_exch_alloc(struct fc_lport *lport, + struct fc_frame *fp) { struct fc_exch_mgr_anchor *ema; + struct fc_exch *ep; - list_for_each_entry(ema, &lport->ema_list, ema_list) - if (!ema->match || ema->match(fp)) - return fc_exch_em_alloc(lport, ema->mp); + list_for_each_entry(ema, &lport->ema_list, ema_list) { + if (!ema->match || ema->match(fp)) { + ep = fc_exch_em_alloc(lport, ema->mp); + if (ep) + return ep; + } + } return NULL; } @@ -906,14 +934,17 @@ static inline struct fc_exch *fc_exch_alloc(struct fc_lport *lport, */ static struct fc_exch *fc_exch_find(struct fc_exch_mgr *mp, u16 xid) { + struct fc_lport *lport = mp->lport; struct fc_exch_pool *pool; struct fc_exch *ep = NULL; u16 cpu = xid & fc_cpu_mask; + if (xid == FC_XID_UNKNOWN) + return NULL; + if (cpu >= nr_cpu_ids || !cpu_possible(cpu)) { - printk_ratelimited(KERN_ERR - "libfc: lookup request for XID = %d, " - "indicates invalid CPU %d\n", xid, cpu); + pr_err("host%u: lport %6.6x: xid %d invalid CPU %d\n:", + lport->host->host_no, lport->port_id, xid, cpu); return NULL; } @@ -921,6 +952,10 @@ static struct fc_exch *fc_exch_find(struct fc_exch_mgr *mp, u16 xid) pool = per_cpu_ptr(mp->pool, cpu); spin_lock_bh(&pool->lock); ep = fc_exch_ptr_get(pool, (xid - mp->min_xid) >> fc_cpu_order); + if (ep == &fc_quarantine_exch) { + FC_LPORT_DBG(lport, "xid %x quarantined\n", xid); + ep = NULL; + } if (ep) { WARN_ON(ep->xid != xid); fc_exch_hold(ep); @@ -938,7 +973,7 @@ static struct fc_exch *fc_exch_find(struct fc_exch_mgr *mp, u16 xid) * * Note: May sleep if invoked from outside a response handler. */ -static void fc_exch_done(struct fc_seq *sp) +void fc_exch_done(struct fc_seq *sp) { struct fc_exch *ep = fc_seq_exch(sp); int rc; @@ -951,6 +986,7 @@ static void fc_exch_done(struct fc_seq *sp) if (!rc) fc_exch_delete(ep); } +EXPORT_SYMBOL(fc_exch_done); /** * fc_exch_resp() - Allocate a new exchange for a response frame @@ -1197,8 +1233,8 @@ static void fc_exch_set_addr(struct fc_exch *ep, * * The received frame is not freed. */ -static void fc_seq_els_rsp_send(struct fc_frame *fp, enum fc_els_cmd els_cmd, - struct fc_seq_els_data *els_data) +void fc_seq_els_rsp_send(struct fc_frame *fp, enum fc_els_cmd els_cmd, + struct fc_seq_els_data *els_data) { switch (els_cmd) { case ELS_LS_RJT: @@ -1217,6 +1253,7 @@ static void fc_seq_els_rsp_send(struct fc_frame *fp, enum fc_els_cmd els_cmd, FC_LPORT_DBG(fr_dev(fp), "Invalid ELS CMD:%x\n", els_cmd); } } +EXPORT_SYMBOL_GPL(fc_seq_els_rsp_send); /** * fc_seq_send_last() - Send a sequence that is the last in the exchange @@ -1258,8 +1295,10 @@ static void fc_seq_send_ack(struct fc_seq *sp, const struct fc_frame *rx_fp) */ if (fc_sof_needs_ack(fr_sof(rx_fp))) { fp = fc_frame_alloc(lport, 0); - if (!fp) + if (!fp) { + FC_EXCH_DBG(ep, "Drop ACK request, out of memory\n"); return; + } fh = fc_frame_header_get(fp); fh->fh_r_ctl = FC_RCTL_ACK_1; @@ -1312,13 +1351,18 @@ static void fc_exch_send_ba_rjt(struct fc_frame *rx_fp, struct fc_frame_header *rx_fh; struct fc_frame_header *fh; struct fc_ba_rjt *rp; + struct fc_seq *sp; struct fc_lport *lport; unsigned int f_ctl; lport = fr_dev(rx_fp); + sp = fr_seq(rx_fp); fp = fc_frame_alloc(lport, sizeof(*rp)); - if (!fp) + if (!fp) { + FC_EXCH_DBG(fc_seq_exch(sp), + "Drop BA_RJT request, out of memory\n"); return; + } fh = fc_frame_header_get(fp); rx_fh = fc_frame_header_get(rx_fp); @@ -1383,14 +1427,17 @@ static void fc_exch_recv_abts(struct fc_exch *ep, struct fc_frame *rx_fp) if (!ep) goto reject; + FC_EXCH_DBG(ep, "exch: ABTS received\n"); fp = fc_frame_alloc(ep->lp, sizeof(*ap)); - if (!fp) + if (!fp) { + FC_EXCH_DBG(ep, "Drop ABTS request, out of memory\n"); goto free; + } spin_lock_bh(&ep->ex_lock); if (ep->esb_stat & ESB_ST_COMPLETE) { spin_unlock_bh(&ep->ex_lock); - + FC_EXCH_DBG(ep, "exch: ABTS rejected, exchange complete\n"); fc_frame_free(fp); goto reject; } @@ -1433,7 +1480,7 @@ reject: * A reference will be held on the exchange/sequence for the caller, which * must call fc_seq_release(). */ -static struct fc_seq *fc_seq_assign(struct fc_lport *lport, struct fc_frame *fp) +struct fc_seq *fc_seq_assign(struct fc_lport *lport, struct fc_frame *fp) { struct fc_exch_mgr_anchor *ema; @@ -1447,15 +1494,17 @@ static struct fc_seq *fc_seq_assign(struct fc_lport *lport, struct fc_frame *fp) break; return fr_seq(fp); } +EXPORT_SYMBOL(fc_seq_assign); /** * fc_seq_release() - Release the hold * @sp: The sequence. */ -static void fc_seq_release(struct fc_seq *sp) +void fc_seq_release(struct fc_seq *sp) { fc_exch_release(fc_seq_exch(sp)); } +EXPORT_SYMBOL(fc_seq_release); /** * fc_exch_recv_req() - Handler for an incoming request @@ -1491,7 +1540,7 @@ static void fc_exch_recv_req(struct fc_lport *lport, struct fc_exch_mgr *mp, * The upper-level protocol may request one later, if needed. */ if (fh->fh_rx_id == htons(FC_XID_UNKNOWN)) - return lport->tt.lport_recv(lport, fp); + return fc_lport_recv(lport, fp); reject = fc_seq_lookup_recip(lport, mp, fp); if (reject == FC_RJT_NONE) { @@ -1512,7 +1561,7 @@ static void fc_exch_recv_req(struct fc_lport *lport, struct fc_exch_mgr *mp, * first. */ if (!fc_invoke_resp(ep, sp, fp)) - lport->tt.lport_recv(lport, fp); + fc_lport_recv(lport, fp); fc_exch_release(ep); /* release from lookup */ } else { FC_LPORT_DBG(lport, "exch/seq lookup failed: reject %x\n", @@ -1562,9 +1611,6 @@ static void fc_exch_recv_seq_resp(struct fc_exch_mgr *mp, struct fc_frame *fp) if (fc_sof_is_init(sof)) { sp->ssb_stat |= SSB_ST_RESP; sp->id = fh->fh_seq_id; - } else if (sp->id != fh->fh_seq_id) { - atomic_inc(&mp->stats.seq_not_found); - goto rel; } f_ctl = ntoh24(fh->fh_f_ctl); @@ -1761,7 +1807,10 @@ static void fc_exch_recv_bls(struct fc_exch_mgr *mp, struct fc_frame *fp) fc_frame_free(fp); break; case FC_RCTL_BA_ABTS: - fc_exch_recv_abts(ep, fp); + if (ep) + fc_exch_recv_abts(ep, fp); + else + fc_frame_free(fp); break; default: /* ignore junk */ fc_frame_free(fp); @@ -1784,11 +1833,16 @@ static void fc_seq_ls_acc(struct fc_frame *rx_fp) struct fc_lport *lport; struct fc_els_ls_acc *acc; struct fc_frame *fp; + struct fc_seq *sp; lport = fr_dev(rx_fp); + sp = fr_seq(rx_fp); fp = fc_frame_alloc(lport, sizeof(*acc)); - if (!fp) + if (!fp) { + FC_EXCH_DBG(fc_seq_exch(sp), + "exch: drop LS_ACC, out of memory\n"); return; + } acc = fc_frame_payload_get(fp, sizeof(*acc)); memset(acc, 0, sizeof(*acc)); acc->la_cmd = ELS_LS_ACC; @@ -1811,11 +1865,16 @@ static void fc_seq_ls_rjt(struct fc_frame *rx_fp, enum fc_els_rjt_reason reason, struct fc_lport *lport; struct fc_els_ls_rjt *rjt; struct fc_frame *fp; + struct fc_seq *sp; lport = fr_dev(rx_fp); + sp = fr_seq(rx_fp); fp = fc_frame_alloc(lport, sizeof(*rjt)); - if (!fp) + if (!fp) { + FC_EXCH_DBG(fc_seq_exch(sp), + "exch: drop LS_ACC, out of memory\n"); return; + } rjt = fc_frame_payload_get(fp, sizeof(*rjt)); memset(rjt, 0, sizeof(*rjt)); rjt->er_cmd = ELS_LS_RJT; @@ -1960,8 +2019,7 @@ static void fc_exch_els_rec(struct fc_frame *rfp) enum fc_els_rjt_reason reason = ELS_RJT_LOGIC; enum fc_els_rjt_explan explan; u32 sid; - u16 rxid; - u16 oxid; + u16 xid, rxid, oxid; lport = fr_dev(rfp); rp = fc_frame_payload_get(rfp, sizeof(*rp)); @@ -1972,18 +2030,35 @@ static void fc_exch_els_rec(struct fc_frame *rfp) rxid = ntohs(rp->rec_rx_id); oxid = ntohs(rp->rec_ox_id); - ep = fc_exch_lookup(lport, - sid == fc_host_port_id(lport->host) ? oxid : rxid); explan = ELS_EXPL_OXID_RXID; - if (!ep) + if (sid == fc_host_port_id(lport->host)) + xid = oxid; + else + xid = rxid; + if (xid == FC_XID_UNKNOWN) { + FC_LPORT_DBG(lport, + "REC request from %x: invalid rxid %x oxid %x\n", + sid, rxid, oxid); + goto reject; + } + ep = fc_exch_lookup(lport, xid); + if (!ep) { + FC_LPORT_DBG(lport, + "REC request from %x: rxid %x oxid %x not found\n", + sid, rxid, oxid); goto reject; + } + FC_EXCH_DBG(ep, "REC request from %x: rxid %x oxid %x\n", + sid, rxid, oxid); if (ep->oid != sid || oxid != ep->oxid) goto rel; if (rxid != FC_XID_UNKNOWN && rxid != ep->rxid) goto rel; fp = fc_frame_alloc(lport, sizeof(*acc)); - if (!fp) + if (!fp) { + FC_EXCH_DBG(ep, "Drop REC request, out of memory\n"); goto out; + } acc = fc_frame_payload_get(fp, sizeof(*acc)); memset(acc, 0, sizeof(*acc)); @@ -2065,6 +2140,24 @@ cleanup: * @arg: The argument to be passed to the response handler * @timer_msec: The timeout period for the exchange * + * The exchange response handler is set in this routine to resp() + * function pointer. It can be called in two scenarios: if a timeout + * occurs or if a response frame is received for the exchange. The + * fc_frame pointer in response handler will also indicate timeout + * as error using IS_ERR related macros. + * + * The exchange destructor handler is also set in this routine. + * The destructor handler is invoked by EM layer when exchange + * is about to free, this can be used by caller to free its + * resources along with exchange free. + * + * The arg is passed back to resp and destructor handler. + * + * The timeout value (in msec) for an exchange is set if non zero + * timer_msec argument is specified. The timer is canceled when + * it fires or when the exchange is done. The exchange timeout handler + * is registered by EM layer. + * * The frame pointer with some of the header's fields must be * filled before calling this routine, those fields are: * @@ -2075,14 +2168,13 @@ cleanup: * - frame control * - parameter or relative offset */ -static struct fc_seq *fc_exch_seq_send(struct fc_lport *lport, - struct fc_frame *fp, - void (*resp)(struct fc_seq *, - struct fc_frame *fp, - void *arg), - void (*destructor)(struct fc_seq *, - void *), - void *arg, u32 timer_msec) +struct fc_seq *fc_exch_seq_send(struct fc_lport *lport, + struct fc_frame *fp, + void (*resp)(struct fc_seq *, + struct fc_frame *fp, + void *arg), + void (*destructor)(struct fc_seq *, void *), + void *arg, u32 timer_msec) { struct fc_exch *ep; struct fc_seq *sp = NULL; @@ -2101,7 +2193,7 @@ static struct fc_seq *fc_exch_seq_send(struct fc_lport *lport, ep->resp = resp; ep->destructor = destructor; ep->arg = arg; - ep->r_a_tov = FC_DEF_R_A_TOV; + ep->r_a_tov = lport->r_a_tov; ep->lp = lport; sp = &ep->seq; @@ -2135,6 +2227,7 @@ err: fc_exch_delete(ep); return NULL; } +EXPORT_SYMBOL(fc_exch_seq_send); /** * fc_exch_rrq() - Send an ELS RRQ (Reinstate Recovery Qualifier) command @@ -2176,6 +2269,7 @@ static void fc_exch_rrq(struct fc_exch *ep) return; retry: + FC_EXCH_DBG(ep, "exch: RRQ send failed\n"); spin_lock_bh(&ep->ex_lock); if (ep->state & (FC_EX_RST_CLEANUP | FC_EX_DONE)) { spin_unlock_bh(&ep->ex_lock); @@ -2218,6 +2312,8 @@ static void fc_exch_els_rrq(struct fc_frame *fp) if (!ep) goto reject; spin_lock_bh(&ep->ex_lock); + FC_EXCH_DBG(ep, "RRQ request from %x: xid %x rxid %x oxid %x\n", + sid, xid, ntohs(rp->rrq_rx_id), ntohs(rp->rrq_ox_id)); if (ep->oxid != ntohs(rp->rrq_ox_id)) goto unlock_reject; if (ep->rxid != ntohs(rp->rrq_rx_id) && @@ -2385,6 +2481,7 @@ struct fc_exch_mgr *fc_exch_mgr_alloc(struct fc_lport *lport, return NULL; mp->class = class; + mp->lport = lport; /* adjust em exch xid range for offload */ mp->min_xid = min_xid; @@ -2558,36 +2655,9 @@ EXPORT_SYMBOL(fc_exch_recv); */ int fc_exch_init(struct fc_lport *lport) { - if (!lport->tt.seq_start_next) - lport->tt.seq_start_next = fc_seq_start_next; - - if (!lport->tt.seq_set_resp) - lport->tt.seq_set_resp = fc_seq_set_resp; - - if (!lport->tt.exch_seq_send) - lport->tt.exch_seq_send = fc_exch_seq_send; - - if (!lport->tt.seq_send) - lport->tt.seq_send = fc_seq_send; - - if (!lport->tt.seq_els_rsp_send) - lport->tt.seq_els_rsp_send = fc_seq_els_rsp_send; - - if (!lport->tt.exch_done) - lport->tt.exch_done = fc_exch_done; - if (!lport->tt.exch_mgr_reset) lport->tt.exch_mgr_reset = fc_exch_mgr_reset; - if (!lport->tt.seq_exch_abort) - lport->tt.seq_exch_abort = fc_seq_exch_abort; - - if (!lport->tt.seq_assign) - lport->tt.seq_assign = fc_seq_assign; - - if (!lport->tt.seq_release) - lport->tt.seq_release = fc_seq_release; - return 0; } EXPORT_SYMBOL(fc_exch_init); diff --git a/drivers/scsi/libfc/fc_fcp.c b/drivers/scsi/libfc/fc_fcp.c index 5121272..0e67621 100644 --- a/drivers/scsi/libfc/fc_fcp.c +++ b/drivers/scsi/libfc/fc_fcp.c @@ -122,6 +122,7 @@ static void fc_fcp_srr_error(struct fc_fcp_pkt *, struct fc_frame *); #define FC_HRD_ERROR 9 #define FC_CRC_ERROR 10 #define FC_TIMED_OUT 11 +#define FC_TRANS_RESET 12 /* * Error recovery timeout values. @@ -195,7 +196,7 @@ static void fc_fcp_pkt_hold(struct fc_fcp_pkt *fsp) * @seq: The sequence that the FCP packet is on (required by destructor API) * @fsp: The FCP packet to be released * - * This routine is called by a destructor callback in the exch_seq_send() + * This routine is called by a destructor callback in the fc_exch_seq_send() * routine of the libfc Transport Template. The 'struct fc_seq' is a required * argument even though it is not used by this routine. * @@ -253,8 +254,21 @@ static inline void fc_fcp_unlock_pkt(struct fc_fcp_pkt *fsp) */ static void fc_fcp_timer_set(struct fc_fcp_pkt *fsp, unsigned long delay) { - if (!(fsp->state & FC_SRB_COMPL)) + if (!(fsp->state & FC_SRB_COMPL)) { mod_timer(&fsp->timer, jiffies + delay); + fsp->timer_delay = delay; + } +} + +static void fc_fcp_abort_done(struct fc_fcp_pkt *fsp) +{ + fsp->state |= FC_SRB_ABORTED; + fsp->state &= ~FC_SRB_ABORT_PENDING; + + if (fsp->wait_for_comp) + complete(&fsp->tm_done); + else + fc_fcp_complete_locked(fsp); } /** @@ -264,6 +278,8 @@ static void fc_fcp_timer_set(struct fc_fcp_pkt *fsp, unsigned long delay) */ static int fc_fcp_send_abort(struct fc_fcp_pkt *fsp) { + int rc; + if (!fsp->seq_ptr) return -EINVAL; @@ -271,7 +287,16 @@ static int fc_fcp_send_abort(struct fc_fcp_pkt *fsp) put_cpu(); fsp->state |= FC_SRB_ABORT_PENDING; - return fsp->lp->tt.seq_exch_abort(fsp->seq_ptr, 0); + rc = fc_seq_exch_abort(fsp->seq_ptr, 0); + /* + * fc_seq_exch_abort() might return -ENXIO if + * the sequence is already completed + */ + if (rc == -ENXIO) { + fc_fcp_abort_done(fsp); + rc = 0; + } + return rc; } /** @@ -283,16 +308,16 @@ static int fc_fcp_send_abort(struct fc_fcp_pkt *fsp) * fc_io_compl() will notify the SCSI-ml that the I/O is done. * The SCSI-ml will retry the command. */ -static void fc_fcp_retry_cmd(struct fc_fcp_pkt *fsp) +static void fc_fcp_retry_cmd(struct fc_fcp_pkt *fsp, int status_code) { if (fsp->seq_ptr) { - fsp->lp->tt.exch_done(fsp->seq_ptr); + fc_exch_done(fsp->seq_ptr); fsp->seq_ptr = NULL; } fsp->state &= ~FC_SRB_ABORT_PENDING; fsp->io_status = 0; - fsp->status_code = FC_ERROR; + fsp->status_code = status_code; fc_fcp_complete_locked(fsp); } @@ -402,8 +427,6 @@ static void fc_fcp_can_queue_ramp_down(struct fc_lport *lport) if (!can_queue) can_queue = 1; lport->host->can_queue = can_queue; - shost_printk(KERN_ERR, lport->host, "libfc: Could not allocate frame.\n" - "Reducing can_queue to %d.\n", can_queue); unlock: spin_unlock_irqrestore(lport->host->host_lock, flags); @@ -430,10 +453,29 @@ static inline struct fc_frame *fc_fcp_frame_alloc(struct fc_lport *lport, put_cpu(); /* error case */ fc_fcp_can_queue_ramp_down(lport); + shost_printk(KERN_ERR, lport->host, + "libfc: Could not allocate frame, " + "reducing can_queue to %d.\n", lport->host->can_queue); return NULL; } /** + * get_fsp_rec_tov() - Helper function to get REC_TOV + * @fsp: the FCP packet + * + * Returns rec tov in jiffies as rpriv->e_d_tov + 1 second + */ +static inline unsigned int get_fsp_rec_tov(struct fc_fcp_pkt *fsp) +{ + struct fc_rport_libfc_priv *rpriv = fsp->rport->dd_data; + unsigned int e_d_tov = FC_DEF_E_D_TOV; + + if (rpriv && rpriv->e_d_tov > e_d_tov) + e_d_tov = rpriv->e_d_tov; + return msecs_to_jiffies(e_d_tov) + HZ; +} + +/** * fc_fcp_recv_data() - Handler for receiving SCSI-FCP data from a target * @fsp: The FCP packet the data is on * @fp: The data frame @@ -536,8 +578,10 @@ crc_err: * and completes the transfer, call the completion handler. */ if (unlikely(fsp->state & FC_SRB_RCV_STATUS) && - fsp->xfer_len == fsp->data_len - fsp->scsi_resid) + fsp->xfer_len == fsp->data_len - fsp->scsi_resid) { + FC_FCP_DBG( fsp, "complete out-of-order sequence\n" ); fc_fcp_complete_locked(fsp); + } return; err: fc_fcp_recovery(fsp, host_bcode); @@ -609,7 +653,7 @@ static int fc_fcp_send_data(struct fc_fcp_pkt *fsp, struct fc_seq *seq, remaining = seq_blen; fh_parm_offset = frame_offset = offset; tlen = 0; - seq = lport->tt.seq_start_next(seq); + seq = fc_seq_start_next(seq); f_ctl = FC_FC_REL_OFF; WARN_ON(!seq); @@ -687,7 +731,7 @@ static int fc_fcp_send_data(struct fc_fcp_pkt *fsp, struct fc_seq *seq, /* * send fragment using for a sequence. */ - error = lport->tt.seq_send(lport, seq, fp); + error = fc_seq_send(lport, seq, fp); if (error) { WARN_ON(1); /* send error should be rare */ return error; @@ -727,15 +771,8 @@ static void fc_fcp_abts_resp(struct fc_fcp_pkt *fsp, struct fc_frame *fp) ba_done = 0; } - if (ba_done) { - fsp->state |= FC_SRB_ABORTED; - fsp->state &= ~FC_SRB_ABORT_PENDING; - - if (fsp->wait_for_comp) - complete(&fsp->tm_done); - else - fc_fcp_complete_locked(fsp); - } + if (ba_done) + fc_fcp_abort_done(fsp); } /** @@ -764,8 +801,11 @@ static void fc_fcp_recv(struct fc_seq *seq, struct fc_frame *fp, void *arg) fh = fc_frame_header_get(fp); r_ctl = fh->fh_r_ctl; - if (lport->state != LPORT_ST_READY) + if (lport->state != LPORT_ST_READY) { + FC_FCP_DBG(fsp, "lport state %d, ignoring r_ctl %x\n", + lport->state, r_ctl); goto out; + } if (fc_fcp_lock_pkt(fsp)) goto out; @@ -774,8 +814,10 @@ static void fc_fcp_recv(struct fc_seq *seq, struct fc_frame *fp, void *arg) goto unlock; } - if (fsp->state & (FC_SRB_ABORTED | FC_SRB_ABORT_PENDING)) + if (fsp->state & (FC_SRB_ABORTED | FC_SRB_ABORT_PENDING)) { + FC_FCP_DBG(fsp, "command aborted, ignoring r_ctl %x\n", r_ctl); goto unlock; + } if (r_ctl == FC_RCTL_DD_DATA_DESC) { /* @@ -910,7 +952,16 @@ static void fc_fcp_resp(struct fc_fcp_pkt *fsp, struct fc_frame *fp) * Wait a at least one jiffy to see if it is delivered. * If this expires without data, we may do SRR. */ - fc_fcp_timer_set(fsp, 2); + if (fsp->lp->qfull) { + FC_FCP_DBG(fsp, "tgt %6.6x queue busy retry\n", + fsp->rport->port_id); + return; + } + FC_FCP_DBG(fsp, "tgt %6.6x xfer len %zx data underrun " + "len %x, data len %x\n", + fsp->rport->port_id, + fsp->xfer_len, expected_len, fsp->data_len); + fc_fcp_timer_set(fsp, get_fsp_rec_tov(fsp)); return; } fsp->status_code = FC_DATA_OVRRUN; @@ -959,8 +1010,11 @@ static void fc_fcp_complete_locked(struct fc_fcp_pkt *fsp) if (fsp->cdb_status == SAM_STAT_GOOD && fsp->xfer_len < fsp->data_len && !fsp->io_status && (!(fsp->scsi_comp_flags & FCP_RESID_UNDER) || - fsp->xfer_len < fsp->data_len - fsp->scsi_resid)) + fsp->xfer_len < fsp->data_len - fsp->scsi_resid)) { + FC_FCP_DBG(fsp, "data underrun, xfer %zx data %x\n", + fsp->xfer_len, fsp->data_len); fsp->status_code = FC_DATA_UNDRUN; + } } seq = fsp->seq_ptr; @@ -970,7 +1024,7 @@ static void fc_fcp_complete_locked(struct fc_fcp_pkt *fsp) struct fc_frame *conf_frame; struct fc_seq *csp; - csp = lport->tt.seq_start_next(seq); + csp = fc_seq_start_next(seq); conf_frame = fc_fcp_frame_alloc(fsp->lp, 0); if (conf_frame) { f_ctl = FC_FC_SEQ_INIT; @@ -979,10 +1033,10 @@ static void fc_fcp_complete_locked(struct fc_fcp_pkt *fsp) fc_fill_fc_hdr(conf_frame, FC_RCTL_DD_SOL_CTL, ep->did, ep->sid, FC_TYPE_FCP, f_ctl, 0); - lport->tt.seq_send(lport, csp, conf_frame); + fc_seq_send(lport, csp, conf_frame); } } - lport->tt.exch_done(seq); + fc_exch_done(seq); } /* * Some resets driven by SCSI are not I/Os and do not have @@ -1000,10 +1054,8 @@ static void fc_fcp_complete_locked(struct fc_fcp_pkt *fsp) */ static void fc_fcp_cleanup_cmd(struct fc_fcp_pkt *fsp, int error) { - struct fc_lport *lport = fsp->lp; - if (fsp->seq_ptr) { - lport->tt.exch_done(fsp->seq_ptr); + fc_exch_done(fsp->seq_ptr); fsp->seq_ptr = NULL; } fsp->status_code = error; @@ -1116,19 +1168,6 @@ static int fc_fcp_pkt_send(struct fc_lport *lport, struct fc_fcp_pkt *fsp) } /** - * get_fsp_rec_tov() - Helper function to get REC_TOV - * @fsp: the FCP packet - * - * Returns rec tov in jiffies as rpriv->e_d_tov + 1 second - */ -static inline unsigned int get_fsp_rec_tov(struct fc_fcp_pkt *fsp) -{ - struct fc_rport_libfc_priv *rpriv = fsp->rport->dd_data; - - return msecs_to_jiffies(rpriv->e_d_tov) + HZ; -} - -/** * fc_fcp_cmd_send() - Send a FCP command * @lport: The local port to send the command on * @fsp: The FCP packet the command is on @@ -1165,8 +1204,7 @@ static int fc_fcp_cmd_send(struct fc_lport *lport, struct fc_fcp_pkt *fsp, rpriv->local_port->port_id, FC_TYPE_FCP, FC_FCTL_REQ, 0); - seq = lport->tt.exch_seq_send(lport, fp, resp, fc_fcp_pkt_destroy, - fsp, 0); + seq = fc_exch_seq_send(lport, fp, resp, fc_fcp_pkt_destroy, fsp, 0); if (!seq) { rc = -1; goto unlock; @@ -1196,7 +1234,7 @@ static void fc_fcp_error(struct fc_fcp_pkt *fsp, struct fc_frame *fp) return; if (error == -FC_EX_CLOSED) { - fc_fcp_retry_cmd(fsp); + fc_fcp_retry_cmd(fsp, FC_ERROR); goto unlock; } @@ -1222,8 +1260,16 @@ static int fc_fcp_pkt_abort(struct fc_fcp_pkt *fsp) int rc = FAILED; unsigned long ticks_left; - if (fc_fcp_send_abort(fsp)) + FC_FCP_DBG(fsp, "pkt abort state %x\n", fsp->state); + if (fc_fcp_send_abort(fsp)) { + FC_FCP_DBG(fsp, "failed to send abort\n"); return FAILED; + } + + if (fsp->state & FC_SRB_ABORTED) { + FC_FCP_DBG(fsp, "target abort cmd completed\n"); + return SUCCESS; + } init_completion(&fsp->tm_done); fsp->wait_for_comp = 1; @@ -1301,7 +1347,7 @@ static int fc_lun_reset(struct fc_lport *lport, struct fc_fcp_pkt *fsp, spin_lock_bh(&fsp->scsi_pkt_lock); if (fsp->seq_ptr) { - lport->tt.exch_done(fsp->seq_ptr); + fc_exch_done(fsp->seq_ptr); fsp->seq_ptr = NULL; } fsp->wait_for_comp = 0; @@ -1355,7 +1401,7 @@ static void fc_tm_done(struct fc_seq *seq, struct fc_frame *fp, void *arg) if (fh->fh_type != FC_TYPE_BLS) fc_fcp_resp(fsp, fp); fsp->seq_ptr = NULL; - fsp->lp->tt.exch_done(seq); + fc_exch_done(seq); out_unlock: fc_fcp_unlock_pkt(fsp); out: @@ -1394,6 +1440,15 @@ static void fc_fcp_timeout(unsigned long data) if (fsp->cdb_cmd.fc_tm_flags) goto unlock; + if (fsp->lp->qfull) { + FC_FCP_DBG(fsp, "fcp timeout, resetting timer delay %d\n", + fsp->timer_delay); + setup_timer(&fsp->timer, fc_fcp_timeout, (unsigned long)fsp); + fc_fcp_timer_set(fsp, fsp->timer_delay); + goto unlock; + } + FC_FCP_DBG(fsp, "fcp timeout, delay %d flags %x state %x\n", + fsp->timer_delay, rpriv->flags, fsp->state); fsp->state |= FC_SRB_FCP_PROCESSING_TMO; if (rpriv->flags & FC_RP_FLAGS_REC_SUPPORTED) @@ -1486,8 +1541,8 @@ static void fc_fcp_rec_resp(struct fc_seq *seq, struct fc_frame *fp, void *arg) rjt = fc_frame_payload_get(fp, sizeof(*rjt)); switch (rjt->er_reason) { default: - FC_FCP_DBG(fsp, "device %x unexpected REC reject " - "reason %d expl %d\n", + FC_FCP_DBG(fsp, + "device %x invalid REC reject %d/%d\n", fsp->rport->port_id, rjt->er_reason, rjt->er_explan); /* fall through */ @@ -1503,18 +1558,23 @@ static void fc_fcp_rec_resp(struct fc_seq *seq, struct fc_frame *fp, void *arg) break; case ELS_RJT_LOGIC: case ELS_RJT_UNAB: + FC_FCP_DBG(fsp, "device %x REC reject %d/%d\n", + fsp->rport->port_id, rjt->er_reason, + rjt->er_explan); /* - * If no data transfer, the command frame got dropped - * so we just retry. If data was transferred, we - * lost the response but the target has no record, - * so we abort and retry. + * If response got lost or is stuck in the + * queue somewhere we have no idea if and when + * the response will be received. So quarantine + * the xid and retry the command. */ - if (rjt->er_explan == ELS_EXPL_OXID_RXID && - fsp->xfer_len == 0) { - fc_fcp_retry_cmd(fsp); + if (rjt->er_explan == ELS_EXPL_OXID_RXID) { + struct fc_exch *ep = fc_seq_exch(fsp->seq_ptr); + ep->state |= FC_EX_QUARANTINE; + fsp->state |= FC_SRB_ABORTED; + fc_fcp_retry_cmd(fsp, FC_TRANS_RESET); break; } - fc_fcp_recovery(fsp, FC_ERROR); + fc_fcp_recovery(fsp, FC_TRANS_RESET); break; } } else if (opcode == ELS_LS_ACC) { @@ -1608,7 +1668,9 @@ static void fc_fcp_rec_error(struct fc_fcp_pkt *fsp, struct fc_frame *fp) switch (error) { case -FC_EX_CLOSED: - fc_fcp_retry_cmd(fsp); + FC_FCP_DBG(fsp, "REC %p fid %6.6x exchange closed\n", + fsp, fsp->rport->port_id); + fc_fcp_retry_cmd(fsp, FC_ERROR); break; default: @@ -1622,8 +1684,8 @@ static void fc_fcp_rec_error(struct fc_fcp_pkt *fsp, struct fc_frame *fp) * Assume REC or LS_ACC was lost. * The exchange manager will have aborted REC, so retry. */ - FC_FCP_DBG(fsp, "REC fid %6.6x error error %d retry %d/%d\n", - fsp->rport->port_id, error, fsp->recov_retry, + FC_FCP_DBG(fsp, "REC %p fid %6.6x exchange timeout retry %d/%d\n", + fsp, fsp->rport->port_id, fsp->recov_retry, FC_MAX_RECOV_RETRY); if (fsp->recov_retry++ < FC_MAX_RECOV_RETRY) fc_fcp_rec(fsp); @@ -1642,6 +1704,7 @@ out: */ static void fc_fcp_recovery(struct fc_fcp_pkt *fsp, u8 code) { + FC_FCP_DBG(fsp, "start recovery code %x\n", code); fsp->status_code = code; fsp->cdb_status = 0; fsp->io_status = 0; @@ -1668,7 +1731,6 @@ static void fc_fcp_srr(struct fc_fcp_pkt *fsp, enum fc_rctl r_ctl, u32 offset) struct fc_seq *seq; struct fcp_srr *srr; struct fc_frame *fp; - unsigned int rec_tov; rport = fsp->rport; rpriv = rport->dd_data; @@ -1692,10 +1754,9 @@ static void fc_fcp_srr(struct fc_fcp_pkt *fsp, enum fc_rctl r_ctl, u32 offset) rpriv->local_port->port_id, FC_TYPE_FCP, FC_FCTL_REQ, 0); - rec_tov = get_fsp_rec_tov(fsp); - seq = lport->tt.exch_seq_send(lport, fp, fc_fcp_srr_resp, - fc_fcp_pkt_destroy, - fsp, jiffies_to_msecs(rec_tov)); + seq = fc_exch_seq_send(lport, fp, fc_fcp_srr_resp, + fc_fcp_pkt_destroy, + fsp, get_fsp_rec_tov(fsp)); if (!seq) goto retry; @@ -1706,7 +1767,7 @@ static void fc_fcp_srr(struct fc_fcp_pkt *fsp, enum fc_rctl r_ctl, u32 offset) fc_fcp_pkt_hold(fsp); /* hold for outstanding SRR */ return; retry: - fc_fcp_retry_cmd(fsp); + fc_fcp_retry_cmd(fsp, FC_TRANS_RESET); } /** @@ -1730,9 +1791,9 @@ static void fc_fcp_srr_resp(struct fc_seq *seq, struct fc_frame *fp, void *arg) fh = fc_frame_header_get(fp); /* - * BUG? fc_fcp_srr_error calls exch_done which would release + * BUG? fc_fcp_srr_error calls fc_exch_done which would release * the ep. But if fc_fcp_srr_error had got -FC_EX_TIMEOUT, - * then fc_exch_timeout would be sending an abort. The exch_done + * then fc_exch_timeout would be sending an abort. The fc_exch_done * call by fc_fcp_srr_error would prevent fc_exch.c from seeing * an abort response though. */ @@ -1753,7 +1814,7 @@ static void fc_fcp_srr_resp(struct fc_seq *seq, struct fc_frame *fp, void *arg) } fc_fcp_unlock_pkt(fsp); out: - fsp->lp->tt.exch_done(seq); + fc_exch_done(seq); fc_frame_free(fp); } @@ -1768,20 +1829,22 @@ static void fc_fcp_srr_error(struct fc_fcp_pkt *fsp, struct fc_frame *fp) goto out; switch (PTR_ERR(fp)) { case -FC_EX_TIMEOUT: + FC_FCP_DBG(fsp, "SRR timeout, retries %d\n", fsp->recov_retry); if (fsp->recov_retry++ < FC_MAX_RECOV_RETRY) fc_fcp_rec(fsp); else fc_fcp_recovery(fsp, FC_TIMED_OUT); break; case -FC_EX_CLOSED: /* e.g., link failure */ + FC_FCP_DBG(fsp, "SRR error, exchange closed\n"); /* fall through */ default: - fc_fcp_retry_cmd(fsp); + fc_fcp_retry_cmd(fsp, FC_ERROR); break; } fc_fcp_unlock_pkt(fsp); out: - fsp->lp->tt.exch_done(fsp->recov_seq); + fc_exch_done(fsp->recov_seq); } /** @@ -1832,8 +1895,13 @@ int fc_queuecommand(struct Scsi_Host *shost, struct scsi_cmnd *sc_cmd) rpriv = rport->dd_data; if (!fc_fcp_lport_queue_ready(lport)) { - if (lport->qfull) + if (lport->qfull) { fc_fcp_can_queue_ramp_down(lport); + shost_printk(KERN_ERR, lport->host, + "libfc: queue full, " + "reducing can_queue to %d.\n", + lport->host->can_queue); + } rc = SCSI_MLQUEUE_HOST_BUSY; goto out; } @@ -1980,15 +2048,26 @@ static void fc_io_compl(struct fc_fcp_pkt *fsp) sc_cmd->result = (DID_ERROR << 16) | fsp->cdb_status; break; case FC_CMD_ABORTED: - FC_FCP_DBG(fsp, "Returning DID_ERROR to scsi-ml " - "due to FC_CMD_ABORTED\n"); - sc_cmd->result = (DID_ERROR << 16) | fsp->io_status; + if (host_byte(sc_cmd->result) == DID_TIME_OUT) + FC_FCP_DBG(fsp, "Returning DID_TIME_OUT to scsi-ml " + "due to FC_CMD_ABORTED\n"); + else { + FC_FCP_DBG(fsp, "Returning DID_ERROR to scsi-ml " + "due to FC_CMD_ABORTED\n"); + set_host_byte(sc_cmd, DID_ERROR); + } + sc_cmd->result |= fsp->io_status; break; case FC_CMD_RESET: FC_FCP_DBG(fsp, "Returning DID_RESET to scsi-ml " "due to FC_CMD_RESET\n"); sc_cmd->result = (DID_RESET << 16); break; + case FC_TRANS_RESET: + FC_FCP_DBG(fsp, "Returning DID_SOFT_ERROR to scsi-ml " + "due to FC_TRANS_RESET\n"); + sc_cmd->result = (DID_SOFT_ERROR << 16); + break; case FC_HRD_ERROR: FC_FCP_DBG(fsp, "Returning DID_NO_CONNECT to scsi-ml " "due to FC_HRD_ERROR\n"); @@ -2142,7 +2221,7 @@ int fc_eh_host_reset(struct scsi_cmnd *sc_cmd) fc_block_scsi_eh(sc_cmd); - lport->tt.lport_reset(lport); + fc_lport_reset(lport); wait_tmo = jiffies + FC_HOST_RESET_TIMEOUT; while (!fc_fcp_lport_queue_ready(lport) && time_before(jiffies, wait_tmo)) diff --git a/drivers/scsi/libfc/fc_libfc.c b/drivers/scsi/libfc/fc_libfc.c index c11a638..d623d08 100644 --- a/drivers/scsi/libfc/fc_libfc.c +++ b/drivers/scsi/libfc/fc_libfc.c @@ -226,7 +226,7 @@ void fc_fill_reply_hdr(struct fc_frame *fp, const struct fc_frame *in_fp, sp = fr_seq(in_fp); if (sp) - fr_seq(fp) = fr_dev(in_fp)->tt.seq_start_next(sp); + fr_seq(fp) = fc_seq_start_next(sp); fc_fill_hdr(fp, in_fp, r_ctl, FC_FCTL_RESP, 0, parm_offset); } EXPORT_SYMBOL(fc_fill_reply_hdr); diff --git a/drivers/scsi/libfc/fc_lport.c b/drivers/scsi/libfc/fc_lport.c index 50c7167..919736a 100644 --- a/drivers/scsi/libfc/fc_lport.c +++ b/drivers/scsi/libfc/fc_lport.c @@ -149,7 +149,7 @@ static const char *fc_lport_state_names[] = { * @offset: The offset into the response data */ struct fc_bsg_info { - struct fc_bsg_job *job; + struct bsg_job *job; struct fc_lport *lport; u16 rsp_code; struct scatterlist *sg; @@ -200,7 +200,7 @@ static void fc_lport_rport_callback(struct fc_lport *lport, "in the DNS or FDMI state, it's in the " "%d state", rdata->ids.port_id, lport->state); - lport->tt.rport_logoff(rdata); + fc_rport_logoff(rdata); } break; case RPORT_EV_LOGO: @@ -237,23 +237,26 @@ static const char *fc_lport_state(struct fc_lport *lport) * @remote_fid: The FID of the ptp rport * @remote_wwpn: The WWPN of the ptp rport * @remote_wwnn: The WWNN of the ptp rport + * + * Locking Note: The lport lock is expected to be held before calling + * this routine. */ static void fc_lport_ptp_setup(struct fc_lport *lport, u32 remote_fid, u64 remote_wwpn, u64 remote_wwnn) { - mutex_lock(&lport->disc.disc_mutex); if (lport->ptp_rdata) { - lport->tt.rport_logoff(lport->ptp_rdata); - kref_put(&lport->ptp_rdata->kref, lport->tt.rport_destroy); + fc_rport_logoff(lport->ptp_rdata); + kref_put(&lport->ptp_rdata->kref, fc_rport_destroy); } - lport->ptp_rdata = lport->tt.rport_create(lport, remote_fid); + mutex_lock(&lport->disc.disc_mutex); + lport->ptp_rdata = fc_rport_create(lport, remote_fid); kref_get(&lport->ptp_rdata->kref); lport->ptp_rdata->ids.port_name = remote_wwpn; lport->ptp_rdata->ids.node_name = remote_wwnn; mutex_unlock(&lport->disc.disc_mutex); - lport->tt.rport_login(lport->ptp_rdata); + fc_rport_login(lport->ptp_rdata); fc_lport_enter_ready(lport); } @@ -409,7 +412,7 @@ static void fc_lport_recv_rlir_req(struct fc_lport *lport, struct fc_frame *fp) FC_LPORT_DBG(lport, "Received RLIR request while in state %s\n", fc_lport_state(lport)); - lport->tt.seq_els_rsp_send(fp, ELS_LS_ACC, NULL); + fc_seq_els_rsp_send(fp, ELS_LS_ACC, NULL); fc_frame_free(fp); } @@ -478,7 +481,7 @@ static void fc_lport_recv_rnid_req(struct fc_lport *lport, if (!req) { rjt_data.reason = ELS_RJT_LOGIC; rjt_data.explan = ELS_EXPL_NONE; - lport->tt.seq_els_rsp_send(in_fp, ELS_LS_RJT, &rjt_data); + fc_seq_els_rsp_send(in_fp, ELS_LS_RJT, &rjt_data); } else { fmt = req->rnid_fmt; len = sizeof(*rp); @@ -518,7 +521,7 @@ static void fc_lport_recv_rnid_req(struct fc_lport *lport, */ static void fc_lport_recv_logo_req(struct fc_lport *lport, struct fc_frame *fp) { - lport->tt.seq_els_rsp_send(fp, ELS_LS_ACC, NULL); + fc_seq_els_rsp_send(fp, ELS_LS_ACC, NULL); fc_lport_enter_reset(lport); fc_frame_free(fp); } @@ -620,9 +623,9 @@ int fc_fabric_logoff(struct fc_lport *lport) lport->tt.disc_stop_final(lport); mutex_lock(&lport->lp_mutex); if (lport->dns_rdata) - lport->tt.rport_logoff(lport->dns_rdata); + fc_rport_logoff(lport->dns_rdata); mutex_unlock(&lport->lp_mutex); - lport->tt.rport_flush_queue(); + fc_rport_flush_queue(); mutex_lock(&lport->lp_mutex); fc_lport_enter_logo(lport); mutex_unlock(&lport->lp_mutex); @@ -899,7 +902,7 @@ static void fc_lport_recv_els_req(struct fc_lport *lport, /* * Check opcode. */ - recv = lport->tt.rport_recv_req; + recv = fc_rport_recv_req; switch (fc_frame_payload_op(fp)) { case ELS_FLOGI: if (!lport->point_to_multipoint) @@ -941,15 +944,14 @@ struct fc4_prov fc_lport_els_prov = { }; /** - * fc_lport_recv_req() - The generic lport request handler + * fc_lport_recv() - The generic lport request handler * @lport: The lport that received the request * @fp: The frame the request is in * * Locking Note: This function should not be called with the lport * lock held because it may grab the lock. */ -static void fc_lport_recv_req(struct fc_lport *lport, - struct fc_frame *fp) +void fc_lport_recv(struct fc_lport *lport, struct fc_frame *fp) { struct fc_frame_header *fh = fc_frame_header_get(fp); struct fc_seq *sp = fr_seq(fp); @@ -978,8 +980,9 @@ drop: FC_LPORT_DBG(lport, "dropping unexpected frame type %x\n", fh->fh_type); fc_frame_free(fp); if (sp) - lport->tt.exch_done(sp); + fc_exch_done(sp); } +EXPORT_SYMBOL(fc_lport_recv); /** * fc_lport_reset() - Reset a local port @@ -1007,12 +1010,14 @@ EXPORT_SYMBOL(fc_lport_reset); */ static void fc_lport_reset_locked(struct fc_lport *lport) { - if (lport->dns_rdata) - lport->tt.rport_logoff(lport->dns_rdata); + if (lport->dns_rdata) { + fc_rport_logoff(lport->dns_rdata); + lport->dns_rdata = NULL; + } if (lport->ptp_rdata) { - lport->tt.rport_logoff(lport->ptp_rdata); - kref_put(&lport->ptp_rdata->kref, lport->tt.rport_destroy); + fc_rport_logoff(lport->ptp_rdata); + kref_put(&lport->ptp_rdata->kref, fc_rport_destroy); lport->ptp_rdata = NULL; } @@ -1426,13 +1431,13 @@ static void fc_lport_enter_dns(struct fc_lport *lport) fc_lport_state_enter(lport, LPORT_ST_DNS); mutex_lock(&lport->disc.disc_mutex); - rdata = lport->tt.rport_create(lport, FC_FID_DIR_SERV); + rdata = fc_rport_create(lport, FC_FID_DIR_SERV); mutex_unlock(&lport->disc.disc_mutex); if (!rdata) goto err; rdata->ops = &fc_lport_rport_ops; - lport->tt.rport_login(rdata); + fc_rport_login(rdata); return; err: @@ -1543,13 +1548,13 @@ static void fc_lport_enter_fdmi(struct fc_lport *lport) fc_lport_state_enter(lport, LPORT_ST_FDMI); mutex_lock(&lport->disc.disc_mutex); - rdata = lport->tt.rport_create(lport, FC_FID_MGMT_SERV); + rdata = fc_rport_create(lport, FC_FID_MGMT_SERV); mutex_unlock(&lport->disc.disc_mutex); if (!rdata) goto err; rdata->ops = &fc_lport_rport_ops; - lport->tt.rport_login(rdata); + fc_rport_login(rdata); return; err: @@ -1772,7 +1777,7 @@ void fc_lport_flogi_resp(struct fc_seq *sp, struct fc_frame *fp, if ((csp_flags & FC_SP_FT_FPORT) == 0) { if (e_d_tov > lport->e_d_tov) lport->e_d_tov = e_d_tov; - lport->r_a_tov = 2 * e_d_tov; + lport->r_a_tov = 2 * lport->e_d_tov; fc_lport_set_port_id(lport, did, fp); printk(KERN_INFO "host%d: libfc: " "Port (%6.6x) entered " @@ -1784,8 +1789,10 @@ void fc_lport_flogi_resp(struct fc_seq *sp, struct fc_frame *fp, get_unaligned_be64( &flp->fl_wwnn)); } else { - lport->e_d_tov = e_d_tov; - lport->r_a_tov = r_a_tov; + if (e_d_tov > lport->e_d_tov) + lport->e_d_tov = e_d_tov; + if (r_a_tov > lport->r_a_tov) + lport->r_a_tov = r_a_tov; fc_host_fabric_name(lport->host) = get_unaligned_be64(&flp->fl_wwnn); fc_lport_set_port_id(lport, did, fp); @@ -1858,12 +1865,6 @@ EXPORT_SYMBOL(fc_lport_config); */ int fc_lport_init(struct fc_lport *lport) { - if (!lport->tt.lport_recv) - lport->tt.lport_recv = fc_lport_recv_req; - - if (!lport->tt.lport_reset) - lport->tt.lport_reset = fc_lport_reset; - fc_host_port_type(lport->host) = FC_PORTTYPE_NPORT; fc_host_node_name(lport->host) = lport->wwnn; fc_host_port_name(lport->host) = lport->wwpn; @@ -1900,18 +1901,19 @@ static void fc_lport_bsg_resp(struct fc_seq *sp, struct fc_frame *fp, void *info_arg) { struct fc_bsg_info *info = info_arg; - struct fc_bsg_job *job = info->job; + struct bsg_job *job = info->job; + struct fc_bsg_reply *bsg_reply = job->reply; struct fc_lport *lport = info->lport; struct fc_frame_header *fh; size_t len; void *buf; if (IS_ERR(fp)) { - job->reply->result = (PTR_ERR(fp) == -FC_EX_CLOSED) ? + bsg_reply->result = (PTR_ERR(fp) == -FC_EX_CLOSED) ? -ECONNABORTED : -ETIMEDOUT; job->reply_len = sizeof(uint32_t); - job->state_flags |= FC_RQST_STATE_DONE; - job->job_done(job); + bsg_job_done(job, bsg_reply->result, + bsg_reply->reply_payload_rcv_len); kfree(info); return; } @@ -1928,25 +1930,25 @@ static void fc_lport_bsg_resp(struct fc_seq *sp, struct fc_frame *fp, (unsigned short)fc_frame_payload_op(fp); /* Save the reply status of the job */ - job->reply->reply_data.ctels_reply.status = + bsg_reply->reply_data.ctels_reply.status = (cmd == info->rsp_code) ? FC_CTELS_STATUS_OK : FC_CTELS_STATUS_REJECT; } - job->reply->reply_payload_rcv_len += + bsg_reply->reply_payload_rcv_len += fc_copy_buffer_to_sglist(buf, len, info->sg, &info->nents, &info->offset, NULL); if (fr_eof(fp) == FC_EOF_T && (ntoh24(fh->fh_f_ctl) & (FC_FC_LAST_SEQ | FC_FC_END_SEQ)) == (FC_FC_LAST_SEQ | FC_FC_END_SEQ)) { - if (job->reply->reply_payload_rcv_len > + if (bsg_reply->reply_payload_rcv_len > job->reply_payload.payload_len) - job->reply->reply_payload_rcv_len = + bsg_reply->reply_payload_rcv_len = job->reply_payload.payload_len; - job->reply->result = 0; - job->state_flags |= FC_RQST_STATE_DONE; - job->job_done(job); + bsg_reply->result = 0; + bsg_job_done(job, bsg_reply->result, + bsg_reply->reply_payload_rcv_len); kfree(info); } fc_frame_free(fp); @@ -1962,7 +1964,7 @@ static void fc_lport_bsg_resp(struct fc_seq *sp, struct fc_frame *fp, * Locking Note: The lport lock is expected to be held before calling * this routine. */ -static int fc_lport_els_request(struct fc_bsg_job *job, +static int fc_lport_els_request(struct bsg_job *job, struct fc_lport *lport, u32 did, u32 tov) { @@ -2005,8 +2007,8 @@ static int fc_lport_els_request(struct fc_bsg_job *job, info->nents = job->reply_payload.sg_cnt; info->sg = job->reply_payload.sg_list; - if (!lport->tt.exch_seq_send(lport, fp, fc_lport_bsg_resp, - NULL, info, tov)) { + if (!fc_exch_seq_send(lport, fp, fc_lport_bsg_resp, + NULL, info, tov)) { kfree(info); return -ECOMM; } @@ -2023,7 +2025,7 @@ static int fc_lport_els_request(struct fc_bsg_job *job, * Locking Note: The lport lock is expected to be held before calling * this routine. */ -static int fc_lport_ct_request(struct fc_bsg_job *job, +static int fc_lport_ct_request(struct bsg_job *job, struct fc_lport *lport, u32 did, u32 tov) { struct fc_bsg_info *info; @@ -2066,8 +2068,8 @@ static int fc_lport_ct_request(struct fc_bsg_job *job, info->nents = job->reply_payload.sg_cnt; info->sg = job->reply_payload.sg_list; - if (!lport->tt.exch_seq_send(lport, fp, fc_lport_bsg_resp, - NULL, info, tov)) { + if (!fc_exch_seq_send(lport, fp, fc_lport_bsg_resp, + NULL, info, tov)) { kfree(info); return -ECOMM; } @@ -2079,25 +2081,27 @@ static int fc_lport_ct_request(struct fc_bsg_job *job, * FC Passthrough requests * @job: The BSG passthrough job */ -int fc_lport_bsg_request(struct fc_bsg_job *job) +int fc_lport_bsg_request(struct bsg_job *job) { + struct fc_bsg_request *bsg_request = job->request; + struct fc_bsg_reply *bsg_reply = job->reply; struct request *rsp = job->req->next_rq; - struct Scsi_Host *shost = job->shost; + struct Scsi_Host *shost = fc_bsg_to_shost(job); struct fc_lport *lport = shost_priv(shost); struct fc_rport *rport; struct fc_rport_priv *rdata; int rc = -EINVAL; u32 did, tov; - job->reply->reply_payload_rcv_len = 0; + bsg_reply->reply_payload_rcv_len = 0; if (rsp) rsp->resid_len = job->reply_payload.payload_len; mutex_lock(&lport->lp_mutex); - switch (job->request->msgcode) { + switch (bsg_request->msgcode) { case FC_BSG_RPT_ELS: - rport = job->rport; + rport = fc_bsg_to_rport(job); if (!rport) break; @@ -2107,7 +2111,7 @@ int fc_lport_bsg_request(struct fc_bsg_job *job) break; case FC_BSG_RPT_CT: - rport = job->rport; + rport = fc_bsg_to_rport(job); if (!rport) break; @@ -2117,25 +2121,25 @@ int fc_lport_bsg_request(struct fc_bsg_job *job) break; case FC_BSG_HST_CT: - did = ntoh24(job->request->rqst_data.h_ct.port_id); + did = ntoh24(bsg_request->rqst_data.h_ct.port_id); if (did == FC_FID_DIR_SERV) { rdata = lport->dns_rdata; if (!rdata) break; tov = rdata->e_d_tov; } else { - rdata = lport->tt.rport_lookup(lport, did); + rdata = fc_rport_lookup(lport, did); if (!rdata) break; tov = rdata->e_d_tov; - kref_put(&rdata->kref, lport->tt.rport_destroy); + kref_put(&rdata->kref, fc_rport_destroy); } rc = fc_lport_ct_request(job, lport, did, tov); break; case FC_BSG_HST_ELS_NOLOGIN: - did = ntoh24(job->request->rqst_data.h_els.port_id); + did = ntoh24(bsg_request->rqst_data.h_els.port_id); rc = fc_lport_els_request(job, lport, did, lport->e_d_tov); break; } diff --git a/drivers/scsi/libfc/fc_rport.c b/drivers/scsi/libfc/fc_rport.c index 97aeadd..c991f3b 100644 --- a/drivers/scsi/libfc/fc_rport.c +++ b/drivers/scsi/libfc/fc_rport.c @@ -44,6 +44,19 @@ * path this potential over-use of the mutex is acceptable. */ +/* + * RPORT REFERENCE COUNTING + * + * A rport reference should be taken when: + * - an rport is allocated + * - a workqueue item is scheduled + * - an ELS request is send + * The reference should be dropped when: + * - the workqueue function has finished + * - the ELS response is handled + * - an rport is removed + */ + #include <linux/kernel.h> #include <linux/spinlock.h> #include <linux/interrupt.h> @@ -74,8 +87,8 @@ static void fc_rport_recv_prli_req(struct fc_rport_priv *, struct fc_frame *); static void fc_rport_recv_prlo_req(struct fc_rport_priv *, struct fc_frame *); static void fc_rport_recv_logo_req(struct fc_lport *, struct fc_frame *); static void fc_rport_timeout(struct work_struct *); -static void fc_rport_error(struct fc_rport_priv *, struct fc_frame *); -static void fc_rport_error_retry(struct fc_rport_priv *, struct fc_frame *); +static void fc_rport_error(struct fc_rport_priv *, int); +static void fc_rport_error_retry(struct fc_rport_priv *, int); static void fc_rport_work(struct work_struct *); static const char *fc_rport_state_names[] = { @@ -98,8 +111,8 @@ static const char *fc_rport_state_names[] = { * The reference count of the fc_rport_priv structure is * increased by one. */ -static struct fc_rport_priv *fc_rport_lookup(const struct fc_lport *lport, - u32 port_id) +struct fc_rport_priv *fc_rport_lookup(const struct fc_lport *lport, + u32 port_id) { struct fc_rport_priv *rdata = NULL, *tmp_rdata; @@ -113,6 +126,7 @@ static struct fc_rport_priv *fc_rport_lookup(const struct fc_lport *lport, rcu_read_unlock(); return rdata; } +EXPORT_SYMBOL(fc_rport_lookup); /** * fc_rport_create() - Create a new remote port @@ -123,12 +137,11 @@ static struct fc_rport_priv *fc_rport_lookup(const struct fc_lport *lport, * * Locking note: must be called with the disc_mutex held. */ -static struct fc_rport_priv *fc_rport_create(struct fc_lport *lport, - u32 port_id) +struct fc_rport_priv *fc_rport_create(struct fc_lport *lport, u32 port_id) { struct fc_rport_priv *rdata; - rdata = lport->tt.rport_lookup(lport, port_id); + rdata = fc_rport_lookup(lport, port_id); if (rdata) return rdata; @@ -158,18 +171,20 @@ static struct fc_rport_priv *fc_rport_create(struct fc_lport *lport, } return rdata; } +EXPORT_SYMBOL(fc_rport_create); /** * fc_rport_destroy() - Free a remote port after last reference is released * @kref: The remote port's kref */ -static void fc_rport_destroy(struct kref *kref) +void fc_rport_destroy(struct kref *kref) { struct fc_rport_priv *rdata; rdata = container_of(kref, struct fc_rport_priv, kref); kfree_rcu(rdata, rcu); } +EXPORT_SYMBOL(fc_rport_destroy); /** * fc_rport_state() - Return a string identifying the remote port's state @@ -242,6 +257,8 @@ static void fc_rport_state_enter(struct fc_rport_priv *rdata, /** * fc_rport_work() - Handler for remote port events in the rport_event_queue * @work: Handle to the remote port being dequeued + * + * Reference counting: drops kref on return */ static void fc_rport_work(struct work_struct *work) { @@ -272,12 +289,14 @@ static void fc_rport_work(struct work_struct *work) kref_get(&rdata->kref); mutex_unlock(&rdata->rp_mutex); - if (!rport) + if (!rport) { + FC_RPORT_DBG(rdata, "No rport!\n"); rport = fc_remote_port_add(lport->host, 0, &ids); + } if (!rport) { FC_RPORT_DBG(rdata, "Failed to add the rport\n"); - lport->tt.rport_logoff(rdata); - kref_put(&rdata->kref, lport->tt.rport_destroy); + fc_rport_logoff(rdata); + kref_put(&rdata->kref, fc_rport_destroy); return; } mutex_lock(&rdata->rp_mutex); @@ -303,7 +322,7 @@ static void fc_rport_work(struct work_struct *work) FC_RPORT_DBG(rdata, "lld callback ev %d\n", event); rdata->lld_event_callback(lport, rdata, event); } - kref_put(&rdata->kref, lport->tt.rport_destroy); + kref_put(&rdata->kref, fc_rport_destroy); break; case RPORT_EV_FAILED: @@ -329,7 +348,8 @@ static void fc_rport_work(struct work_struct *work) FC_RPORT_DBG(rdata, "lld callback ev %d\n", event); rdata->lld_event_callback(lport, rdata, event); } - cancel_delayed_work_sync(&rdata->retry_work); + if (cancel_delayed_work_sync(&rdata->retry_work)) + kref_put(&rdata->kref, fc_rport_destroy); /* * Reset any outstanding exchanges before freeing rport. @@ -351,7 +371,7 @@ static void fc_rport_work(struct work_struct *work) if (port_id == FC_FID_DIR_SERV) { rdata->event = RPORT_EV_NONE; mutex_unlock(&rdata->rp_mutex); - kref_put(&rdata->kref, lport->tt.rport_destroy); + kref_put(&rdata->kref, fc_rport_destroy); } else if ((rdata->flags & FC_RP_STARTED) && rdata->major_retries < lport->max_rport_retry_count) { @@ -362,17 +382,21 @@ static void fc_rport_work(struct work_struct *work) mutex_unlock(&rdata->rp_mutex); } else { FC_RPORT_DBG(rdata, "work delete\n"); + mutex_lock(&lport->disc.disc_mutex); list_del_rcu(&rdata->peers); + mutex_unlock(&lport->disc.disc_mutex); mutex_unlock(&rdata->rp_mutex); - kref_put(&rdata->kref, lport->tt.rport_destroy); + kref_put(&rdata->kref, fc_rport_destroy); } } else { /* * Re-open for events. Reissue READY event if ready. */ rdata->event = RPORT_EV_NONE; - if (rdata->rp_state == RPORT_ST_READY) + if (rdata->rp_state == RPORT_ST_READY) { + FC_RPORT_DBG(rdata, "work reopen\n"); fc_rport_enter_ready(rdata); + } mutex_unlock(&rdata->rp_mutex); } break; @@ -381,12 +405,21 @@ static void fc_rport_work(struct work_struct *work) mutex_unlock(&rdata->rp_mutex); break; } + kref_put(&rdata->kref, fc_rport_destroy); } /** * fc_rport_login() - Start the remote port login state machine * @rdata: The remote port to be logged in to * + * Initiates the RP state machine. It is called from the LP module. + * This function will issue the following commands to the N_Port + * identified by the FC ID provided. + * + * - PLOGI + * - PRLI + * - RTV + * * Locking Note: Called without the rport lock held. This * function will hold the rport lock, call an _enter_* * function and then unlock the rport. @@ -395,10 +428,16 @@ static void fc_rport_work(struct work_struct *work) * If it appears we are already logged in, ADISC is used to verify * the setup. */ -static int fc_rport_login(struct fc_rport_priv *rdata) +int fc_rport_login(struct fc_rport_priv *rdata) { mutex_lock(&rdata->rp_mutex); + if (rdata->flags & FC_RP_STARTED) { + FC_RPORT_DBG(rdata, "port already started\n"); + mutex_unlock(&rdata->rp_mutex); + return 0; + } + rdata->flags |= FC_RP_STARTED; switch (rdata->rp_state) { case RPORT_ST_READY: @@ -408,15 +447,20 @@ static int fc_rport_login(struct fc_rport_priv *rdata) case RPORT_ST_DELETE: FC_RPORT_DBG(rdata, "Restart deleted port\n"); break; - default: + case RPORT_ST_INIT: FC_RPORT_DBG(rdata, "Login to port\n"); fc_rport_enter_flogi(rdata); break; + default: + FC_RPORT_DBG(rdata, "Login in progress, state %s\n", + fc_rport_state(rdata)); + break; } mutex_unlock(&rdata->rp_mutex); return 0; } +EXPORT_SYMBOL(fc_rport_login); /** * fc_rport_enter_delete() - Schedule a remote port to be deleted @@ -431,6 +475,8 @@ static int fc_rport_login(struct fc_rport_priv *rdata) * Set the new event so that the old pending event will not occur. * Since we have the mutex, even if fc_rport_work() is already started, * it'll see the new event. + * + * Reference counting: does not modify kref */ static void fc_rport_enter_delete(struct fc_rport_priv *rdata, enum fc_rport_event event) @@ -442,8 +488,11 @@ static void fc_rport_enter_delete(struct fc_rport_priv *rdata, fc_rport_state_enter(rdata, RPORT_ST_DELETE); - if (rdata->event == RPORT_EV_NONE) - queue_work(rport_event_queue, &rdata->event_work); + kref_get(&rdata->kref); + if (rdata->event == RPORT_EV_NONE && + !queue_work(rport_event_queue, &rdata->event_work)) + kref_put(&rdata->kref, fc_rport_destroy); + rdata->event = event; } @@ -455,7 +504,7 @@ static void fc_rport_enter_delete(struct fc_rport_priv *rdata, * function will hold the rport lock, call an _enter_* * function and then unlock the rport. */ -static int fc_rport_logoff(struct fc_rport_priv *rdata) +int fc_rport_logoff(struct fc_rport_priv *rdata) { struct fc_lport *lport = rdata->local_port; u32 port_id = rdata->ids.port_id; @@ -489,6 +538,7 @@ out: mutex_unlock(&rdata->rp_mutex); return 0; } +EXPORT_SYMBOL(fc_rport_logoff); /** * fc_rport_enter_ready() - Transition to the RPORT_ST_READY state @@ -496,6 +546,8 @@ out: * * Locking Note: The rport lock is expected to be held before calling * this routine. + * + * Reference counting: schedules workqueue, does not modify kref */ static void fc_rport_enter_ready(struct fc_rport_priv *rdata) { @@ -503,8 +555,11 @@ static void fc_rport_enter_ready(struct fc_rport_priv *rdata) FC_RPORT_DBG(rdata, "Port is Ready\n"); - if (rdata->event == RPORT_EV_NONE) - queue_work(rport_event_queue, &rdata->event_work); + kref_get(&rdata->kref); + if (rdata->event == RPORT_EV_NONE && + !queue_work(rport_event_queue, &rdata->event_work)) + kref_put(&rdata->kref, fc_rport_destroy); + rdata->event = RPORT_EV_READY; } @@ -515,6 +570,8 @@ static void fc_rport_enter_ready(struct fc_rport_priv *rdata) * Locking Note: Called without the rport lock held. This * function will hold the rport lock, call an _enter_* * function and then unlock the rport. + * + * Reference counting: Drops kref on return. */ static void fc_rport_timeout(struct work_struct *work) { @@ -522,6 +579,7 @@ static void fc_rport_timeout(struct work_struct *work) container_of(work, struct fc_rport_priv, retry_work.work); mutex_lock(&rdata->rp_mutex); + FC_RPORT_DBG(rdata, "Port timeout, state %s\n", fc_rport_state(rdata)); switch (rdata->rp_state) { case RPORT_ST_FLOGI: @@ -547,23 +605,25 @@ static void fc_rport_timeout(struct work_struct *work) } mutex_unlock(&rdata->rp_mutex); + kref_put(&rdata->kref, fc_rport_destroy); } /** * fc_rport_error() - Error handler, called once retries have been exhausted * @rdata: The remote port the error is happened on - * @fp: The error code encapsulated in a frame pointer + * @err: The error code * * Locking Note: The rport lock is expected to be held before * calling this routine + * + * Reference counting: does not modify kref */ -static void fc_rport_error(struct fc_rport_priv *rdata, struct fc_frame *fp) +static void fc_rport_error(struct fc_rport_priv *rdata, int err) { struct fc_lport *lport = rdata->local_port; - FC_RPORT_DBG(rdata, "Error %ld in state %s, retries %d\n", - IS_ERR(fp) ? -PTR_ERR(fp) : 0, - fc_rport_state(rdata), rdata->retries); + FC_RPORT_DBG(rdata, "Error %d in state %s, retries %d\n", + -err, fc_rport_state(rdata), rdata->retries); switch (rdata->rp_state) { case RPORT_ST_FLOGI: @@ -595,36 +655,39 @@ static void fc_rport_error(struct fc_rport_priv *rdata, struct fc_frame *fp) /** * fc_rport_error_retry() - Handler for remote port state retries * @rdata: The remote port whose state is to be retried - * @fp: The error code encapsulated in a frame pointer + * @err: The error code * * If the error was an exchange timeout retry immediately, * otherwise wait for E_D_TOV. * * Locking Note: The rport lock is expected to be held before * calling this routine + * + * Reference counting: increments kref when scheduling retry_work */ -static void fc_rport_error_retry(struct fc_rport_priv *rdata, - struct fc_frame *fp) +static void fc_rport_error_retry(struct fc_rport_priv *rdata, int err) { - unsigned long delay = msecs_to_jiffies(FC_DEF_E_D_TOV); + unsigned long delay = msecs_to_jiffies(rdata->e_d_tov); /* make sure this isn't an FC_EX_CLOSED error, never retry those */ - if (PTR_ERR(fp) == -FC_EX_CLOSED) + if (err == -FC_EX_CLOSED) goto out; if (rdata->retries < rdata->local_port->max_rport_retry_count) { - FC_RPORT_DBG(rdata, "Error %ld in state %s, retrying\n", - PTR_ERR(fp), fc_rport_state(rdata)); + FC_RPORT_DBG(rdata, "Error %d in state %s, retrying\n", + err, fc_rport_state(rdata)); rdata->retries++; /* no additional delay on exchange timeouts */ - if (PTR_ERR(fp) == -FC_EX_TIMEOUT) + if (err == -FC_EX_TIMEOUT) delay = 0; - schedule_delayed_work(&rdata->retry_work, delay); + kref_get(&rdata->kref); + if (!schedule_delayed_work(&rdata->retry_work, delay)) + kref_put(&rdata->kref, fc_rport_destroy); return; } out: - fc_rport_error(rdata, fp); + fc_rport_error(rdata, err); } /** @@ -684,8 +747,11 @@ static void fc_rport_flogi_resp(struct fc_seq *sp, struct fc_frame *fp, struct fc_lport *lport = rdata->local_port; struct fc_els_flogi *flogi; unsigned int r_a_tov; + u8 opcode; + int err = 0; - FC_RPORT_DBG(rdata, "Received a FLOGI %s\n", fc_els_resp_type(fp)); + FC_RPORT_DBG(rdata, "Received a FLOGI %s\n", + IS_ERR(fp) ? "error" : fc_els_resp_type(fp)); if (fp == ERR_PTR(-FC_EX_CLOSED)) goto put; @@ -701,18 +767,34 @@ static void fc_rport_flogi_resp(struct fc_seq *sp, struct fc_frame *fp, } if (IS_ERR(fp)) { - fc_rport_error(rdata, fp); + fc_rport_error(rdata, PTR_ERR(fp)); goto err; } - - if (fc_frame_payload_op(fp) != ELS_LS_ACC) + opcode = fc_frame_payload_op(fp); + if (opcode == ELS_LS_RJT) { + struct fc_els_ls_rjt *rjt; + + rjt = fc_frame_payload_get(fp, sizeof(*rjt)); + FC_RPORT_DBG(rdata, "FLOGI ELS rejected, reason %x expl %x\n", + rjt->er_reason, rjt->er_explan); + err = -FC_EX_ELS_RJT; goto bad; - if (fc_rport_login_complete(rdata, fp)) + } else if (opcode != ELS_LS_ACC) { + FC_RPORT_DBG(rdata, "FLOGI ELS invalid opcode %x\n", opcode); + err = -FC_EX_ELS_RJT; goto bad; + } + if (fc_rport_login_complete(rdata, fp)) { + FC_RPORT_DBG(rdata, "FLOGI failed, no login\n"); + err = -FC_EX_INV_LOGIN; + goto bad; + } flogi = fc_frame_payload_get(fp, sizeof(*flogi)); - if (!flogi) + if (!flogi) { + err = -FC_EX_ALLOC_ERR; goto bad; + } r_a_tov = ntohl(flogi->fl_csp.sp_r_a_tov); if (r_a_tov > rdata->r_a_tov) rdata->r_a_tov = r_a_tov; @@ -726,11 +808,11 @@ out: err: mutex_unlock(&rdata->rp_mutex); put: - kref_put(&rdata->kref, lport->tt.rport_destroy); + kref_put(&rdata->kref, fc_rport_destroy); return; bad: FC_RPORT_DBG(rdata, "Bad FLOGI response\n"); - fc_rport_error_retry(rdata, fp); + fc_rport_error_retry(rdata, err); goto out; } @@ -740,6 +822,8 @@ bad: * * Locking Note: The rport lock is expected to be held before calling * this routine. + * + * Reference counting: increments kref when sending ELS */ static void fc_rport_enter_flogi(struct fc_rport_priv *rdata) { @@ -756,20 +840,23 @@ static void fc_rport_enter_flogi(struct fc_rport_priv *rdata) fp = fc_frame_alloc(lport, sizeof(struct fc_els_flogi)); if (!fp) - return fc_rport_error_retry(rdata, fp); + return fc_rport_error_retry(rdata, -FC_EX_ALLOC_ERR); + kref_get(&rdata->kref); if (!lport->tt.elsct_send(lport, rdata->ids.port_id, fp, ELS_FLOGI, fc_rport_flogi_resp, rdata, - 2 * lport->r_a_tov)) - fc_rport_error_retry(rdata, NULL); - else - kref_get(&rdata->kref); + 2 * lport->r_a_tov)) { + fc_rport_error_retry(rdata, -FC_EX_XMIT_ERR); + kref_put(&rdata->kref, fc_rport_destroy); + } } /** * fc_rport_recv_flogi_req() - Handle Fabric Login (FLOGI) request in p-mp mode * @lport: The local port that received the PLOGI request * @rx_fp: The PLOGI request frame + * + * Reference counting: drops kref on return */ static void fc_rport_recv_flogi_req(struct fc_lport *lport, struct fc_frame *rx_fp) @@ -799,7 +886,7 @@ static void fc_rport_recv_flogi_req(struct fc_lport *lport, goto reject; } - rdata = lport->tt.rport_lookup(lport, sid); + rdata = fc_rport_lookup(lport, sid); if (!rdata) { rjt_data.reason = ELS_RJT_FIP; rjt_data.explan = ELS_EXPL_NOT_NEIGHBOR; @@ -824,8 +911,7 @@ static void fc_rport_recv_flogi_req(struct fc_lport *lport, * RPORT wouldn;t have created and 'rport_lookup' would have * failed anyway in that case. */ - if (lport->point_to_multipoint) - break; + break; case RPORT_ST_DELETE: mutex_unlock(&rdata->rp_mutex); rjt_data.reason = ELS_RJT_FIP; @@ -867,20 +953,27 @@ static void fc_rport_recv_flogi_req(struct fc_lport *lport, fc_fill_reply_hdr(fp, rx_fp, FC_RCTL_ELS_REP, 0); lport->tt.frame_send(lport, fp); - if (rdata->ids.port_name < lport->wwpn) - fc_rport_enter_plogi(rdata); - else - fc_rport_state_enter(rdata, RPORT_ST_PLOGI_WAIT); + /* + * Do not proceed with the state machine if our + * FLOGI has crossed with an FLOGI from the + * remote port; wait for the FLOGI response instead. + */ + if (rdata->rp_state != RPORT_ST_FLOGI) { + if (rdata->ids.port_name < lport->wwpn) + fc_rport_enter_plogi(rdata); + else + fc_rport_state_enter(rdata, RPORT_ST_PLOGI_WAIT); + } out: mutex_unlock(&rdata->rp_mutex); - kref_put(&rdata->kref, lport->tt.rport_destroy); + kref_put(&rdata->kref, fc_rport_destroy); fc_frame_free(rx_fp); return; reject_put: - kref_put(&rdata->kref, lport->tt.rport_destroy); + kref_put(&rdata->kref, fc_rport_destroy); reject: - lport->tt.seq_els_rsp_send(rx_fp, ELS_LS_RJT, &rjt_data); + fc_seq_els_rsp_send(rx_fp, ELS_LS_RJT, &rjt_data); fc_frame_free(rx_fp); } @@ -904,10 +997,13 @@ static void fc_rport_plogi_resp(struct fc_seq *sp, struct fc_frame *fp, u16 cssp_seq; u8 op; - mutex_lock(&rdata->rp_mutex); - FC_RPORT_DBG(rdata, "Received a PLOGI %s\n", fc_els_resp_type(fp)); + if (fp == ERR_PTR(-FC_EX_CLOSED)) + goto put; + + mutex_lock(&rdata->rp_mutex); + if (rdata->rp_state != RPORT_ST_PLOGI) { FC_RPORT_DBG(rdata, "Received a PLOGI response, but in state " "%s\n", fc_rport_state(rdata)); @@ -917,7 +1013,7 @@ static void fc_rport_plogi_resp(struct fc_seq *sp, struct fc_frame *fp, } if (IS_ERR(fp)) { - fc_rport_error_retry(rdata, fp); + fc_rport_error_retry(rdata, PTR_ERR(fp)); goto err; } @@ -939,14 +1035,20 @@ static void fc_rport_plogi_resp(struct fc_seq *sp, struct fc_frame *fp, rdata->max_seq = csp_seq; rdata->maxframe_size = fc_plogi_get_maxframe(plp, lport->mfs); fc_rport_enter_prli(rdata); - } else - fc_rport_error_retry(rdata, fp); + } else { + struct fc_els_ls_rjt *rjt; + rjt = fc_frame_payload_get(fp, sizeof(*rjt)); + FC_RPORT_DBG(rdata, "PLOGI ELS rejected, reason %x expl %x\n", + rjt->er_reason, rjt->er_explan); + fc_rport_error_retry(rdata, -FC_EX_ELS_RJT); + } out: fc_frame_free(fp); err: mutex_unlock(&rdata->rp_mutex); - kref_put(&rdata->kref, lport->tt.rport_destroy); +put: + kref_put(&rdata->kref, fc_rport_destroy); } static bool @@ -969,6 +1071,8 @@ fc_rport_compatible_roles(struct fc_lport *lport, struct fc_rport_priv *rdata) * * Locking Note: The rport lock is expected to be held before calling * this routine. + * + * Reference counting: increments kref when sending ELS */ static void fc_rport_enter_plogi(struct fc_rport_priv *rdata) { @@ -990,17 +1094,18 @@ static void fc_rport_enter_plogi(struct fc_rport_priv *rdata) fp = fc_frame_alloc(lport, sizeof(struct fc_els_flogi)); if (!fp) { FC_RPORT_DBG(rdata, "%s frame alloc failed\n", __func__); - fc_rport_error_retry(rdata, fp); + fc_rport_error_retry(rdata, -FC_EX_ALLOC_ERR); return; } rdata->e_d_tov = lport->e_d_tov; + kref_get(&rdata->kref); if (!lport->tt.elsct_send(lport, rdata->ids.port_id, fp, ELS_PLOGI, fc_rport_plogi_resp, rdata, - 2 * lport->r_a_tov)) - fc_rport_error_retry(rdata, NULL); - else - kref_get(&rdata->kref); + 2 * lport->r_a_tov)) { + fc_rport_error_retry(rdata, -FC_EX_XMIT_ERR); + kref_put(&rdata->kref, fc_rport_destroy); + } } /** @@ -1022,16 +1127,20 @@ static void fc_rport_prli_resp(struct fc_seq *sp, struct fc_frame *fp, struct fc_els_spp spp; } *pp; struct fc_els_spp temp_spp; + struct fc_els_ls_rjt *rjt; struct fc4_prov *prov; u32 roles = FC_RPORT_ROLE_UNKNOWN; u32 fcp_parm = 0; u8 op; - u8 resp_code = 0; - - mutex_lock(&rdata->rp_mutex); + enum fc_els_spp_resp resp_code; FC_RPORT_DBG(rdata, "Received a PRLI %s\n", fc_els_resp_type(fp)); + if (fp == ERR_PTR(-FC_EX_CLOSED)) + goto put; + + mutex_lock(&rdata->rp_mutex); + if (rdata->rp_state != RPORT_ST_PRLI) { FC_RPORT_DBG(rdata, "Received a PRLI response, but in state " "%s\n", fc_rport_state(rdata)); @@ -1041,7 +1150,7 @@ static void fc_rport_prli_resp(struct fc_seq *sp, struct fc_frame *fp, } if (IS_ERR(fp)) { - fc_rport_error_retry(rdata, fp); + fc_rport_error_retry(rdata, PTR_ERR(fp)); goto err; } @@ -1055,14 +1164,14 @@ static void fc_rport_prli_resp(struct fc_seq *sp, struct fc_frame *fp, goto out; resp_code = (pp->spp.spp_flags & FC_SPP_RESP_MASK); - FC_RPORT_DBG(rdata, "PRLI spp_flags = 0x%x\n", - pp->spp.spp_flags); + FC_RPORT_DBG(rdata, "PRLI spp_flags = 0x%x spp_type 0x%x\n", + pp->spp.spp_flags, pp->spp.spp_type); rdata->spp_type = pp->spp.spp_type; if (resp_code != FC_SPP_RESP_ACK) { if (resp_code == FC_SPP_RESP_CONF) - fc_rport_error(rdata, fp); + fc_rport_error(rdata, -FC_EX_SEQ_ERR); else - fc_rport_error_retry(rdata, fp); + fc_rport_error_retry(rdata, -FC_EX_SEQ_ERR); goto out; } if (pp->prli.prli_spp_len < sizeof(pp->spp)) @@ -1074,13 +1183,25 @@ static void fc_rport_prli_resp(struct fc_seq *sp, struct fc_frame *fp, if (fcp_parm & FCP_SPPF_CONF_COMPL) rdata->flags |= FC_RP_FLAGS_CONF_REQ; - prov = fc_passive_prov[FC_TYPE_FCP]; + /* + * Call prli provider if we should act as a target + */ + prov = fc_passive_prov[rdata->spp_type]; if (prov) { memset(&temp_spp, 0, sizeof(temp_spp)); prov->prli(rdata, pp->prli.prli_spp_len, &pp->spp, &temp_spp); } - + /* + * Check if the image pair could be established + */ + if (rdata->spp_type != FC_TYPE_FCP || + !(pp->spp.spp_flags & FC_SPP_EST_IMG_PAIR)) { + /* + * Nope; we can't use this port as a target. + */ + fcp_parm &= ~FCP_SPPF_TARG_FCN; + } rdata->supported_classes = FC_COS_CLASS3; if (fcp_parm & FCP_SPPF_INIT_FCN) roles |= FC_RPORT_ROLE_FCP_INITIATOR; @@ -1091,15 +1212,18 @@ static void fc_rport_prli_resp(struct fc_seq *sp, struct fc_frame *fp, fc_rport_enter_rtv(rdata); } else { - FC_RPORT_DBG(rdata, "Bad ELS response for PRLI command\n"); - fc_rport_error_retry(rdata, fp); + rjt = fc_frame_payload_get(fp, sizeof(*rjt)); + FC_RPORT_DBG(rdata, "PRLI ELS rejected, reason %x expl %x\n", + rjt->er_reason, rjt->er_explan); + fc_rport_error_retry(rdata, FC_EX_ELS_RJT); } out: fc_frame_free(fp); err: mutex_unlock(&rdata->rp_mutex); - kref_put(&rdata->kref, rdata->local_port->tt.rport_destroy); +put: + kref_put(&rdata->kref, fc_rport_destroy); } /** @@ -1108,6 +1232,8 @@ err: * * Locking Note: The rport lock is expected to be held before calling * this routine. + * + * Reference counting: increments kref when sending ELS */ static void fc_rport_enter_prli(struct fc_rport_priv *rdata) { @@ -1128,6 +1254,15 @@ static void fc_rport_enter_prli(struct fc_rport_priv *rdata) return; } + /* + * And if the local port does not support the initiator function + * there's no need to send a PRLI, either. + */ + if (!(lport->service_params & FCP_SPPF_INIT_FCN)) { + fc_rport_enter_ready(rdata); + return; + } + FC_RPORT_DBG(rdata, "Port entered PRLI state from %s state\n", fc_rport_state(rdata)); @@ -1135,7 +1270,7 @@ static void fc_rport_enter_prli(struct fc_rport_priv *rdata) fp = fc_frame_alloc(lport, sizeof(*pp)); if (!fp) { - fc_rport_error_retry(rdata, fp); + fc_rport_error_retry(rdata, -FC_EX_ALLOC_ERR); return; } @@ -1151,15 +1286,16 @@ static void fc_rport_enter_prli(struct fc_rport_priv *rdata) fc_host_port_id(lport->host), FC_TYPE_ELS, FC_FC_FIRST_SEQ | FC_FC_END_SEQ | FC_FC_SEQ_INIT, 0); - if (!lport->tt.exch_seq_send(lport, fp, fc_rport_prli_resp, - NULL, rdata, 2 * lport->r_a_tov)) - fc_rport_error_retry(rdata, NULL); - else - kref_get(&rdata->kref); + kref_get(&rdata->kref); + if (!fc_exch_seq_send(lport, fp, fc_rport_prli_resp, + NULL, rdata, 2 * lport->r_a_tov)) { + fc_rport_error_retry(rdata, -FC_EX_XMIT_ERR); + kref_put(&rdata->kref, fc_rport_destroy); + } } /** - * fc_rport_els_rtv_resp() - Handler for Request Timeout Value (RTV) responses + * fc_rport_rtv_resp() - Handler for Request Timeout Value (RTV) responses * @sp: The sequence the RTV was on * @fp: The RTV response frame * @rdata_arg: The remote port that sent the RTV response @@ -1176,10 +1312,13 @@ static void fc_rport_rtv_resp(struct fc_seq *sp, struct fc_frame *fp, struct fc_rport_priv *rdata = rdata_arg; u8 op; - mutex_lock(&rdata->rp_mutex); - FC_RPORT_DBG(rdata, "Received a RTV %s\n", fc_els_resp_type(fp)); + if (fp == ERR_PTR(-FC_EX_CLOSED)) + goto put; + + mutex_lock(&rdata->rp_mutex); + if (rdata->rp_state != RPORT_ST_RTV) { FC_RPORT_DBG(rdata, "Received a RTV response, but in state " "%s\n", fc_rport_state(rdata)); @@ -1189,7 +1328,7 @@ static void fc_rport_rtv_resp(struct fc_seq *sp, struct fc_frame *fp, } if (IS_ERR(fp)) { - fc_rport_error(rdata, fp); + fc_rport_error(rdata, PTR_ERR(fp)); goto err; } @@ -1205,13 +1344,15 @@ static void fc_rport_rtv_resp(struct fc_seq *sp, struct fc_frame *fp, tov = ntohl(rtv->rtv_r_a_tov); if (tov == 0) tov = 1; - rdata->r_a_tov = tov; + if (tov > rdata->r_a_tov) + rdata->r_a_tov = tov; tov = ntohl(rtv->rtv_e_d_tov); if (toq & FC_ELS_RTV_EDRES) tov /= 1000000; if (tov == 0) tov = 1; - rdata->e_d_tov = tov; + if (tov > rdata->e_d_tov) + rdata->e_d_tov = tov; } } @@ -1221,7 +1362,8 @@ out: fc_frame_free(fp); err: mutex_unlock(&rdata->rp_mutex); - kref_put(&rdata->kref, rdata->local_port->tt.rport_destroy); +put: + kref_put(&rdata->kref, fc_rport_destroy); } /** @@ -1230,6 +1372,8 @@ err: * * Locking Note: The rport lock is expected to be held before calling * this routine. + * + * Reference counting: increments kref when sending ELS */ static void fc_rport_enter_rtv(struct fc_rport_priv *rdata) { @@ -1243,16 +1387,52 @@ static void fc_rport_enter_rtv(struct fc_rport_priv *rdata) fp = fc_frame_alloc(lport, sizeof(struct fc_els_rtv)); if (!fp) { - fc_rport_error_retry(rdata, fp); + fc_rport_error_retry(rdata, -FC_EX_ALLOC_ERR); return; } + kref_get(&rdata->kref); if (!lport->tt.elsct_send(lport, rdata->ids.port_id, fp, ELS_RTV, fc_rport_rtv_resp, rdata, - 2 * lport->r_a_tov)) - fc_rport_error_retry(rdata, NULL); - else - kref_get(&rdata->kref); + 2 * lport->r_a_tov)) { + fc_rport_error_retry(rdata, -FC_EX_XMIT_ERR); + kref_put(&rdata->kref, fc_rport_destroy); + } +} + +/** + * fc_rport_recv_rtv_req() - Handler for Read Timeout Value (RTV) requests + * @rdata: The remote port that sent the RTV request + * @in_fp: The RTV request frame + * + * Locking Note: Called with the lport and rport locks held. + */ +static void fc_rport_recv_rtv_req(struct fc_rport_priv *rdata, + struct fc_frame *in_fp) +{ + struct fc_lport *lport = rdata->local_port; + struct fc_frame *fp; + struct fc_els_rtv_acc *rtv; + struct fc_seq_els_data rjt_data; + + FC_RPORT_DBG(rdata, "Received RTV request\n"); + + fp = fc_frame_alloc(lport, sizeof(*rtv)); + if (!fp) { + rjt_data.reason = ELS_RJT_UNAB; + rjt_data.reason = ELS_EXPL_INSUF_RES; + fc_seq_els_rsp_send(in_fp, ELS_LS_RJT, &rjt_data); + goto drop; + } + rtv = fc_frame_payload_get(fp, sizeof(*rtv)); + rtv->rtv_cmd = ELS_LS_ACC; + rtv->rtv_r_a_tov = htonl(lport->r_a_tov); + rtv->rtv_e_d_tov = htonl(lport->e_d_tov); + rtv->rtv_toq = 0; + fc_fill_reply_hdr(fp, in_fp, FC_RCTL_ELS_REP, 0); + lport->tt.frame_send(lport, fp); +drop: + fc_frame_free(in_fp); } /** @@ -1262,15 +1442,16 @@ static void fc_rport_enter_rtv(struct fc_rport_priv *rdata) * @lport_arg: The local port */ static void fc_rport_logo_resp(struct fc_seq *sp, struct fc_frame *fp, - void *lport_arg) + void *rdata_arg) { - struct fc_lport *lport = lport_arg; + struct fc_rport_priv *rdata = rdata_arg; + struct fc_lport *lport = rdata->local_port; FC_RPORT_ID_DBG(lport, fc_seq_exch(sp)->did, "Received a LOGO %s\n", fc_els_resp_type(fp)); - if (IS_ERR(fp)) - return; - fc_frame_free(fp); + if (!IS_ERR(fp)) + fc_frame_free(fp); + kref_put(&rdata->kref, fc_rport_destroy); } /** @@ -1279,6 +1460,8 @@ static void fc_rport_logo_resp(struct fc_seq *sp, struct fc_frame *fp, * * Locking Note: The rport lock is expected to be held before calling * this routine. + * + * Reference counting: increments kref when sending ELS */ static void fc_rport_enter_logo(struct fc_rport_priv *rdata) { @@ -1291,8 +1474,10 @@ static void fc_rport_enter_logo(struct fc_rport_priv *rdata) fp = fc_frame_alloc(lport, sizeof(struct fc_els_logo)); if (!fp) return; - (void)lport->tt.elsct_send(lport, rdata->ids.port_id, fp, ELS_LOGO, - fc_rport_logo_resp, lport, 0); + kref_get(&rdata->kref); + if (!lport->tt.elsct_send(lport, rdata->ids.port_id, fp, ELS_LOGO, + fc_rport_logo_resp, rdata, 0)) + kref_put(&rdata->kref, fc_rport_destroy); } /** @@ -1312,10 +1497,13 @@ static void fc_rport_adisc_resp(struct fc_seq *sp, struct fc_frame *fp, struct fc_els_adisc *adisc; u8 op; - mutex_lock(&rdata->rp_mutex); - FC_RPORT_DBG(rdata, "Received a ADISC response\n"); + if (fp == ERR_PTR(-FC_EX_CLOSED)) + goto put; + + mutex_lock(&rdata->rp_mutex); + if (rdata->rp_state != RPORT_ST_ADISC) { FC_RPORT_DBG(rdata, "Received a ADISC resp but in state %s\n", fc_rport_state(rdata)); @@ -1325,7 +1513,7 @@ static void fc_rport_adisc_resp(struct fc_seq *sp, struct fc_frame *fp, } if (IS_ERR(fp)) { - fc_rport_error(rdata, fp); + fc_rport_error(rdata, PTR_ERR(fp)); goto err; } @@ -1350,7 +1538,8 @@ out: fc_frame_free(fp); err: mutex_unlock(&rdata->rp_mutex); - kref_put(&rdata->kref, rdata->local_port->tt.rport_destroy); +put: + kref_put(&rdata->kref, fc_rport_destroy); } /** @@ -1359,6 +1548,8 @@ err: * * Locking Note: The rport lock is expected to be held before calling * this routine. + * + * Reference counting: increments kref when sending ELS */ static void fc_rport_enter_adisc(struct fc_rport_priv *rdata) { @@ -1372,15 +1563,16 @@ static void fc_rport_enter_adisc(struct fc_rport_priv *rdata) fp = fc_frame_alloc(lport, sizeof(struct fc_els_adisc)); if (!fp) { - fc_rport_error_retry(rdata, fp); + fc_rport_error_retry(rdata, -FC_EX_ALLOC_ERR); return; } + kref_get(&rdata->kref); if (!lport->tt.elsct_send(lport, rdata->ids.port_id, fp, ELS_ADISC, fc_rport_adisc_resp, rdata, - 2 * lport->r_a_tov)) - fc_rport_error_retry(rdata, NULL); - else - kref_get(&rdata->kref); + 2 * lport->r_a_tov)) { + fc_rport_error_retry(rdata, -FC_EX_XMIT_ERR); + kref_put(&rdata->kref, fc_rport_destroy); + } } /** @@ -1404,7 +1596,7 @@ static void fc_rport_recv_adisc_req(struct fc_rport_priv *rdata, if (!adisc) { rjt_data.reason = ELS_RJT_PROT; rjt_data.explan = ELS_EXPL_INV_LEN; - lport->tt.seq_els_rsp_send(in_fp, ELS_LS_RJT, &rjt_data); + fc_seq_els_rsp_send(in_fp, ELS_LS_RJT, &rjt_data); goto drop; } @@ -1480,7 +1672,7 @@ static void fc_rport_recv_rls_req(struct fc_rport_priv *rdata, goto out; out_rjt: - lport->tt.seq_els_rsp_send(rx_fp, ELS_LS_RJT, &rjt_data); + fc_seq_els_rsp_send(rx_fp, ELS_LS_RJT, &rjt_data); out: fc_frame_free(rx_fp); } @@ -1494,15 +1686,21 @@ out: * The ELS opcode has already been validated by the caller. * * Locking Note: Called with the lport lock held. + * + * Reference counting: does not modify kref */ static void fc_rport_recv_els_req(struct fc_lport *lport, struct fc_frame *fp) { struct fc_rport_priv *rdata; struct fc_seq_els_data els_data; - rdata = lport->tt.rport_lookup(lport, fc_frame_sid(fp)); - if (!rdata) + rdata = fc_rport_lookup(lport, fc_frame_sid(fp)); + if (!rdata) { + FC_RPORT_ID_DBG(lport, fc_frame_sid(fp), + "Received ELS 0x%02x from non-logged-in port\n", + fc_frame_payload_op(fp)); goto reject; + } mutex_lock(&rdata->rp_mutex); @@ -1512,9 +1710,21 @@ static void fc_rport_recv_els_req(struct fc_lport *lport, struct fc_frame *fp) case RPORT_ST_READY: case RPORT_ST_ADISC: break; + case RPORT_ST_PLOGI: + if (fc_frame_payload_op(fp) == ELS_PRLI) { + FC_RPORT_DBG(rdata, "Reject ELS PRLI " + "while in state %s\n", + fc_rport_state(rdata)); + mutex_unlock(&rdata->rp_mutex); + kref_put(&rdata->kref, fc_rport_destroy); + goto busy; + } default: + FC_RPORT_DBG(rdata, + "Reject ELS 0x%02x while in state %s\n", + fc_frame_payload_op(fp), fc_rport_state(rdata)); mutex_unlock(&rdata->rp_mutex); - kref_put(&rdata->kref, lport->tt.rport_destroy); + kref_put(&rdata->kref, fc_rport_destroy); goto reject; } @@ -1529,30 +1739,41 @@ static void fc_rport_recv_els_req(struct fc_lport *lport, struct fc_frame *fp) fc_rport_recv_adisc_req(rdata, fp); break; case ELS_RRQ: - lport->tt.seq_els_rsp_send(fp, ELS_RRQ, NULL); + fc_seq_els_rsp_send(fp, ELS_RRQ, NULL); fc_frame_free(fp); break; case ELS_REC: - lport->tt.seq_els_rsp_send(fp, ELS_REC, NULL); + fc_seq_els_rsp_send(fp, ELS_REC, NULL); fc_frame_free(fp); break; case ELS_RLS: fc_rport_recv_rls_req(rdata, fp); break; + case ELS_RTV: + fc_rport_recv_rtv_req(rdata, fp); + break; default: fc_frame_free(fp); /* can't happen */ break; } mutex_unlock(&rdata->rp_mutex); - kref_put(&rdata->kref, rdata->local_port->tt.rport_destroy); + kref_put(&rdata->kref, fc_rport_destroy); return; reject: els_data.reason = ELS_RJT_UNAB; els_data.explan = ELS_EXPL_PLOGI_REQD; - lport->tt.seq_els_rsp_send(fp, ELS_LS_RJT, &els_data); + fc_seq_els_rsp_send(fp, ELS_LS_RJT, &els_data); + fc_frame_free(fp); + return; + +busy: + els_data.reason = ELS_RJT_BUSY; + els_data.explan = ELS_EXPL_NONE; + fc_seq_els_rsp_send(fp, ELS_LS_RJT, &els_data); fc_frame_free(fp); + return; } /** @@ -1561,8 +1782,10 @@ reject: * @fp: The request frame * * Locking Note: Called with the lport lock held. + * + * Reference counting: does not modify kref */ -static void fc_rport_recv_req(struct fc_lport *lport, struct fc_frame *fp) +void fc_rport_recv_req(struct fc_lport *lport, struct fc_frame *fp) { struct fc_seq_els_data els_data; @@ -1588,16 +1811,18 @@ static void fc_rport_recv_req(struct fc_lport *lport, struct fc_frame *fp) case ELS_RRQ: case ELS_REC: case ELS_RLS: + case ELS_RTV: fc_rport_recv_els_req(lport, fp); break; default: els_data.reason = ELS_RJT_UNSUP; els_data.explan = ELS_EXPL_NONE; - lport->tt.seq_els_rsp_send(fp, ELS_LS_RJT, &els_data); + fc_seq_els_rsp_send(fp, ELS_LS_RJT, &els_data); fc_frame_free(fp); break; } } +EXPORT_SYMBOL(fc_rport_recv_req); /** * fc_rport_recv_plogi_req() - Handler for Port Login (PLOGI) requests @@ -1605,6 +1830,8 @@ static void fc_rport_recv_req(struct fc_lport *lport, struct fc_frame *fp) * @rx_fp: The PLOGI request frame * * Locking Note: The rport lock is held before calling this function. + * + * Reference counting: increments kref on return */ static void fc_rport_recv_plogi_req(struct fc_lport *lport, struct fc_frame *rx_fp) @@ -1630,7 +1857,7 @@ static void fc_rport_recv_plogi_req(struct fc_lport *lport, disc = &lport->disc; mutex_lock(&disc->disc_mutex); - rdata = lport->tt.rport_create(lport, sid); + rdata = fc_rport_create(lport, sid); if (!rdata) { mutex_unlock(&disc->disc_mutex); rjt_data.reason = ELS_RJT_UNAB; @@ -1718,7 +1945,7 @@ out: return; reject: - lport->tt.seq_els_rsp_send(fp, ELS_LS_RJT, &rjt_data); + fc_seq_els_rsp_send(fp, ELS_LS_RJT, &rjt_data); fc_frame_free(fp); } @@ -1744,7 +1971,6 @@ static void fc_rport_recv_prli_req(struct fc_rport_priv *rdata, unsigned int len; unsigned int plen; enum fc_els_spp_resp resp; - enum fc_els_spp_resp passive; struct fc_seq_els_data rjt_data; struct fc4_prov *prov; @@ -1794,15 +2020,21 @@ static void fc_rport_recv_prli_req(struct fc_rport_priv *rdata, resp = 0; if (rspp->spp_type < FC_FC4_PROV_SIZE) { + enum fc_els_spp_resp active = 0, passive = 0; + prov = fc_active_prov[rspp->spp_type]; if (prov) - resp = prov->prli(rdata, plen, rspp, spp); + active = prov->prli(rdata, plen, rspp, spp); prov = fc_passive_prov[rspp->spp_type]; - if (prov) { + if (prov) passive = prov->prli(rdata, plen, rspp, spp); - if (!resp || passive == FC_SPP_RESP_ACK) - resp = passive; - } + if (!active || passive == FC_SPP_RESP_ACK) + resp = passive; + else + resp = active; + FC_RPORT_DBG(rdata, "PRLI rspp type %x " + "active %x passive %x\n", + rspp->spp_type, active, passive); } if (!resp) { if (spp->spp_flags & FC_SPP_EST_IMG_PAIR) @@ -1823,20 +2055,13 @@ static void fc_rport_recv_prli_req(struct fc_rport_priv *rdata, fc_fill_reply_hdr(fp, rx_fp, FC_RCTL_ELS_REP, 0); lport->tt.frame_send(lport, fp); - switch (rdata->rp_state) { - case RPORT_ST_PRLI: - fc_rport_enter_ready(rdata); - break; - default: - break; - } goto drop; reject_len: rjt_data.reason = ELS_RJT_PROT; rjt_data.explan = ELS_EXPL_INV_LEN; reject: - lport->tt.seq_els_rsp_send(rx_fp, ELS_LS_RJT, &rjt_data); + fc_seq_els_rsp_send(rx_fp, ELS_LS_RJT, &rjt_data); drop: fc_frame_free(rx_fp); } @@ -1907,7 +2132,7 @@ reject_len: rjt_data.reason = ELS_RJT_PROT; rjt_data.explan = ELS_EXPL_INV_LEN; reject: - lport->tt.seq_els_rsp_send(rx_fp, ELS_LS_RJT, &rjt_data); + fc_seq_els_rsp_send(rx_fp, ELS_LS_RJT, &rjt_data); drop: fc_frame_free(rx_fp); } @@ -1919,17 +2144,19 @@ drop: * * Locking Note: The rport lock is expected to be held before calling * this function. + * + * Reference counting: drops kref on return */ static void fc_rport_recv_logo_req(struct fc_lport *lport, struct fc_frame *fp) { struct fc_rport_priv *rdata; u32 sid; - lport->tt.seq_els_rsp_send(fp, ELS_LS_ACC, NULL); + fc_seq_els_rsp_send(fp, ELS_LS_ACC, NULL); sid = fc_frame_sid(fp); - rdata = lport->tt.rport_lookup(lport, sid); + rdata = fc_rport_lookup(lport, sid); if (rdata) { mutex_lock(&rdata->rp_mutex); FC_RPORT_DBG(rdata, "Received LOGO request while in state %s\n", @@ -1937,7 +2164,7 @@ static void fc_rport_recv_logo_req(struct fc_lport *lport, struct fc_frame *fp) fc_rport_enter_delete(rdata, RPORT_EV_STOP); mutex_unlock(&rdata->rp_mutex); - kref_put(&rdata->kref, rdata->local_port->tt.rport_destroy); + kref_put(&rdata->kref, fc_rport_destroy); } else FC_RPORT_ID_DBG(lport, sid, "Received LOGO from non-logged-in port\n"); @@ -1947,41 +2174,11 @@ static void fc_rport_recv_logo_req(struct fc_lport *lport, struct fc_frame *fp) /** * fc_rport_flush_queue() - Flush the rport_event_queue */ -static void fc_rport_flush_queue(void) +void fc_rport_flush_queue(void) { flush_workqueue(rport_event_queue); } - -/** - * fc_rport_init() - Initialize the remote port layer for a local port - * @lport: The local port to initialize the remote port layer for - */ -int fc_rport_init(struct fc_lport *lport) -{ - if (!lport->tt.rport_lookup) - lport->tt.rport_lookup = fc_rport_lookup; - - if (!lport->tt.rport_create) - lport->tt.rport_create = fc_rport_create; - - if (!lport->tt.rport_login) - lport->tt.rport_login = fc_rport_login; - - if (!lport->tt.rport_logoff) - lport->tt.rport_logoff = fc_rport_logoff; - - if (!lport->tt.rport_recv_req) - lport->tt.rport_recv_req = fc_rport_recv_req; - - if (!lport->tt.rport_flush_queue) - lport->tt.rport_flush_queue = fc_rport_flush_queue; - - if (!lport->tt.rport_destroy) - lport->tt.rport_destroy = fc_rport_destroy; - - return 0; -} -EXPORT_SYMBOL(fc_rport_init); +EXPORT_SYMBOL(fc_rport_flush_queue); /** * fc_rport_fcp_prli() - Handle incoming PRLI for the FCP initiator. diff --git a/drivers/scsi/lpfc/lpfc.h b/drivers/scsi/lpfc/lpfc.h index b484859..8a20b4e 100644 --- a/drivers/scsi/lpfc/lpfc.h +++ b/drivers/scsi/lpfc/lpfc.h @@ -648,6 +648,10 @@ struct lpfc_hba { #define HBA_FCP_IOQ_FLUSH 0x8000 /* FCP I/O queues being flushed */ #define HBA_FW_DUMP_OP 0x10000 /* Skips fn reset before FW dump */ #define HBA_RECOVERABLE_UE 0x20000 /* Firmware supports recoverable UE */ +#define HBA_FORCED_LINK_SPEED 0x40000 /* + * Firmware supports Forced Link Speed + * capability + */ uint32_t fcp_ring_in_use; /* When polling test if intr-hndlr active*/ struct lpfc_dmabuf slim2p; @@ -746,6 +750,8 @@ struct lpfc_hba { uint32_t cfg_oas_priority; uint32_t cfg_XLanePriority; uint32_t cfg_enable_bg; + uint32_t cfg_prot_mask; + uint32_t cfg_prot_guard; uint32_t cfg_hostmem_hgp; uint32_t cfg_log_verbose; uint32_t cfg_aer_support; diff --git a/drivers/scsi/lpfc/lpfc_attr.c b/drivers/scsi/lpfc/lpfc_attr.c index f101990..c847755 100644 --- a/drivers/scsi/lpfc/lpfc_attr.c +++ b/drivers/scsi/lpfc/lpfc_attr.c @@ -2759,18 +2759,14 @@ LPFC_ATTR_R(enable_npiv, 1, 0, 1, LPFC_ATTR_R(fcf_failover_policy, 1, 1, 2, "FCF Fast failover=1 Priority failover=2"); -int lpfc_enable_rrq = 2; -module_param(lpfc_enable_rrq, int, S_IRUGO); -MODULE_PARM_DESC(lpfc_enable_rrq, "Enable RRQ functionality"); -lpfc_param_show(enable_rrq); /* # lpfc_enable_rrq: Track XRI/OXID reuse after IO failures # 0x0 = disabled, XRI/OXID use not tracked. # 0x1 = XRI/OXID reuse is timed with ratov, RRQ sent. # 0x2 = XRI/OXID reuse is timed with ratov, No RRQ sent. */ -lpfc_param_init(enable_rrq, 2, 0, 2); -static DEVICE_ATTR(lpfc_enable_rrq, S_IRUGO, lpfc_enable_rrq_show, NULL); +LPFC_ATTR_R(enable_rrq, 2, 0, 2, + "Enable RRQ functionality"); /* # lpfc_suppress_link_up: Bring link up at initialization @@ -2827,14 +2823,8 @@ lpfc_txcmplq_hw_show(struct device *dev, struct device_attribute *attr, static DEVICE_ATTR(txcmplq_hw, S_IRUGO, lpfc_txcmplq_hw_show, NULL); -int lpfc_iocb_cnt = 2; -module_param(lpfc_iocb_cnt, int, S_IRUGO); -MODULE_PARM_DESC(lpfc_iocb_cnt, +LPFC_ATTR_R(iocb_cnt, 2, 1, 5, "Number of IOCBs alloc for ELS, CT, and ABTS: 1k to 5k IOCBs"); -lpfc_param_show(iocb_cnt); -lpfc_param_init(iocb_cnt, 2, 1, 5); -static DEVICE_ATTR(lpfc_iocb_cnt, S_IRUGO, - lpfc_iocb_cnt_show, NULL); /* # lpfc_nodev_tmo: If set, it will hold all I/O errors on devices that disappear @@ -2887,9 +2877,9 @@ lpfc_nodev_tmo_init(struct lpfc_vport *vport, int val) vport->cfg_nodev_tmo = vport->cfg_devloss_tmo; if (val != LPFC_DEF_DEVLOSS_TMO) lpfc_printf_vlog(vport, KERN_ERR, LOG_INIT, - "0407 Ignoring nodev_tmo module " - "parameter because devloss_tmo is " - "set.\n"); + "0407 Ignoring lpfc_nodev_tmo module " + "parameter because lpfc_devloss_tmo " + "is set.\n"); return 0; } @@ -2948,8 +2938,8 @@ lpfc_nodev_tmo_set(struct lpfc_vport *vport, int val) if (vport->dev_loss_tmo_changed || (lpfc_devloss_tmo != LPFC_DEF_DEVLOSS_TMO)) { lpfc_printf_vlog(vport, KERN_ERR, LOG_INIT, - "0401 Ignoring change to nodev_tmo " - "because devloss_tmo is set.\n"); + "0401 Ignoring change to lpfc_nodev_tmo " + "because lpfc_devloss_tmo is set.\n"); return 0; } if (val >= LPFC_MIN_DEVLOSS_TMO && val <= LPFC_MAX_DEVLOSS_TMO) { @@ -2964,7 +2954,7 @@ lpfc_nodev_tmo_set(struct lpfc_vport *vport, int val) return 0; } lpfc_printf_vlog(vport, KERN_ERR, LOG_INIT, - "0403 lpfc_nodev_tmo attribute cannot be set to" + "0403 lpfc_nodev_tmo attribute cannot be set to " "%d, allowed range is [%d, %d]\n", val, LPFC_MIN_DEVLOSS_TMO, LPFC_MAX_DEVLOSS_TMO); return -EINVAL; @@ -3015,8 +3005,8 @@ lpfc_devloss_tmo_set(struct lpfc_vport *vport, int val) } lpfc_printf_vlog(vport, KERN_ERR, LOG_INIT, - "0404 lpfc_devloss_tmo attribute cannot be set to" - " %d, allowed range is [%d, %d]\n", + "0404 lpfc_devloss_tmo attribute cannot be set to " + "%d, allowed range is [%d, %d]\n", val, LPFC_MIN_DEVLOSS_TMO, LPFC_MAX_DEVLOSS_TMO); return -EINVAL; } @@ -3204,6 +3194,8 @@ LPFC_VPORT_ATTR_R(scan_down, 1, 0, 1, # Set loop mode if you want to run as an NL_Port. Value range is [0,0x6]. # Default value is 0. */ +LPFC_ATTR(topology, 0, 0, 6, + "Select Fibre Channel topology"); /** * lpfc_topology_set - Set the adapters topology field @@ -3281,11 +3273,8 @@ lpfc_topology_store(struct device *dev, struct device_attribute *attr, phba->brd_no, val); return -EINVAL; } -static int lpfc_topology = 0; -module_param(lpfc_topology, int, S_IRUGO); -MODULE_PARM_DESC(lpfc_topology, "Select Fibre Channel topology"); + lpfc_param_show(topology) -lpfc_param_init(topology, 0, 0, 6) static DEVICE_ATTR(lpfc_topology, S_IRUGO | S_IWUSR, lpfc_topology_show, lpfc_topology_store); @@ -3679,7 +3668,12 @@ lpfc_link_speed_store(struct device *dev, struct device_attribute *attr, int nolip = 0; const char *val_buf = buf; int err; - uint32_t prev_val; + uint32_t prev_val, if_type; + + if_type = bf_get(lpfc_sli_intf_if_type, &phba->sli4_hba.sli_intf); + if (if_type == LPFC_SLI_INTF_IF_TYPE_2 && + phba->hba_flag & HBA_FORCED_LINK_SPEED) + return -EPERM; if (!strncmp(buf, "nolip ", strlen("nolip "))) { nolip = 1; @@ -3789,6 +3783,9 @@ static DEVICE_ATTR(lpfc_link_speed, S_IRUGO | S_IWUSR, # 1 = aer supported and enabled (default) # Value range is [0,1]. Default value is 1. */ +LPFC_ATTR(aer_support, 1, 0, 1, + "Enable PCIe device AER support"); +lpfc_param_show(aer_support) /** * lpfc_aer_support_store - Set the adapter for aer support @@ -3871,46 +3868,6 @@ lpfc_aer_support_store(struct device *dev, struct device_attribute *attr, return rc; } -static int lpfc_aer_support = 1; -module_param(lpfc_aer_support, int, S_IRUGO); -MODULE_PARM_DESC(lpfc_aer_support, "Enable PCIe device AER support"); -lpfc_param_show(aer_support) - -/** - * lpfc_aer_support_init - Set the initial adapters aer support flag - * @phba: lpfc_hba pointer. - * @val: enable aer or disable aer flag. - * - * Description: - * If val is in a valid range [0,1], then set the adapter's initial - * cfg_aer_support field. It will be up to the driver's probe_one - * routine to determine whether the device's AER support can be set - * or not. - * - * Notes: - * If the value is not in range log a kernel error message, and - * choose the default value of setting AER support and return. - * - * Returns: - * zero if val saved. - * -EINVAL val out of range - **/ -static int -lpfc_aer_support_init(struct lpfc_hba *phba, int val) -{ - if (val == 0 || val == 1) { - phba->cfg_aer_support = val; - return 0; - } - lpfc_printf_log(phba, KERN_ERR, LOG_INIT, - "2712 lpfc_aer_support attribute value %d out " - "of range, allowed values are 0|1, setting it " - "to default value of 1\n", val); - /* By default, try to enable AER on a device */ - phba->cfg_aer_support = 1; - return -EINVAL; -} - static DEVICE_ATTR(lpfc_aer_support, S_IRUGO | S_IWUSR, lpfc_aer_support_show, lpfc_aer_support_store); @@ -4055,39 +4012,10 @@ lpfc_sriov_nr_virtfn_store(struct device *dev, struct device_attribute *attr, return rc; } -static int lpfc_sriov_nr_virtfn = LPFC_DEF_VFN_PER_PFN; -module_param(lpfc_sriov_nr_virtfn, int, S_IRUGO|S_IWUSR); -MODULE_PARM_DESC(lpfc_sriov_nr_virtfn, "Enable PCIe device SR-IOV virtual fn"); -lpfc_param_show(sriov_nr_virtfn) - -/** - * lpfc_sriov_nr_virtfn_init - Set the initial sr-iov virtual function enable - * @phba: lpfc_hba pointer. - * @val: link speed value. - * - * Description: - * If val is in a valid range [0,255], then set the adapter's initial - * cfg_sriov_nr_virtfn field. If it's greater than the maximum, the maximum - * number shall be used instead. It will be up to the driver's probe_one - * routine to determine whether the device's SR-IOV is supported or not. - * - * Returns: - * zero if val saved. - * -EINVAL val out of range - **/ -static int -lpfc_sriov_nr_virtfn_init(struct lpfc_hba *phba, int val) -{ - if (val >= 0 && val <= LPFC_MAX_VFN_PER_PFN) { - phba->cfg_sriov_nr_virtfn = val; - return 0; - } +LPFC_ATTR(sriov_nr_virtfn, LPFC_DEF_VFN_PER_PFN, 0, LPFC_MAX_VFN_PER_PFN, + "Enable PCIe device SR-IOV virtual fn"); - lpfc_printf_log(phba, KERN_ERR, LOG_INIT, - "3017 Enabling %d virtual functions is not " - "allowed.\n", val); - return -EINVAL; -} +lpfc_param_show(sriov_nr_virtfn) static DEVICE_ATTR(lpfc_sriov_nr_virtfn, S_IRUGO | S_IWUSR, lpfc_sriov_nr_virtfn_show, lpfc_sriov_nr_virtfn_store); @@ -4251,7 +4179,8 @@ lpfc_fcp_imax_init(struct lpfc_hba *phba, int val) } lpfc_printf_log(phba, KERN_ERR, LOG_INIT, - "3016 fcp_imax: %d out of range, using default\n", val); + "3016 lpfc_fcp_imax: %d out of range, using default\n", + val); phba->cfg_fcp_imax = LPFC_DEF_IMAX; return 0; @@ -4401,8 +4330,8 @@ lpfc_fcp_cpu_map_init(struct lpfc_hba *phba, int val) } lpfc_printf_log(phba, KERN_ERR, LOG_INIT, - "3326 fcp_cpu_map: %d out of range, using default\n", - val); + "3326 lpfc_fcp_cpu_map: %d out of range, using " + "default\n", val); phba->cfg_fcp_cpu_map = LPFC_DRIVER_CPU_MAP; return 0; @@ -4441,12 +4370,10 @@ LPFC_VPORT_ATTR_RW(first_burst_size, 0, 0, 65536, # to limit the I/O completion time to the parameter value. # The value is set in milliseconds. */ -static int lpfc_max_scsicmpl_time; -module_param(lpfc_max_scsicmpl_time, int, S_IRUGO); -MODULE_PARM_DESC(lpfc_max_scsicmpl_time, +LPFC_VPORT_ATTR(max_scsicmpl_time, 0, 0, 60000, "Use command completion time to control queue depth"); + lpfc_vport_param_show(max_scsicmpl_time); -lpfc_vport_param_init(max_scsicmpl_time, 0, 0, 60000); static int lpfc_max_scsicmpl_time_set(struct lpfc_vport *vport, int val) { @@ -4691,12 +4618,15 @@ unsigned int lpfc_fcp_look_ahead = LPFC_LOOK_AHEAD_OFF; # HBA supports DIX Type 1: Host to HBA Type 1 protection # */ -unsigned int lpfc_prot_mask = SHOST_DIF_TYPE1_PROTECTION | - SHOST_DIX_TYPE0_PROTECTION | - SHOST_DIX_TYPE1_PROTECTION; - -module_param(lpfc_prot_mask, uint, S_IRUGO); -MODULE_PARM_DESC(lpfc_prot_mask, "host protection mask"); +LPFC_ATTR(prot_mask, + (SHOST_DIF_TYPE1_PROTECTION | + SHOST_DIX_TYPE0_PROTECTION | + SHOST_DIX_TYPE1_PROTECTION), + 0, + (SHOST_DIF_TYPE1_PROTECTION | + SHOST_DIX_TYPE0_PROTECTION | + SHOST_DIX_TYPE1_PROTECTION), + "T10-DIF host protection capabilities mask"); /* # lpfc_prot_guard: i @@ -4706,9 +4636,9 @@ MODULE_PARM_DESC(lpfc_prot_mask, "host protection mask"); # - Default will result in registering capabilities for all guard types # */ -unsigned char lpfc_prot_guard = SHOST_DIX_GUARD_IP; -module_param(lpfc_prot_guard, byte, S_IRUGO); -MODULE_PARM_DESC(lpfc_prot_guard, "host protection guard type"); +LPFC_ATTR(prot_guard, + SHOST_DIX_GUARD_IP, SHOST_DIX_GUARD_CRC, SHOST_DIX_GUARD_IP, + "T10-DIF host protection guard type"); /* * Delay initial NPort discovery when Clean Address bit is cleared in @@ -5828,6 +5758,8 @@ lpfc_get_cfgparam(struct lpfc_hba *phba) phba->cfg_oas_flags = 0; phba->cfg_oas_priority = 0; lpfc_enable_bg_init(phba, lpfc_enable_bg); + lpfc_prot_mask_init(phba, lpfc_prot_mask); + lpfc_prot_guard_init(phba, lpfc_prot_guard); if (phba->sli_rev == LPFC_SLI_REV4) phba->cfg_poll = 0; else diff --git a/drivers/scsi/lpfc/lpfc_bsg.c b/drivers/scsi/lpfc/lpfc_bsg.c index 05dcc2a..7dca4d6 100644 --- a/drivers/scsi/lpfc/lpfc_bsg.c +++ b/drivers/scsi/lpfc/lpfc_bsg.c @@ -24,6 +24,7 @@ #include <linux/slab.h> #include <linux/delay.h> #include <linux/list.h> +#include <linux/bsg-lib.h> #include <scsi/scsi.h> #include <scsi/scsi_host.h> @@ -97,7 +98,7 @@ struct lpfc_bsg_menlo { #define TYPE_MENLO 4 struct bsg_job_data { uint32_t type; - struct fc_bsg_job *set_job; /* job waiting for this iocb to finish */ + struct bsg_job *set_job; /* job waiting for this iocb to finish */ union { struct lpfc_bsg_event *evt; struct lpfc_bsg_iocb iocb; @@ -211,7 +212,7 @@ lpfc_alloc_bsg_buffers(struct lpfc_hba *phba, unsigned int size, static unsigned int lpfc_bsg_copy_data(struct lpfc_dmabuf *dma_buffers, - struct fc_bsg_buffer *bsg_buffers, + struct bsg_buffer *bsg_buffers, unsigned int bytes_to_transfer, int to_buffers) { @@ -297,7 +298,8 @@ lpfc_bsg_send_mgmt_cmd_cmp(struct lpfc_hba *phba, struct lpfc_iocbq *rspiocbq) { struct bsg_job_data *dd_data; - struct fc_bsg_job *job; + struct bsg_job *job; + struct fc_bsg_reply *bsg_reply; IOCB_t *rsp; struct lpfc_dmabuf *bmp, *cmp, *rmp; struct lpfc_nodelist *ndlp; @@ -312,6 +314,7 @@ lpfc_bsg_send_mgmt_cmd_cmp(struct lpfc_hba *phba, spin_lock_irqsave(&phba->ct_ev_lock, flags); job = dd_data->set_job; if (job) { + bsg_reply = job->reply; /* Prevent timeout handling from trying to abort job */ job->dd_data = NULL; } @@ -350,7 +353,7 @@ lpfc_bsg_send_mgmt_cmd_cmp(struct lpfc_hba *phba, } } else { rsp_size = rsp->un.genreq64.bdl.bdeSize; - job->reply->reply_payload_rcv_len = + bsg_reply->reply_payload_rcv_len = lpfc_bsg_copy_data(rmp, &job->reply_payload, rsp_size, 0); } @@ -367,8 +370,9 @@ lpfc_bsg_send_mgmt_cmd_cmp(struct lpfc_hba *phba, /* Complete the job if the job is still active */ if (job) { - job->reply->result = rc; - job->job_done(job); + bsg_reply->result = rc; + bsg_job_done(job, bsg_reply->result, + bsg_reply->reply_payload_rcv_len); } return; } @@ -378,12 +382,13 @@ lpfc_bsg_send_mgmt_cmd_cmp(struct lpfc_hba *phba, * @job: fc_bsg_job to handle **/ static int -lpfc_bsg_send_mgmt_cmd(struct fc_bsg_job *job) +lpfc_bsg_send_mgmt_cmd(struct bsg_job *job) { - struct lpfc_vport *vport = (struct lpfc_vport *)job->shost->hostdata; + struct lpfc_vport *vport = shost_priv(fc_bsg_to_shost(job)); struct lpfc_hba *phba = vport->phba; - struct lpfc_rport_data *rdata = job->rport->dd_data; + struct lpfc_rport_data *rdata = fc_bsg_to_rport(job)->dd_data; struct lpfc_nodelist *ndlp = rdata->pnode; + struct fc_bsg_reply *bsg_reply = job->reply; struct ulp_bde64 *bpl = NULL; uint32_t timeout; struct lpfc_iocbq *cmdiocbq = NULL; @@ -398,7 +403,7 @@ lpfc_bsg_send_mgmt_cmd(struct fc_bsg_job *job) int iocb_stat; /* in case no data is transferred */ - job->reply->reply_payload_rcv_len = 0; + bsg_reply->reply_payload_rcv_len = 0; /* allocate our bsg tracking structure */ dd_data = kmalloc(sizeof(struct bsg_job_data), GFP_KERNEL); @@ -542,7 +547,7 @@ no_ndlp: kfree(dd_data); no_dd_data: /* make error code available to userspace */ - job->reply->result = rc; + bsg_reply->result = rc; job->dd_data = NULL; return rc; } @@ -570,7 +575,8 @@ lpfc_bsg_rport_els_cmp(struct lpfc_hba *phba, struct lpfc_iocbq *rspiocbq) { struct bsg_job_data *dd_data; - struct fc_bsg_job *job; + struct bsg_job *job; + struct fc_bsg_reply *bsg_reply; IOCB_t *rsp; struct lpfc_nodelist *ndlp; struct lpfc_dmabuf *pcmd = NULL, *prsp = NULL; @@ -588,6 +594,7 @@ lpfc_bsg_rport_els_cmp(struct lpfc_hba *phba, spin_lock_irqsave(&phba->ct_ev_lock, flags); job = dd_data->set_job; if (job) { + bsg_reply = job->reply; /* Prevent timeout handling from trying to abort job */ job->dd_data = NULL; } @@ -609,17 +616,17 @@ lpfc_bsg_rport_els_cmp(struct lpfc_hba *phba, if (job) { if (rsp->ulpStatus == IOSTAT_SUCCESS) { rsp_size = rsp->un.elsreq64.bdl.bdeSize; - job->reply->reply_payload_rcv_len = + bsg_reply->reply_payload_rcv_len = sg_copy_from_buffer(job->reply_payload.sg_list, job->reply_payload.sg_cnt, prsp->virt, rsp_size); } else if (rsp->ulpStatus == IOSTAT_LS_RJT) { - job->reply->reply_payload_rcv_len = + bsg_reply->reply_payload_rcv_len = sizeof(struct fc_bsg_ctels_reply); /* LS_RJT data returned in word 4 */ rjt_data = (uint8_t *)&rsp->un.ulpWord[4]; - els_reply = &job->reply->reply_data.ctels_reply; + els_reply = &bsg_reply->reply_data.ctels_reply; els_reply->status = FC_CTELS_STATUS_REJECT; els_reply->rjt_data.action = rjt_data[3]; els_reply->rjt_data.reason_code = rjt_data[2]; @@ -637,8 +644,9 @@ lpfc_bsg_rport_els_cmp(struct lpfc_hba *phba, /* Complete the job if the job is still active */ if (job) { - job->reply->result = rc; - job->job_done(job); + bsg_reply->result = rc; + bsg_job_done(job, bsg_reply->result, + bsg_reply->reply_payload_rcv_len); } return; } @@ -648,12 +656,14 @@ lpfc_bsg_rport_els_cmp(struct lpfc_hba *phba, * @job: fc_bsg_job to handle **/ static int -lpfc_bsg_rport_els(struct fc_bsg_job *job) +lpfc_bsg_rport_els(struct bsg_job *job) { - struct lpfc_vport *vport = (struct lpfc_vport *)job->shost->hostdata; + struct lpfc_vport *vport = shost_priv(fc_bsg_to_shost(job)); struct lpfc_hba *phba = vport->phba; - struct lpfc_rport_data *rdata = job->rport->dd_data; + struct lpfc_rport_data *rdata = fc_bsg_to_rport(job)->dd_data; struct lpfc_nodelist *ndlp = rdata->pnode; + struct fc_bsg_request *bsg_request = job->request; + struct fc_bsg_reply *bsg_reply = job->reply; uint32_t elscmd; uint32_t cmdsize; struct lpfc_iocbq *cmdiocbq; @@ -664,7 +674,7 @@ lpfc_bsg_rport_els(struct fc_bsg_job *job) int rc = 0; /* in case no data is transferred */ - job->reply->reply_payload_rcv_len = 0; + bsg_reply->reply_payload_rcv_len = 0; /* verify the els command is not greater than the * maximum ELS transfer size. @@ -684,7 +694,7 @@ lpfc_bsg_rport_els(struct fc_bsg_job *job) goto no_dd_data; } - elscmd = job->request->rqst_data.r_els.els_code; + elscmd = bsg_request->rqst_data.r_els.els_code; cmdsize = job->request_payload.payload_len; if (!lpfc_nlp_get(ndlp)) { @@ -771,7 +781,7 @@ free_dd_data: no_dd_data: /* make error code available to userspace */ - job->reply->result = rc; + bsg_reply->result = rc; job->dd_data = NULL; return rc; } @@ -917,7 +927,8 @@ lpfc_bsg_ct_unsol_event(struct lpfc_hba *phba, struct lpfc_sli_ring *pring, struct lpfc_dmabuf *bdeBuf2 = piocbq->context3; struct lpfc_hbq_entry *hbqe; struct lpfc_sli_ct_request *ct_req; - struct fc_bsg_job *job = NULL; + struct bsg_job *job = NULL; + struct fc_bsg_reply *bsg_reply; struct bsg_job_data *dd_data = NULL; unsigned long flags; int size = 0; @@ -1120,13 +1131,15 @@ lpfc_bsg_ct_unsol_event(struct lpfc_hba *phba, struct lpfc_sli_ring *pring, dd_data->set_job = NULL; lpfc_bsg_event_unref(evt); if (job) { - job->reply->reply_payload_rcv_len = size; + bsg_reply = job->reply; + bsg_reply->reply_payload_rcv_len = size; /* make error code available to userspace */ - job->reply->result = 0; + bsg_reply->result = 0; job->dd_data = NULL; /* complete the job back to userspace */ spin_unlock_irqrestore(&phba->ct_ev_lock, flags); - job->job_done(job); + bsg_job_done(job, bsg_reply->result, + bsg_reply->reply_payload_rcv_len); spin_lock_irqsave(&phba->ct_ev_lock, flags); } } @@ -1187,10 +1200,11 @@ lpfc_bsg_ct_unsol_abort(struct lpfc_hba *phba, struct hbq_dmabuf *dmabuf) * @job: SET_EVENT fc_bsg_job **/ static int -lpfc_bsg_hba_set_event(struct fc_bsg_job *job) +lpfc_bsg_hba_set_event(struct bsg_job *job) { - struct lpfc_vport *vport = (struct lpfc_vport *)job->shost->hostdata; + struct lpfc_vport *vport = shost_priv(fc_bsg_to_shost(job)); struct lpfc_hba *phba = vport->phba; + struct fc_bsg_request *bsg_request = job->request; struct set_ct_event *event_req; struct lpfc_bsg_event *evt; int rc = 0; @@ -1208,7 +1222,7 @@ lpfc_bsg_hba_set_event(struct fc_bsg_job *job) } event_req = (struct set_ct_event *) - job->request->rqst_data.h_vendor.vendor_cmd; + bsg_request->rqst_data.h_vendor.vendor_cmd; ev_mask = ((uint32_t)(unsigned long)event_req->type_mask & FC_REG_EVENT_MASK); spin_lock_irqsave(&phba->ct_ev_lock, flags); @@ -1271,10 +1285,12 @@ job_error: * @job: GET_EVENT fc_bsg_job **/ static int -lpfc_bsg_hba_get_event(struct fc_bsg_job *job) +lpfc_bsg_hba_get_event(struct bsg_job *job) { - struct lpfc_vport *vport = (struct lpfc_vport *)job->shost->hostdata; + struct lpfc_vport *vport = shost_priv(fc_bsg_to_shost(job)); struct lpfc_hba *phba = vport->phba; + struct fc_bsg_request *bsg_request = job->request; + struct fc_bsg_reply *bsg_reply = job->reply; struct get_ct_event *event_req; struct get_ct_event_reply *event_reply; struct lpfc_bsg_event *evt, *evt_next; @@ -1292,10 +1308,10 @@ lpfc_bsg_hba_get_event(struct fc_bsg_job *job) } event_req = (struct get_ct_event *) - job->request->rqst_data.h_vendor.vendor_cmd; + bsg_request->rqst_data.h_vendor.vendor_cmd; event_reply = (struct get_ct_event_reply *) - job->reply->reply_data.vendor_reply.vendor_rsp; + bsg_reply->reply_data.vendor_reply.vendor_rsp; spin_lock_irqsave(&phba->ct_ev_lock, flags); list_for_each_entry_safe(evt, evt_next, &phba->ct_ev_waiters, node) { if (evt->reg_id == event_req->ev_reg_id) { @@ -1315,7 +1331,7 @@ lpfc_bsg_hba_get_event(struct fc_bsg_job *job) * an error indicating that there isn't anymore */ if (evt_dat == NULL) { - job->reply->reply_payload_rcv_len = 0; + bsg_reply->reply_payload_rcv_len = 0; rc = -ENOENT; goto job_error; } @@ -1331,12 +1347,12 @@ lpfc_bsg_hba_get_event(struct fc_bsg_job *job) event_reply->type = evt_dat->type; event_reply->immed_data = evt_dat->immed_dat; if (evt_dat->len > 0) - job->reply->reply_payload_rcv_len = + bsg_reply->reply_payload_rcv_len = sg_copy_from_buffer(job->request_payload.sg_list, job->request_payload.sg_cnt, evt_dat->data, evt_dat->len); else - job->reply->reply_payload_rcv_len = 0; + bsg_reply->reply_payload_rcv_len = 0; if (evt_dat) { kfree(evt_dat->data); @@ -1347,13 +1363,14 @@ lpfc_bsg_hba_get_event(struct fc_bsg_job *job) lpfc_bsg_event_unref(evt); spin_unlock_irqrestore(&phba->ct_ev_lock, flags); job->dd_data = NULL; - job->reply->result = 0; - job->job_done(job); + bsg_reply->result = 0; + bsg_job_done(job, bsg_reply->result, + bsg_reply->reply_payload_rcv_len); return 0; job_error: job->dd_data = NULL; - job->reply->result = rc; + bsg_reply->result = rc; return rc; } @@ -1380,7 +1397,8 @@ lpfc_issue_ct_rsp_cmp(struct lpfc_hba *phba, struct lpfc_iocbq *rspiocbq) { struct bsg_job_data *dd_data; - struct fc_bsg_job *job; + struct bsg_job *job; + struct fc_bsg_reply *bsg_reply; IOCB_t *rsp; struct lpfc_dmabuf *bmp, *cmp; struct lpfc_nodelist *ndlp; @@ -1411,6 +1429,7 @@ lpfc_issue_ct_rsp_cmp(struct lpfc_hba *phba, /* Copy the completed job data or set the error status */ if (job) { + bsg_reply = job->reply; if (rsp->ulpStatus) { if (rsp->ulpStatus == IOSTAT_LOCAL_REJECT) { switch (rsp->un.ulpWord[4] & IOERR_PARAM_MASK) { @@ -1428,7 +1447,7 @@ lpfc_issue_ct_rsp_cmp(struct lpfc_hba *phba, rc = -EACCES; } } else { - job->reply->reply_payload_rcv_len = 0; + bsg_reply->reply_payload_rcv_len = 0; } } @@ -1442,8 +1461,9 @@ lpfc_issue_ct_rsp_cmp(struct lpfc_hba *phba, /* Complete the job if the job is still active */ if (job) { - job->reply->result = rc; - job->job_done(job); + bsg_reply->result = rc; + bsg_job_done(job, bsg_reply->result, + bsg_reply->reply_payload_rcv_len); } return; } @@ -1457,7 +1477,7 @@ lpfc_issue_ct_rsp_cmp(struct lpfc_hba *phba, * @num_entry: Number of enties in the bde. **/ static int -lpfc_issue_ct_rsp(struct lpfc_hba *phba, struct fc_bsg_job *job, uint32_t tag, +lpfc_issue_ct_rsp(struct lpfc_hba *phba, struct bsg_job *job, uint32_t tag, struct lpfc_dmabuf *cmp, struct lpfc_dmabuf *bmp, int num_entry) { @@ -1603,12 +1623,14 @@ no_dd_data: * @job: SEND_MGMT_RESP fc_bsg_job **/ static int -lpfc_bsg_send_mgmt_rsp(struct fc_bsg_job *job) +lpfc_bsg_send_mgmt_rsp(struct bsg_job *job) { - struct lpfc_vport *vport = (struct lpfc_vport *)job->shost->hostdata; + struct lpfc_vport *vport = shost_priv(fc_bsg_to_shost(job)); struct lpfc_hba *phba = vport->phba; + struct fc_bsg_request *bsg_request = job->request; + struct fc_bsg_reply *bsg_reply = job->reply; struct send_mgmt_resp *mgmt_resp = (struct send_mgmt_resp *) - job->request->rqst_data.h_vendor.vendor_cmd; + bsg_request->rqst_data.h_vendor.vendor_cmd; struct ulp_bde64 *bpl; struct lpfc_dmabuf *bmp = NULL, *cmp = NULL; int bpl_entries; @@ -1618,7 +1640,7 @@ lpfc_bsg_send_mgmt_rsp(struct fc_bsg_job *job) int rc = 0; /* in case no data is transferred */ - job->reply->reply_payload_rcv_len = 0; + bsg_reply->reply_payload_rcv_len = 0; if (!reqbfrcnt || (reqbfrcnt > (80 * BUF_SZ_4K))) { rc = -ERANGE; @@ -1664,7 +1686,7 @@ send_mgmt_rsp_free_bmp: kfree(bmp); send_mgmt_rsp_exit: /* make error code available to userspace */ - job->reply->result = rc; + bsg_reply->result = rc; job->dd_data = NULL; return rc; } @@ -1760,8 +1782,10 @@ lpfc_bsg_diag_mode_exit(struct lpfc_hba *phba) * All of this is done in-line. */ static int -lpfc_sli3_bsg_diag_loopback_mode(struct lpfc_hba *phba, struct fc_bsg_job *job) +lpfc_sli3_bsg_diag_loopback_mode(struct lpfc_hba *phba, struct bsg_job *job) { + struct fc_bsg_request *bsg_request = job->request; + struct fc_bsg_reply *bsg_reply = job->reply; struct diag_mode_set *loopback_mode; uint32_t link_flags; uint32_t timeout; @@ -1771,7 +1795,7 @@ lpfc_sli3_bsg_diag_loopback_mode(struct lpfc_hba *phba, struct fc_bsg_job *job) int rc = 0; /* no data to return just the return code */ - job->reply->reply_payload_rcv_len = 0; + bsg_reply->reply_payload_rcv_len = 0; if (job->request_len < sizeof(struct fc_bsg_request) + sizeof(struct diag_mode_set)) { @@ -1791,7 +1815,7 @@ lpfc_sli3_bsg_diag_loopback_mode(struct lpfc_hba *phba, struct fc_bsg_job *job) /* bring the link to diagnostic mode */ loopback_mode = (struct diag_mode_set *) - job->request->rqst_data.h_vendor.vendor_cmd; + bsg_request->rqst_data.h_vendor.vendor_cmd; link_flags = loopback_mode->type; timeout = loopback_mode->timeout * 100; @@ -1864,10 +1888,11 @@ loopback_mode_exit: job_error: /* make error code available to userspace */ - job->reply->result = rc; + bsg_reply->result = rc; /* complete the job back to userspace if no error */ if (rc == 0) - job->job_done(job); + bsg_job_done(job, bsg_reply->result, + bsg_reply->reply_payload_rcv_len); return rc; } @@ -2015,14 +2040,16 @@ lpfc_sli4_diag_fcport_reg_setup(struct lpfc_hba *phba) * loopback mode in order to perform a diagnostic loopback test. */ static int -lpfc_sli4_bsg_diag_loopback_mode(struct lpfc_hba *phba, struct fc_bsg_job *job) +lpfc_sli4_bsg_diag_loopback_mode(struct lpfc_hba *phba, struct bsg_job *job) { + struct fc_bsg_request *bsg_request = job->request; + struct fc_bsg_reply *bsg_reply = job->reply; struct diag_mode_set *loopback_mode; uint32_t link_flags, timeout; int i, rc = 0; /* no data to return just the return code */ - job->reply->reply_payload_rcv_len = 0; + bsg_reply->reply_payload_rcv_len = 0; if (job->request_len < sizeof(struct fc_bsg_request) + sizeof(struct diag_mode_set)) { @@ -2054,7 +2081,7 @@ lpfc_sli4_bsg_diag_loopback_mode(struct lpfc_hba *phba, struct fc_bsg_job *job) lpfc_printf_log(phba, KERN_INFO, LOG_LIBDFC, "3129 Bring link to diagnostic state.\n"); loopback_mode = (struct diag_mode_set *) - job->request->rqst_data.h_vendor.vendor_cmd; + bsg_request->rqst_data.h_vendor.vendor_cmd; link_flags = loopback_mode->type; timeout = loopback_mode->timeout * 100; @@ -2151,10 +2178,11 @@ loopback_mode_exit: job_error: /* make error code available to userspace */ - job->reply->result = rc; + bsg_reply->result = rc; /* complete the job back to userspace if no error */ if (rc == 0) - job->job_done(job); + bsg_job_done(job, bsg_reply->result, + bsg_reply->reply_payload_rcv_len); return rc; } @@ -2166,17 +2194,17 @@ job_error: * command from the user to proper driver action routines. */ static int -lpfc_bsg_diag_loopback_mode(struct fc_bsg_job *job) +lpfc_bsg_diag_loopback_mode(struct bsg_job *job) { struct Scsi_Host *shost; struct lpfc_vport *vport; struct lpfc_hba *phba; int rc; - shost = job->shost; + shost = fc_bsg_to_shost(job); if (!shost) return -ENODEV; - vport = (struct lpfc_vport *)job->shost->hostdata; + vport = shost_priv(shost); if (!vport) return -ENODEV; phba = vport->phba; @@ -2202,8 +2230,10 @@ lpfc_bsg_diag_loopback_mode(struct fc_bsg_job *job) * command from the user to proper driver action routines. */ static int -lpfc_sli4_bsg_diag_mode_end(struct fc_bsg_job *job) +lpfc_sli4_bsg_diag_mode_end(struct bsg_job *job) { + struct fc_bsg_request *bsg_request = job->request; + struct fc_bsg_reply *bsg_reply = job->reply; struct Scsi_Host *shost; struct lpfc_vport *vport; struct lpfc_hba *phba; @@ -2211,10 +2241,10 @@ lpfc_sli4_bsg_diag_mode_end(struct fc_bsg_job *job) uint32_t timeout; int rc, i; - shost = job->shost; + shost = fc_bsg_to_shost(job); if (!shost) return -ENODEV; - vport = (struct lpfc_vport *)job->shost->hostdata; + vport = shost_priv(shost); if (!vport) return -ENODEV; phba = vport->phba; @@ -2232,7 +2262,7 @@ lpfc_sli4_bsg_diag_mode_end(struct fc_bsg_job *job) phba->link_flag &= ~LS_LOOPBACK_MODE; spin_unlock_irq(&phba->hbalock); loopback_mode_end_cmd = (struct diag_mode_set *) - job->request->rqst_data.h_vendor.vendor_cmd; + bsg_request->rqst_data.h_vendor.vendor_cmd; timeout = loopback_mode_end_cmd->timeout * 100; rc = lpfc_sli4_bsg_set_link_diag_state(phba, 0); @@ -2263,10 +2293,11 @@ lpfc_sli4_bsg_diag_mode_end(struct fc_bsg_job *job) loopback_mode_end_exit: /* make return code available to userspace */ - job->reply->result = rc; + bsg_reply->result = rc; /* complete the job back to userspace if no error */ if (rc == 0) - job->job_done(job); + bsg_job_done(job, bsg_reply->result, + bsg_reply->reply_payload_rcv_len); return rc; } @@ -2278,8 +2309,10 @@ loopback_mode_end_exit: * applicaiton. */ static int -lpfc_sli4_bsg_link_diag_test(struct fc_bsg_job *job) +lpfc_sli4_bsg_link_diag_test(struct bsg_job *job) { + struct fc_bsg_request *bsg_request = job->request; + struct fc_bsg_reply *bsg_reply = job->reply; struct Scsi_Host *shost; struct lpfc_vport *vport; struct lpfc_hba *phba; @@ -2292,12 +2325,12 @@ lpfc_sli4_bsg_link_diag_test(struct fc_bsg_job *job) struct diag_status *diag_status_reply; int mbxstatus, rc = 0; - shost = job->shost; + shost = fc_bsg_to_shost(job); if (!shost) { rc = -ENODEV; goto job_error; } - vport = (struct lpfc_vport *)job->shost->hostdata; + vport = shost_priv(shost); if (!vport) { rc = -ENODEV; goto job_error; @@ -2335,7 +2368,7 @@ lpfc_sli4_bsg_link_diag_test(struct fc_bsg_job *job) goto job_error; link_diag_test_cmd = (struct sli4_link_diag *) - job->request->rqst_data.h_vendor.vendor_cmd; + bsg_request->rqst_data.h_vendor.vendor_cmd; rc = lpfc_sli4_bsg_set_link_diag_state(phba, 1); @@ -2385,7 +2418,7 @@ lpfc_sli4_bsg_link_diag_test(struct fc_bsg_job *job) } diag_status_reply = (struct diag_status *) - job->reply->reply_data.vendor_reply.vendor_rsp; + bsg_reply->reply_data.vendor_reply.vendor_rsp; if (job->reply_len < sizeof(struct fc_bsg_request) + sizeof(struct diag_status)) { @@ -2413,10 +2446,11 @@ link_diag_test_exit: job_error: /* make error code available to userspace */ - job->reply->result = rc; + bsg_reply->result = rc; /* complete the job back to userspace if no error */ if (rc == 0) - job->job_done(job); + bsg_job_done(job, bsg_reply->result, + bsg_reply->reply_payload_rcv_len); return rc; } @@ -2982,9 +3016,10 @@ err_post_rxbufs_exit: * of loopback mode. **/ static int -lpfc_bsg_diag_loopback_run(struct fc_bsg_job *job) +lpfc_bsg_diag_loopback_run(struct bsg_job *job) { - struct lpfc_vport *vport = (struct lpfc_vport *)job->shost->hostdata; + struct lpfc_vport *vport = shost_priv(fc_bsg_to_shost(job)); + struct fc_bsg_reply *bsg_reply = job->reply; struct lpfc_hba *phba = vport->phba; struct lpfc_bsg_event *evt; struct event_data *evdat; @@ -3012,7 +3047,7 @@ lpfc_bsg_diag_loopback_run(struct fc_bsg_job *job) uint32_t total_mem; /* in case no data is returned return just the return code */ - job->reply->reply_payload_rcv_len = 0; + bsg_reply->reply_payload_rcv_len = 0; if (job->request_len < sizeof(struct fc_bsg_request) + sizeof(struct diag_mode_test)) { @@ -3237,11 +3272,11 @@ lpfc_bsg_diag_loopback_run(struct fc_bsg_job *job) rc = IOCB_SUCCESS; /* skip over elx loopback header */ rx_databuf += ELX_LOOPBACK_HEADER_SZ; - job->reply->reply_payload_rcv_len = + bsg_reply->reply_payload_rcv_len = sg_copy_from_buffer(job->reply_payload.sg_list, job->reply_payload.sg_cnt, rx_databuf, size); - job->reply->reply_payload_rcv_len = size; + bsg_reply->reply_payload_rcv_len = size; } } @@ -3271,11 +3306,12 @@ err_loopback_test_exit: loopback_test_exit: kfree(dataout); /* make error code available to userspace */ - job->reply->result = rc; + bsg_reply->result = rc; job->dd_data = NULL; /* complete the job back to userspace if no error */ if (rc == IOCB_SUCCESS) - job->job_done(job); + bsg_job_done(job, bsg_reply->result, + bsg_reply->reply_payload_rcv_len); return rc; } @@ -3284,9 +3320,10 @@ loopback_test_exit: * @job: GET_DFC_REV fc_bsg_job **/ static int -lpfc_bsg_get_dfc_rev(struct fc_bsg_job *job) +lpfc_bsg_get_dfc_rev(struct bsg_job *job) { - struct lpfc_vport *vport = (struct lpfc_vport *)job->shost->hostdata; + struct lpfc_vport *vport = shost_priv(fc_bsg_to_shost(job)); + struct fc_bsg_reply *bsg_reply = job->reply; struct lpfc_hba *phba = vport->phba; struct get_mgmt_rev_reply *event_reply; int rc = 0; @@ -3301,7 +3338,7 @@ lpfc_bsg_get_dfc_rev(struct fc_bsg_job *job) } event_reply = (struct get_mgmt_rev_reply *) - job->reply->reply_data.vendor_reply.vendor_rsp; + bsg_reply->reply_data.vendor_reply.vendor_rsp; if (job->reply_len < sizeof(struct fc_bsg_request) + sizeof(struct get_mgmt_rev_reply)) { @@ -3315,9 +3352,10 @@ lpfc_bsg_get_dfc_rev(struct fc_bsg_job *job) event_reply->info.a_Major = MANAGEMENT_MAJOR_REV; event_reply->info.a_Minor = MANAGEMENT_MINOR_REV; job_error: - job->reply->result = rc; + bsg_reply->result = rc; if (rc == 0) - job->job_done(job); + bsg_job_done(job, bsg_reply->result, + bsg_reply->reply_payload_rcv_len); return rc; } @@ -3336,7 +3374,8 @@ static void lpfc_bsg_issue_mbox_cmpl(struct lpfc_hba *phba, LPFC_MBOXQ_t *pmboxq) { struct bsg_job_data *dd_data; - struct fc_bsg_job *job; + struct fc_bsg_reply *bsg_reply; + struct bsg_job *job; uint32_t size; unsigned long flags; uint8_t *pmb, *pmb_buf; @@ -3364,8 +3403,9 @@ lpfc_bsg_issue_mbox_cmpl(struct lpfc_hba *phba, LPFC_MBOXQ_t *pmboxq) /* Copy the mailbox data to the job if it is still active */ if (job) { + bsg_reply = job->reply; size = job->reply_payload.payload_len; - job->reply->reply_payload_rcv_len = + bsg_reply->reply_payload_rcv_len = sg_copy_from_buffer(job->reply_payload.sg_list, job->reply_payload.sg_cnt, pmb_buf, size); @@ -3379,8 +3419,9 @@ lpfc_bsg_issue_mbox_cmpl(struct lpfc_hba *phba, LPFC_MBOXQ_t *pmboxq) /* Complete the job if the job is still active */ if (job) { - job->reply->result = 0; - job->job_done(job); + bsg_reply->result = 0; + bsg_job_done(job, bsg_reply->result, + bsg_reply->reply_payload_rcv_len); } return; } @@ -3510,11 +3551,12 @@ lpfc_bsg_mbox_ext_session_reset(struct lpfc_hba *phba) * This is routine handles BSG job for mailbox commands completions with * multiple external buffers. **/ -static struct fc_bsg_job * +static struct bsg_job * lpfc_bsg_issue_mbox_ext_handle_job(struct lpfc_hba *phba, LPFC_MBOXQ_t *pmboxq) { struct bsg_job_data *dd_data; - struct fc_bsg_job *job; + struct bsg_job *job; + struct fc_bsg_reply *bsg_reply; uint8_t *pmb, *pmb_buf; unsigned long flags; uint32_t size; @@ -3529,6 +3571,7 @@ lpfc_bsg_issue_mbox_ext_handle_job(struct lpfc_hba *phba, LPFC_MBOXQ_t *pmboxq) spin_lock_irqsave(&phba->ct_ev_lock, flags); job = dd_data->set_job; if (job) { + bsg_reply = job->reply; /* Prevent timeout handling from trying to abort job */ job->dd_data = NULL; } @@ -3559,13 +3602,13 @@ lpfc_bsg_issue_mbox_ext_handle_job(struct lpfc_hba *phba, LPFC_MBOXQ_t *pmboxq) if (job) { size = job->reply_payload.payload_len; - job->reply->reply_payload_rcv_len = + bsg_reply->reply_payload_rcv_len = sg_copy_from_buffer(job->reply_payload.sg_list, job->reply_payload.sg_cnt, pmb_buf, size); /* result for successful */ - job->reply->result = 0; + bsg_reply->result = 0; lpfc_printf_log(phba, KERN_INFO, LOG_LIBDFC, "2937 SLI_CONFIG ext-buffer maibox command " @@ -3603,7 +3646,8 @@ lpfc_bsg_issue_mbox_ext_handle_job(struct lpfc_hba *phba, LPFC_MBOXQ_t *pmboxq) static void lpfc_bsg_issue_read_mbox_ext_cmpl(struct lpfc_hba *phba, LPFC_MBOXQ_t *pmboxq) { - struct fc_bsg_job *job; + struct bsg_job *job; + struct fc_bsg_reply *bsg_reply; job = lpfc_bsg_issue_mbox_ext_handle_job(phba, pmboxq); @@ -3623,9 +3667,11 @@ lpfc_bsg_issue_read_mbox_ext_cmpl(struct lpfc_hba *phba, LPFC_MBOXQ_t *pmboxq) mempool_free(pmboxq, phba->mbox_mem_pool); /* if the job is still active, call job done */ - if (job) - job->job_done(job); - + if (job) { + bsg_reply = job->reply; + bsg_job_done(job, bsg_reply->result, + bsg_reply->reply_payload_rcv_len); + } return; } @@ -3640,7 +3686,8 @@ lpfc_bsg_issue_read_mbox_ext_cmpl(struct lpfc_hba *phba, LPFC_MBOXQ_t *pmboxq) static void lpfc_bsg_issue_write_mbox_ext_cmpl(struct lpfc_hba *phba, LPFC_MBOXQ_t *pmboxq) { - struct fc_bsg_job *job; + struct bsg_job *job; + struct fc_bsg_reply *bsg_reply; job = lpfc_bsg_issue_mbox_ext_handle_job(phba, pmboxq); @@ -3658,8 +3705,11 @@ lpfc_bsg_issue_write_mbox_ext_cmpl(struct lpfc_hba *phba, LPFC_MBOXQ_t *pmboxq) lpfc_bsg_mbox_ext_session_reset(phba); /* if the job is still active, call job done */ - if (job) - job->job_done(job); + if (job) { + bsg_reply = job->reply; + bsg_job_done(job, bsg_reply->result, + bsg_reply->reply_payload_rcv_len); + } return; } @@ -3768,10 +3818,11 @@ lpfc_bsg_sli_cfg_dma_desc_setup(struct lpfc_hba *phba, enum nemb_type nemb_tp, * non-embedded external bufffers. **/ static int -lpfc_bsg_sli_cfg_read_cmd_ext(struct lpfc_hba *phba, struct fc_bsg_job *job, +lpfc_bsg_sli_cfg_read_cmd_ext(struct lpfc_hba *phba, struct bsg_job *job, enum nemb_type nemb_tp, struct lpfc_dmabuf *dmabuf) { + struct fc_bsg_request *bsg_request = job->request; struct lpfc_sli_config_mbox *sli_cfg_mbx; struct dfc_mbox_req *mbox_req; struct lpfc_dmabuf *curr_dmabuf, *next_dmabuf; @@ -3784,7 +3835,7 @@ lpfc_bsg_sli_cfg_read_cmd_ext(struct lpfc_hba *phba, struct fc_bsg_job *job, int rc, i; mbox_req = - (struct dfc_mbox_req *)job->request->rqst_data.h_vendor.vendor_cmd; + (struct dfc_mbox_req *)bsg_request->rqst_data.h_vendor.vendor_cmd; /* pointer to the start of mailbox command */ sli_cfg_mbx = (struct lpfc_sli_config_mbox *)dmabuf->virt; @@ -3955,10 +4006,12 @@ job_error: * non-embedded external bufffers. **/ static int -lpfc_bsg_sli_cfg_write_cmd_ext(struct lpfc_hba *phba, struct fc_bsg_job *job, +lpfc_bsg_sli_cfg_write_cmd_ext(struct lpfc_hba *phba, struct bsg_job *job, enum nemb_type nemb_tp, struct lpfc_dmabuf *dmabuf) { + struct fc_bsg_request *bsg_request = job->request; + struct fc_bsg_reply *bsg_reply = job->reply; struct dfc_mbox_req *mbox_req; struct lpfc_sli_config_mbox *sli_cfg_mbx; uint32_t ext_buf_cnt; @@ -3969,7 +4022,7 @@ lpfc_bsg_sli_cfg_write_cmd_ext(struct lpfc_hba *phba, struct fc_bsg_job *job, int rc = SLI_CONFIG_NOT_HANDLED, i; mbox_req = - (struct dfc_mbox_req *)job->request->rqst_data.h_vendor.vendor_cmd; + (struct dfc_mbox_req *)bsg_request->rqst_data.h_vendor.vendor_cmd; /* pointer to the start of mailbox command */ sli_cfg_mbx = (struct lpfc_sli_config_mbox *)dmabuf->virt; @@ -4096,8 +4149,9 @@ lpfc_bsg_sli_cfg_write_cmd_ext(struct lpfc_hba *phba, struct fc_bsg_job *job, /* wait for additoinal external buffers */ - job->reply->result = 0; - job->job_done(job); + bsg_reply->result = 0; + bsg_job_done(job, bsg_reply->result, + bsg_reply->reply_payload_rcv_len); return SLI_CONFIG_HANDLED; job_error: @@ -4119,7 +4173,7 @@ job_error: * with embedded sussystem 0x1 and opcodes with external HBDs. **/ static int -lpfc_bsg_handle_sli_cfg_mbox(struct lpfc_hba *phba, struct fc_bsg_job *job, +lpfc_bsg_handle_sli_cfg_mbox(struct lpfc_hba *phba, struct bsg_job *job, struct lpfc_dmabuf *dmabuf) { struct lpfc_sli_config_mbox *sli_cfg_mbx; @@ -4268,8 +4322,9 @@ lpfc_bsg_mbox_ext_abort(struct lpfc_hba *phba) * user space through BSG. **/ static int -lpfc_bsg_read_ebuf_get(struct lpfc_hba *phba, struct fc_bsg_job *job) +lpfc_bsg_read_ebuf_get(struct lpfc_hba *phba, struct bsg_job *job) { + struct fc_bsg_reply *bsg_reply = job->reply; struct lpfc_sli_config_mbox *sli_cfg_mbx; struct lpfc_dmabuf *dmabuf; uint8_t *pbuf; @@ -4307,7 +4362,7 @@ lpfc_bsg_read_ebuf_get(struct lpfc_hba *phba, struct fc_bsg_job *job) dmabuf, index); pbuf = (uint8_t *)dmabuf->virt; - job->reply->reply_payload_rcv_len = + bsg_reply->reply_payload_rcv_len = sg_copy_from_buffer(job->reply_payload.sg_list, job->reply_payload.sg_cnt, pbuf, size); @@ -4321,8 +4376,9 @@ lpfc_bsg_read_ebuf_get(struct lpfc_hba *phba, struct fc_bsg_job *job) lpfc_bsg_mbox_ext_session_reset(phba); } - job->reply->result = 0; - job->job_done(job); + bsg_reply->result = 0; + bsg_job_done(job, bsg_reply->result, + bsg_reply->reply_payload_rcv_len); return SLI_CONFIG_HANDLED; } @@ -4336,9 +4392,10 @@ lpfc_bsg_read_ebuf_get(struct lpfc_hba *phba, struct fc_bsg_job *job) * from user space through BSG. **/ static int -lpfc_bsg_write_ebuf_set(struct lpfc_hba *phba, struct fc_bsg_job *job, +lpfc_bsg_write_ebuf_set(struct lpfc_hba *phba, struct bsg_job *job, struct lpfc_dmabuf *dmabuf) { + struct fc_bsg_reply *bsg_reply = job->reply; struct bsg_job_data *dd_data = NULL; LPFC_MBOXQ_t *pmboxq = NULL; MAILBOX_t *pmb; @@ -4436,8 +4493,9 @@ lpfc_bsg_write_ebuf_set(struct lpfc_hba *phba, struct fc_bsg_job *job, } /* wait for additoinal external buffers */ - job->reply->result = 0; - job->job_done(job); + bsg_reply->result = 0; + bsg_job_done(job, bsg_reply->result, + bsg_reply->reply_payload_rcv_len); return SLI_CONFIG_HANDLED; job_error: @@ -4457,7 +4515,7 @@ job_error: * command with multiple non-embedded external buffers. **/ static int -lpfc_bsg_handle_sli_cfg_ebuf(struct lpfc_hba *phba, struct fc_bsg_job *job, +lpfc_bsg_handle_sli_cfg_ebuf(struct lpfc_hba *phba, struct bsg_job *job, struct lpfc_dmabuf *dmabuf) { int rc; @@ -4502,14 +4560,15 @@ lpfc_bsg_handle_sli_cfg_ebuf(struct lpfc_hba *phba, struct fc_bsg_job *job, * (0x9B) mailbox commands and external buffers. **/ static int -lpfc_bsg_handle_sli_cfg_ext(struct lpfc_hba *phba, struct fc_bsg_job *job, +lpfc_bsg_handle_sli_cfg_ext(struct lpfc_hba *phba, struct bsg_job *job, struct lpfc_dmabuf *dmabuf) { + struct fc_bsg_request *bsg_request = job->request; struct dfc_mbox_req *mbox_req; int rc = SLI_CONFIG_NOT_HANDLED; mbox_req = - (struct dfc_mbox_req *)job->request->rqst_data.h_vendor.vendor_cmd; + (struct dfc_mbox_req *)bsg_request->rqst_data.h_vendor.vendor_cmd; /* mbox command with/without single external buffer */ if (mbox_req->extMboxTag == 0 && mbox_req->extSeqNum == 0) @@ -4579,9 +4638,11 @@ sli_cfg_ext_error: * let our completion handler finish the command. **/ static int -lpfc_bsg_issue_mbox(struct lpfc_hba *phba, struct fc_bsg_job *job, +lpfc_bsg_issue_mbox(struct lpfc_hba *phba, struct bsg_job *job, struct lpfc_vport *vport) { + struct fc_bsg_request *bsg_request = job->request; + struct fc_bsg_reply *bsg_reply = job->reply; LPFC_MBOXQ_t *pmboxq = NULL; /* internal mailbox queue */ MAILBOX_t *pmb; /* shortcut to the pmboxq mailbox */ /* a 4k buffer to hold the mb and extended data from/to the bsg */ @@ -4600,7 +4661,7 @@ lpfc_bsg_issue_mbox(struct lpfc_hba *phba, struct fc_bsg_job *job, uint32_t size; /* in case no data is transferred */ - job->reply->reply_payload_rcv_len = 0; + bsg_reply->reply_payload_rcv_len = 0; /* sanity check to protect driver */ if (job->reply_payload.payload_len > BSG_MBOX_SIZE || @@ -4619,7 +4680,7 @@ lpfc_bsg_issue_mbox(struct lpfc_hba *phba, struct fc_bsg_job *job, } mbox_req = - (struct dfc_mbox_req *)job->request->rqst_data.h_vendor.vendor_cmd; + (struct dfc_mbox_req *)bsg_request->rqst_data.h_vendor.vendor_cmd; /* check if requested extended data lengths are valid */ if ((mbox_req->inExtWLen > BSG_MBOX_SIZE/sizeof(uint32_t)) || @@ -4841,7 +4902,7 @@ lpfc_bsg_issue_mbox(struct lpfc_hba *phba, struct fc_bsg_job *job, /* job finished, copy the data */ memcpy(pmbx, pmb, sizeof(*pmb)); - job->reply->reply_payload_rcv_len = + bsg_reply->reply_payload_rcv_len = sg_copy_from_buffer(job->reply_payload.sg_list, job->reply_payload.sg_cnt, pmbx, size); @@ -4870,15 +4931,17 @@ job_cont: * @job: MBOX fc_bsg_job for LPFC_BSG_VENDOR_MBOX. **/ static int -lpfc_bsg_mbox_cmd(struct fc_bsg_job *job) +lpfc_bsg_mbox_cmd(struct bsg_job *job) { - struct lpfc_vport *vport = (struct lpfc_vport *)job->shost->hostdata; + struct lpfc_vport *vport = shost_priv(fc_bsg_to_shost(job)); + struct fc_bsg_request *bsg_request = job->request; + struct fc_bsg_reply *bsg_reply = job->reply; struct lpfc_hba *phba = vport->phba; struct dfc_mbox_req *mbox_req; int rc = 0; /* mix-and-match backward compatibility */ - job->reply->reply_payload_rcv_len = 0; + bsg_reply->reply_payload_rcv_len = 0; if (job->request_len < sizeof(struct fc_bsg_request) + sizeof(struct dfc_mbox_req)) { lpfc_printf_log(phba, KERN_INFO, LOG_LIBDFC, @@ -4889,7 +4952,7 @@ lpfc_bsg_mbox_cmd(struct fc_bsg_job *job) sizeof(struct fc_bsg_request)), (int)sizeof(struct dfc_mbox_req)); mbox_req = (struct dfc_mbox_req *) - job->request->rqst_data.h_vendor.vendor_cmd; + bsg_request->rqst_data.h_vendor.vendor_cmd; mbox_req->extMboxTag = 0; mbox_req->extSeqNum = 0; } @@ -4898,15 +4961,16 @@ lpfc_bsg_mbox_cmd(struct fc_bsg_job *job) if (rc == 0) { /* job done */ - job->reply->result = 0; + bsg_reply->result = 0; job->dd_data = NULL; - job->job_done(job); + bsg_job_done(job, bsg_reply->result, + bsg_reply->reply_payload_rcv_len); } else if (rc == 1) /* job submitted, will complete later*/ rc = 0; /* return zero, no error */ else { /* some error occurred */ - job->reply->result = rc; + bsg_reply->result = rc; job->dd_data = NULL; } @@ -4936,7 +5000,8 @@ lpfc_bsg_menlo_cmd_cmp(struct lpfc_hba *phba, struct lpfc_iocbq *rspiocbq) { struct bsg_job_data *dd_data; - struct fc_bsg_job *job; + struct bsg_job *job; + struct fc_bsg_reply *bsg_reply; IOCB_t *rsp; struct lpfc_dmabuf *bmp, *cmp, *rmp; struct lpfc_bsg_menlo *menlo; @@ -4956,6 +5021,7 @@ lpfc_bsg_menlo_cmd_cmp(struct lpfc_hba *phba, spin_lock_irqsave(&phba->ct_ev_lock, flags); job = dd_data->set_job; if (job) { + bsg_reply = job->reply; /* Prevent timeout handling from trying to abort job */ job->dd_data = NULL; } @@ -4970,7 +5036,7 @@ lpfc_bsg_menlo_cmd_cmp(struct lpfc_hba *phba, */ menlo_resp = (struct menlo_response *) - job->reply->reply_data.vendor_reply.vendor_rsp; + bsg_reply->reply_data.vendor_reply.vendor_rsp; menlo_resp->xri = rsp->ulpContext; if (rsp->ulpStatus) { if (rsp->ulpStatus == IOSTAT_LOCAL_REJECT) { @@ -4990,7 +5056,7 @@ lpfc_bsg_menlo_cmd_cmp(struct lpfc_hba *phba, } } else { rsp_size = rsp->un.genreq64.bdl.bdeSize; - job->reply->reply_payload_rcv_len = + bsg_reply->reply_payload_rcv_len = lpfc_bsg_copy_data(rmp, &job->reply_payload, rsp_size, 0); } @@ -5007,8 +5073,9 @@ lpfc_bsg_menlo_cmd_cmp(struct lpfc_hba *phba, /* Complete the job if active */ if (job) { - job->reply->result = rc; - job->job_done(job); + bsg_reply->result = rc; + bsg_job_done(job, bsg_reply->result, + bsg_reply->reply_payload_rcv_len); } return; @@ -5024,9 +5091,11 @@ lpfc_bsg_menlo_cmd_cmp(struct lpfc_hba *phba, * supplied in the menlo request header xri field. **/ static int -lpfc_menlo_cmd(struct fc_bsg_job *job) +lpfc_menlo_cmd(struct bsg_job *job) { - struct lpfc_vport *vport = (struct lpfc_vport *)job->shost->hostdata; + struct lpfc_vport *vport = shost_priv(fc_bsg_to_shost(job)); + struct fc_bsg_request *bsg_request = job->request; + struct fc_bsg_reply *bsg_reply = job->reply; struct lpfc_hba *phba = vport->phba; struct lpfc_iocbq *cmdiocbq; IOCB_t *cmd; @@ -5039,7 +5108,7 @@ lpfc_menlo_cmd(struct fc_bsg_job *job) struct ulp_bde64 *bpl = NULL; /* in case no data is returned return just the return code */ - job->reply->reply_payload_rcv_len = 0; + bsg_reply->reply_payload_rcv_len = 0; if (job->request_len < sizeof(struct fc_bsg_request) + @@ -5069,7 +5138,7 @@ lpfc_menlo_cmd(struct fc_bsg_job *job) } menlo_cmd = (struct menlo_command *) - job->request->rqst_data.h_vendor.vendor_cmd; + bsg_request->rqst_data.h_vendor.vendor_cmd; /* allocate our bsg tracking structure */ dd_data = kmalloc(sizeof(struct bsg_job_data), GFP_KERNEL); @@ -5180,19 +5249,65 @@ free_dd: kfree(dd_data); no_dd_data: /* make error code available to userspace */ - job->reply->result = rc; + bsg_reply->result = rc; job->dd_data = NULL; return rc; } +static int +lpfc_forced_link_speed(struct bsg_job *job) +{ + struct Scsi_Host *shost = fc_bsg_to_shost(job); + struct lpfc_vport *vport = shost_priv(shost); + struct lpfc_hba *phba = vport->phba; + struct fc_bsg_reply *bsg_reply = job->reply; + struct forced_link_speed_support_reply *forced_reply; + int rc = 0; + + if (job->request_len < + sizeof(struct fc_bsg_request) + + sizeof(struct get_forced_link_speed_support)) { + lpfc_printf_log(phba, KERN_WARNING, LOG_LIBDFC, + "0048 Received FORCED_LINK_SPEED request " + "below minimum size\n"); + rc = -EINVAL; + goto job_error; + } + + forced_reply = (struct forced_link_speed_support_reply *) + bsg_reply->reply_data.vendor_reply.vendor_rsp; + + if (job->reply_len < + sizeof(struct fc_bsg_request) + + sizeof(struct forced_link_speed_support_reply)) { + lpfc_printf_log(phba, KERN_WARNING, LOG_LIBDFC, + "0049 Received FORCED_LINK_SPEED reply below " + "minimum size\n"); + rc = -EINVAL; + goto job_error; + } + + forced_reply->supported = (phba->hba_flag & HBA_FORCED_LINK_SPEED) + ? LPFC_FORCED_LINK_SPEED_SUPPORTED + : LPFC_FORCED_LINK_SPEED_NOT_SUPPORTED; +job_error: + bsg_reply->result = rc; + if (rc == 0) + bsg_job_done(job, bsg_reply->result, + bsg_reply->reply_payload_rcv_len); + return rc; +} + /** * lpfc_bsg_hst_vendor - process a vendor-specific fc_bsg_job * @job: fc_bsg_job to handle **/ static int -lpfc_bsg_hst_vendor(struct fc_bsg_job *job) +lpfc_bsg_hst_vendor(struct bsg_job *job) { - int command = job->request->rqst_data.h_vendor.vendor_cmd[0]; + struct fc_bsg_request *bsg_request = job->request; + struct fc_bsg_reply *bsg_reply = job->reply; + int command = bsg_request->rqst_data.h_vendor.vendor_cmd[0]; int rc; switch (command) { @@ -5227,11 +5342,14 @@ lpfc_bsg_hst_vendor(struct fc_bsg_job *job) case LPFC_BSG_VENDOR_MENLO_DATA: rc = lpfc_menlo_cmd(job); break; + case LPFC_BSG_VENDOR_FORCED_LINK_SPEED: + rc = lpfc_forced_link_speed(job); + break; default: rc = -EINVAL; - job->reply->reply_payload_rcv_len = 0; + bsg_reply->reply_payload_rcv_len = 0; /* make error code available to userspace */ - job->reply->result = rc; + bsg_reply->result = rc; break; } @@ -5243,12 +5361,14 @@ lpfc_bsg_hst_vendor(struct fc_bsg_job *job) * @job: fc_bsg_job to handle **/ int -lpfc_bsg_request(struct fc_bsg_job *job) +lpfc_bsg_request(struct bsg_job *job) { + struct fc_bsg_request *bsg_request = job->request; + struct fc_bsg_reply *bsg_reply = job->reply; uint32_t msgcode; int rc; - msgcode = job->request->msgcode; + msgcode = bsg_request->msgcode; switch (msgcode) { case FC_BSG_HST_VENDOR: rc = lpfc_bsg_hst_vendor(job); @@ -5261,9 +5381,9 @@ lpfc_bsg_request(struct fc_bsg_job *job) break; default: rc = -EINVAL; - job->reply->reply_payload_rcv_len = 0; + bsg_reply->reply_payload_rcv_len = 0; /* make error code available to userspace */ - job->reply->result = rc; + bsg_reply->result = rc; break; } @@ -5278,9 +5398,9 @@ lpfc_bsg_request(struct fc_bsg_job *job) * the waiting function which will handle passing the error back to userspace **/ int -lpfc_bsg_timeout(struct fc_bsg_job *job) +lpfc_bsg_timeout(struct bsg_job *job) { - struct lpfc_vport *vport = (struct lpfc_vport *)job->shost->hostdata; + struct lpfc_vport *vport = shost_priv(fc_bsg_to_shost(job)); struct lpfc_hba *phba = vport->phba; struct lpfc_iocbq *cmdiocb; struct lpfc_sli_ring *pring = &phba->sli.ring[LPFC_ELS_RING]; diff --git a/drivers/scsi/lpfc/lpfc_bsg.h b/drivers/scsi/lpfc/lpfc_bsg.h index e557bcd..f2247aa 100644 --- a/drivers/scsi/lpfc/lpfc_bsg.h +++ b/drivers/scsi/lpfc/lpfc_bsg.h @@ -35,6 +35,7 @@ #define LPFC_BSG_VENDOR_MENLO_DATA 9 #define LPFC_BSG_VENDOR_DIAG_MODE_END 10 #define LPFC_BSG_VENDOR_LINK_DIAG_TEST 11 +#define LPFC_BSG_VENDOR_FORCED_LINK_SPEED 14 struct set_ct_event { uint32_t command; @@ -284,6 +285,15 @@ struct lpfc_sli_config_mbox { } un; }; +#define LPFC_FORCED_LINK_SPEED_NOT_SUPPORTED 0 +#define LPFC_FORCED_LINK_SPEED_SUPPORTED 1 +struct get_forced_link_speed_support { + uint32_t command; +}; +struct forced_link_speed_support_reply { + uint8_t supported; +}; + /* driver only */ #define SLI_CONFIG_NOT_HANDLED 0 #define SLI_CONFIG_HANDLED 1 diff --git a/drivers/scsi/lpfc/lpfc_crtn.h b/drivers/scsi/lpfc/lpfc_crtn.h index bd7576d..15d2bfd 100644 --- a/drivers/scsi/lpfc/lpfc_crtn.h +++ b/drivers/scsi/lpfc/lpfc_crtn.h @@ -397,8 +397,6 @@ extern spinlock_t _dump_buf_lock; extern int _dump_buf_done; extern spinlock_t pgcnt_lock; extern unsigned int pgcnt; -extern unsigned int lpfc_prot_mask; -extern unsigned char lpfc_prot_guard; extern unsigned int lpfc_fcp_look_ahead; /* Interface exported by fabric iocb scheduler */ @@ -431,8 +429,8 @@ struct lpfc_sglq *__lpfc_get_active_sglq(struct lpfc_hba *, uint16_t); #define HBA_EVENT_LINK_DOWN 3 /* functions to support SGIOv4/bsg interface */ -int lpfc_bsg_request(struct fc_bsg_job *); -int lpfc_bsg_timeout(struct fc_bsg_job *); +int lpfc_bsg_request(struct bsg_job *); +int lpfc_bsg_timeout(struct bsg_job *); int lpfc_bsg_ct_unsol_event(struct lpfc_hba *, struct lpfc_sli_ring *, struct lpfc_iocbq *); int lpfc_bsg_ct_unsol_abort(struct lpfc_hba *, struct hbq_dmabuf *); diff --git a/drivers/scsi/lpfc/lpfc_els.c b/drivers/scsi/lpfc/lpfc_els.c index b7d54bf..236e4e5 100644 --- a/drivers/scsi/lpfc/lpfc_els.c +++ b/drivers/scsi/lpfc/lpfc_els.c @@ -7610,7 +7610,7 @@ lpfc_els_unsol_buffer(struct lpfc_hba *phba, struct lpfc_sli_ring *pring, /* reject till our FLOGI completes */ if ((vport->port_state < LPFC_FABRIC_CFG_LINK) && (cmd != ELS_CMD_FLOGI)) { - rjt_err = LSRJT_UNABLE_TPC; + rjt_err = LSRJT_LOGICAL_BSY; rjt_exp = LSEXP_NOTHING_MORE; goto lsrjt; } diff --git a/drivers/scsi/lpfc/lpfc_hw4.h b/drivers/scsi/lpfc/lpfc_hw4.h index ee80227..5646699 100644 --- a/drivers/scsi/lpfc/lpfc_hw4.h +++ b/drivers/scsi/lpfc/lpfc_hw4.h @@ -921,6 +921,7 @@ struct mbox_header { #define LPFC_MBOX_OPCODE_GET_PORT_NAME 0x4D #define LPFC_MBOX_OPCODE_MQ_CREATE_EXT 0x5A #define LPFC_MBOX_OPCODE_GET_VPD_DATA 0x5B +#define LPFC_MBOX_OPCODE_SET_HOST_DATA 0x5D #define LPFC_MBOX_OPCODE_SEND_ACTIVATION 0x73 #define LPFC_MBOX_OPCODE_RESET_LICENSES 0x74 #define LPFC_MBOX_OPCODE_GET_RSRC_EXTENT_INFO 0x9A @@ -2289,6 +2290,9 @@ struct lpfc_mbx_read_config { #define lpfc_mbx_rd_conf_r_a_tov_SHIFT 0 #define lpfc_mbx_rd_conf_r_a_tov_MASK 0x0000FFFF #define lpfc_mbx_rd_conf_r_a_tov_WORD word6 +#define lpfc_mbx_rd_conf_link_speed_SHIFT 16 +#define lpfc_mbx_rd_conf_link_speed_MASK 0x0000FFFF +#define lpfc_mbx_rd_conf_link_speed_WORD word6 uint32_t rsvd_7; uint32_t rsvd_8; uint32_t word9; @@ -2919,6 +2923,16 @@ struct lpfc_mbx_set_feature { }; +#define LPFC_SET_HOST_OS_DRIVER_VERSION 0x2 +struct lpfc_mbx_set_host_data { +#define LPFC_HOST_OS_DRIVER_VERSION_SIZE 48 + struct mbox_header header; + uint32_t param_id; + uint32_t param_len; + uint8_t data[LPFC_HOST_OS_DRIVER_VERSION_SIZE]; +}; + + struct lpfc_mbx_get_sli4_parameters { struct mbox_header header; struct lpfc_sli4_parameters sli4_parameters; @@ -3313,6 +3327,7 @@ struct lpfc_mqe { struct lpfc_mbx_get_port_name get_port_name; struct lpfc_mbx_set_feature set_feature; struct lpfc_mbx_memory_dump_type3 mem_dump_type3; + struct lpfc_mbx_set_host_data set_host_data; struct lpfc_mbx_nop nop; } un; }; @@ -3981,7 +3996,8 @@ union lpfc_wqe128 { struct gen_req64_wqe gen_req; }; -#define LPFC_GROUP_OJECT_MAGIC_NUM 0xfeaa0001 +#define LPFC_GROUP_OJECT_MAGIC_G5 0xfeaa0001 +#define LPFC_GROUP_OJECT_MAGIC_G6 0xfeaa0003 #define LPFC_FILE_TYPE_GROUP 0xf7 #define LPFC_FILE_ID_GROUP 0xa2 struct lpfc_grp_hdr { diff --git a/drivers/scsi/lpfc/lpfc_init.c b/drivers/scsi/lpfc/lpfc_init.c index 734a042..4776fd8 100644 --- a/drivers/scsi/lpfc/lpfc_init.c +++ b/drivers/scsi/lpfc/lpfc_init.c @@ -6279,34 +6279,36 @@ lpfc_setup_bg(struct lpfc_hba *phba, struct Scsi_Host *shost) uint32_t old_guard; int pagecnt = 10; - if (lpfc_prot_mask && lpfc_prot_guard) { + if (phba->cfg_prot_mask && phba->cfg_prot_guard) { lpfc_printf_log(phba, KERN_INFO, LOG_INIT, "1478 Registering BlockGuard with the " "SCSI layer\n"); - old_mask = lpfc_prot_mask; - old_guard = lpfc_prot_guard; + old_mask = phba->cfg_prot_mask; + old_guard = phba->cfg_prot_guard; /* Only allow supported values */ - lpfc_prot_mask &= (SHOST_DIF_TYPE1_PROTECTION | + phba->cfg_prot_mask &= (SHOST_DIF_TYPE1_PROTECTION | SHOST_DIX_TYPE0_PROTECTION | SHOST_DIX_TYPE1_PROTECTION); - lpfc_prot_guard &= (SHOST_DIX_GUARD_IP | SHOST_DIX_GUARD_CRC); + phba->cfg_prot_guard &= (SHOST_DIX_GUARD_IP | + SHOST_DIX_GUARD_CRC); /* DIF Type 1 protection for profiles AST1/C1 is end to end */ - if (lpfc_prot_mask == SHOST_DIX_TYPE1_PROTECTION) - lpfc_prot_mask |= SHOST_DIF_TYPE1_PROTECTION; + if (phba->cfg_prot_mask == SHOST_DIX_TYPE1_PROTECTION) + phba->cfg_prot_mask |= SHOST_DIF_TYPE1_PROTECTION; - if (lpfc_prot_mask && lpfc_prot_guard) { - if ((old_mask != lpfc_prot_mask) || - (old_guard != lpfc_prot_guard)) + if (phba->cfg_prot_mask && phba->cfg_prot_guard) { + if ((old_mask != phba->cfg_prot_mask) || + (old_guard != phba->cfg_prot_guard)) lpfc_printf_log(phba, KERN_ERR, LOG_INIT, "1475 Registering BlockGuard with the " "SCSI layer: mask %d guard %d\n", - lpfc_prot_mask, lpfc_prot_guard); + phba->cfg_prot_mask, + phba->cfg_prot_guard); - scsi_host_set_prot(shost, lpfc_prot_mask); - scsi_host_set_guard(shost, lpfc_prot_guard); + scsi_host_set_prot(shost, phba->cfg_prot_mask); + scsi_host_set_guard(shost, phba->cfg_prot_guard); } else lpfc_printf_log(phba, KERN_ERR, LOG_INIT, "1479 Not Registering BlockGuard with the SCSI " @@ -6929,6 +6931,8 @@ lpfc_sli4_read_config(struct lpfc_hba *phba) struct lpfc_mbx_get_func_cfg *get_func_cfg; struct lpfc_rsrc_desc_fcfcoe *desc; char *pdesc_0; + uint16_t forced_link_speed; + uint32_t if_type; int length, i, rc = 0, rc2; pmb = (LPFC_MBOXQ_t *) mempool_alloc(phba->mbox_mem_pool, GFP_KERNEL); @@ -7022,6 +7026,58 @@ lpfc_sli4_read_config(struct lpfc_hba *phba) if (rc) goto read_cfg_out; + /* Update link speed if forced link speed is supported */ + if_type = bf_get(lpfc_sli_intf_if_type, &phba->sli4_hba.sli_intf); + if (if_type == LPFC_SLI_INTF_IF_TYPE_2) { + forced_link_speed = + bf_get(lpfc_mbx_rd_conf_link_speed, rd_config); + if (forced_link_speed) { + phba->hba_flag |= HBA_FORCED_LINK_SPEED; + + switch (forced_link_speed) { + case LINK_SPEED_1G: + phba->cfg_link_speed = + LPFC_USER_LINK_SPEED_1G; + break; + case LINK_SPEED_2G: + phba->cfg_link_speed = + LPFC_USER_LINK_SPEED_2G; + break; + case LINK_SPEED_4G: + phba->cfg_link_speed = + LPFC_USER_LINK_SPEED_4G; + break; + case LINK_SPEED_8G: + phba->cfg_link_speed = + LPFC_USER_LINK_SPEED_8G; + break; + case LINK_SPEED_10G: + phba->cfg_link_speed = + LPFC_USER_LINK_SPEED_10G; + break; + case LINK_SPEED_16G: + phba->cfg_link_speed = + LPFC_USER_LINK_SPEED_16G; + break; + case LINK_SPEED_32G: + phba->cfg_link_speed = + LPFC_USER_LINK_SPEED_32G; + break; + case 0xffff: + phba->cfg_link_speed = + LPFC_USER_LINK_SPEED_AUTO; + break; + default: + lpfc_printf_log(phba, KERN_ERR, LOG_SLI, + "0047 Unrecognized link " + "speed : %d\n", + forced_link_speed); + phba->cfg_link_speed = + LPFC_USER_LINK_SPEED_AUTO; + } + } + } + /* Reset the DFT_HBA_Q_DEPTH to the max xri */ length = phba->sli4_hba.max_cfg_param.max_xri - lpfc_sli4_get_els_iocb_cnt(phba); @@ -7256,6 +7312,7 @@ int lpfc_sli4_queue_create(struct lpfc_hba *phba) { struct lpfc_queue *qdesc; + uint32_t wqesize; int idx; /* @@ -7340,15 +7397,10 @@ lpfc_sli4_queue_create(struct lpfc_hba *phba) phba->sli4_hba.fcp_cq[idx] = qdesc; /* Create Fast Path FCP WQs */ - if (phba->fcp_embed_io) { - qdesc = lpfc_sli4_queue_alloc(phba, - LPFC_WQE128_SIZE, - LPFC_WQE128_DEF_COUNT); - } else { - qdesc = lpfc_sli4_queue_alloc(phba, - phba->sli4_hba.wq_esize, - phba->sli4_hba.wq_ecount); - } + wqesize = (phba->fcp_embed_io) ? + LPFC_WQE128_SIZE : phba->sli4_hba.wq_esize; + qdesc = lpfc_sli4_queue_alloc(phba, wqesize, + phba->sli4_hba.wq_ecount); if (!qdesc) { lpfc_printf_log(phba, KERN_ERR, LOG_INIT, "0503 Failed allocate fast-path FCP " @@ -10260,6 +10312,7 @@ lpfc_write_firmware(const struct firmware *fw, void *context) int i, rc = 0; struct lpfc_dmabuf *dmabuf, *next; uint32_t offset = 0, temp_offset = 0; + uint32_t magic_number, ftype, fid, fsize; /* It can be null in no-wait mode, sanity check */ if (!fw) { @@ -10268,18 +10321,19 @@ lpfc_write_firmware(const struct firmware *fw, void *context) } image = (struct lpfc_grp_hdr *)fw->data; + magic_number = be32_to_cpu(image->magic_number); + ftype = bf_get_be32(lpfc_grp_hdr_file_type, image); + fid = bf_get_be32(lpfc_grp_hdr_id, image), + fsize = be32_to_cpu(image->size); + INIT_LIST_HEAD(&dma_buffer_list); - if ((be32_to_cpu(image->magic_number) != LPFC_GROUP_OJECT_MAGIC_NUM) || - (bf_get_be32(lpfc_grp_hdr_file_type, image) != - LPFC_FILE_TYPE_GROUP) || - (bf_get_be32(lpfc_grp_hdr_id, image) != LPFC_FILE_ID_GROUP) || - (be32_to_cpu(image->size) != fw->size)) { + if ((magic_number != LPFC_GROUP_OJECT_MAGIC_G5 && + magic_number != LPFC_GROUP_OJECT_MAGIC_G6) || + ftype != LPFC_FILE_TYPE_GROUP || fsize != fw->size) { lpfc_printf_log(phba, KERN_ERR, LOG_INIT, "3022 Invalid FW image found. " - "Magic:%x Type:%x ID:%x\n", - be32_to_cpu(image->magic_number), - bf_get_be32(lpfc_grp_hdr_file_type, image), - bf_get_be32(lpfc_grp_hdr_id, image)); + "Magic:%x Type:%x ID:%x Size %d %zd\n", + magic_number, ftype, fid, fsize, fw->size); rc = -EINVAL; goto release_out; } diff --git a/drivers/scsi/lpfc/lpfc_scsi.c b/drivers/scsi/lpfc/lpfc_scsi.c index d197aa1..ad350d9 100644 --- a/drivers/scsi/lpfc/lpfc_scsi.c +++ b/drivers/scsi/lpfc/lpfc_scsi.c @@ -413,15 +413,13 @@ lpfc_new_scsi_buf_s3(struct lpfc_vport *vport, int num_to_alloc) * struct fcp_cmnd, struct fcp_rsp and the number of bde's * necessary to support the sg_tablesize. */ - psb->data = pci_pool_alloc(phba->lpfc_scsi_dma_buf_pool, + psb->data = pci_pool_zalloc(phba->lpfc_scsi_dma_buf_pool, GFP_KERNEL, &psb->dma_handle); if (!psb->data) { kfree(psb); break; } - /* Initialize virtual ptrs to dma_buf region. */ - memset(psb->data, 0, phba->cfg_sg_dma_buf_size); /* Allocate iotag for psb->cur_iocbq. */ iotag = lpfc_sli_next_iotag(phba, &psb->cur_iocbq); @@ -607,7 +605,7 @@ lpfc_sli4_fcp_xri_aborted(struct lpfc_hba *phba, } /** - * lpfc_sli4_post_scsi_sgl_list - Psot blocks of scsi buffer sgls from a list + * lpfc_sli4_post_scsi_sgl_list - Post blocks of scsi buffer sgls from a list * @phba: pointer to lpfc hba data structure. * @post_sblist: pointer to the scsi buffer list. * @@ -736,7 +734,7 @@ lpfc_sli4_post_scsi_sgl_list(struct lpfc_hba *phba, } /** - * lpfc_sli4_repost_scsi_sgl_list - Repsot all the allocated scsi buffer sgls + * lpfc_sli4_repost_scsi_sgl_list - Repost all the allocated scsi buffer sgls * @phba: pointer to lpfc hba data structure. * * This routine walks the list of scsi buffers that have been allocated and @@ -821,13 +819,12 @@ lpfc_new_scsi_buf_s4(struct lpfc_vport *vport, int num_to_alloc) * for the struct fcp_cmnd, struct fcp_rsp and the number * of bde's necessary to support the sg_tablesize. */ - psb->data = pci_pool_alloc(phba->lpfc_scsi_dma_buf_pool, + psb->data = pci_pool_zalloc(phba->lpfc_scsi_dma_buf_pool, GFP_KERNEL, &psb->dma_handle); if (!psb->data) { kfree(psb); break; } - memset(psb->data, 0, phba->cfg_sg_dma_buf_size); /* * 4K Page alignment is CRITICAL to BlockGuard, double check @@ -857,7 +854,7 @@ lpfc_new_scsi_buf_s4(struct lpfc_vport *vport, int num_to_alloc) psb->data, psb->dma_handle); kfree(psb); lpfc_printf_log(phba, KERN_ERR, LOG_FCP, - "3368 Failed to allocated IOTAG for" + "3368 Failed to allocate IOTAG for" " XRI:0x%x\n", lxri); lpfc_sli4_free_xri(phba, lxri); break; @@ -1136,7 +1133,7 @@ lpfc_release_scsi_buf(struct lpfc_hba *phba, struct lpfc_scsi_buf *psb) * * This routine does the pci dma mapping for scatter-gather list of scsi cmnd * field of @lpfc_cmd for device with SLI-3 interface spec. This routine scans - * through sg elements and format the bdea. This routine also initializes all + * through sg elements and format the bde. This routine also initializes all * IOCB fields which are dependent on scsi command request buffer. * * Return codes: @@ -1269,13 +1266,16 @@ lpfc_scsi_prep_dma_buf_s3(struct lpfc_hba *phba, struct lpfc_scsi_buf *lpfc_cmd) #ifdef CONFIG_SCSI_LPFC_DEBUG_FS -/* Return if if error injection is detected by Initiator */ +/* Return BG_ERR_INIT if error injection is detected by Initiator */ #define BG_ERR_INIT 0x1 -/* Return if if error injection is detected by Target */ +/* Return BG_ERR_TGT if error injection is detected by Target */ #define BG_ERR_TGT 0x2 -/* Return if if swapping CSUM<-->CRC is required for error injection */ +/* Return BG_ERR_SWAP if swapping CSUM<-->CRC is required for error injection */ #define BG_ERR_SWAP 0x10 -/* Return if disabling Guard/Ref/App checking is required for error injection */ +/** + * Return BG_ERR_CHECK if disabling Guard/Ref/App checking is required for + * error injection + **/ #define BG_ERR_CHECK 0x20 /** @@ -4139,13 +4139,13 @@ lpfc_scsi_cmd_iocb_cmpl(struct lpfc_hba *phba, struct lpfc_iocbq *pIocbIn, lpfc_scsi_unprep_dma_buf(phba, lpfc_cmd); - /* The sdev is not guaranteed to be valid post scsi_done upcall. */ - cmd->scsi_done(cmd); - spin_lock_irqsave(&phba->hbalock, flags); lpfc_cmd->pCmd = NULL; spin_unlock_irqrestore(&phba->hbalock, flags); + /* The sdev is not guaranteed to be valid post scsi_done upcall. */ + cmd->scsi_done(cmd); + /* * If there is a thread waiting for command completion * wake up the thread. @@ -4822,7 +4822,7 @@ wait_for_cmpl: ret = FAILED; lpfc_printf_vlog(vport, KERN_ERR, LOG_FCP, "0748 abort handler timed out waiting " - "for abortng I/O (xri:x%x) to complete: " + "for aborting I/O (xri:x%x) to complete: " "ret %#x, ID %d, LUN %llu\n", iocb->sli4_xritag, ret, cmnd->device->id, cmnd->device->lun); @@ -4945,26 +4945,30 @@ lpfc_check_fcp_rsp(struct lpfc_vport *vport, struct lpfc_scsi_buf *lpfc_cmd) * 0x2002 - Success. **/ static int -lpfc_send_taskmgmt(struct lpfc_vport *vport, struct lpfc_rport_data *rdata, - unsigned tgt_id, uint64_t lun_id, - uint8_t task_mgmt_cmd) +lpfc_send_taskmgmt(struct lpfc_vport *vport, struct scsi_cmnd *cmnd, + unsigned int tgt_id, uint64_t lun_id, + uint8_t task_mgmt_cmd) { struct lpfc_hba *phba = vport->phba; struct lpfc_scsi_buf *lpfc_cmd; struct lpfc_iocbq *iocbq; struct lpfc_iocbq *iocbqrsp; - struct lpfc_nodelist *pnode = rdata->pnode; + struct lpfc_rport_data *rdata; + struct lpfc_nodelist *pnode; int ret; int status; - if (!pnode || !NLP_CHK_NODE_ACT(pnode)) + rdata = lpfc_rport_data_from_scsi_device(cmnd->device); + if (!rdata || !rdata->pnode || !NLP_CHK_NODE_ACT(rdata->pnode)) return FAILED; + pnode = rdata->pnode; - lpfc_cmd = lpfc_get_scsi_buf(phba, rdata->pnode); + lpfc_cmd = lpfc_get_scsi_buf(phba, pnode); if (lpfc_cmd == NULL) return FAILED; lpfc_cmd->timeout = phba->cfg_task_mgmt_tmo; lpfc_cmd->rdata = rdata; + lpfc_cmd->pCmd = cmnd; status = lpfc_scsi_prep_task_mgmt_cmd(vport, lpfc_cmd, lun_id, task_mgmt_cmd); @@ -5171,7 +5175,7 @@ lpfc_device_reset_handler(struct scsi_cmnd *cmnd) fc_host_post_vendor_event(shost, fc_get_event_number(), sizeof(scsi_event), (char *)&scsi_event, LPFC_NL_VENDOR_ID); - status = lpfc_send_taskmgmt(vport, rdata, tgt_id, lun_id, + status = lpfc_send_taskmgmt(vport, cmnd, tgt_id, lun_id, FCP_LUN_RESET); lpfc_printf_vlog(vport, KERN_ERR, LOG_FCP, @@ -5249,7 +5253,7 @@ lpfc_target_reset_handler(struct scsi_cmnd *cmnd) fc_host_post_vendor_event(shost, fc_get_event_number(), sizeof(scsi_event), (char *)&scsi_event, LPFC_NL_VENDOR_ID); - status = lpfc_send_taskmgmt(vport, rdata, tgt_id, lun_id, + status = lpfc_send_taskmgmt(vport, cmnd, tgt_id, lun_id, FCP_TARGET_RESET); lpfc_printf_vlog(vport, KERN_ERR, LOG_FCP, @@ -5328,7 +5332,7 @@ lpfc_bus_reset_handler(struct scsi_cmnd *cmnd) if (!match) continue; - status = lpfc_send_taskmgmt(vport, ndlp->rport->dd_data, + status = lpfc_send_taskmgmt(vport, cmnd, i, 0, FCP_TARGET_RESET); if (status != SUCCESS) { diff --git a/drivers/scsi/lpfc/lpfc_sli.c b/drivers/scsi/lpfc/lpfc_sli.c index f4f77c5..4faa767 100644 --- a/drivers/scsi/lpfc/lpfc_sli.c +++ b/drivers/scsi/lpfc/lpfc_sli.c @@ -47,6 +47,7 @@ #include "lpfc_compat.h" #include "lpfc_debugfs.h" #include "lpfc_vport.h" +#include "lpfc_version.h" /* There are only four IOCB completion types. */ typedef enum _lpfc_iocb_type { @@ -2678,15 +2679,16 @@ lpfc_sli_iocbq_lookup(struct lpfc_hba *phba, if (iotag != 0 && iotag <= phba->sli.last_iotag) { cmd_iocb = phba->sli.iocbq_lookup[iotag]; - list_del_init(&cmd_iocb->list); if (cmd_iocb->iocb_flag & LPFC_IO_ON_TXCMPLQ) { + /* remove from txcmpl queue list */ + list_del_init(&cmd_iocb->list); cmd_iocb->iocb_flag &= ~LPFC_IO_ON_TXCMPLQ; + return cmd_iocb; } - return cmd_iocb; } lpfc_printf_log(phba, KERN_ERR, LOG_SLI, - "0317 iotag x%x is out off " + "0317 iotag x%x is out of " "range: max iotag x%x wd0 x%x\n", iotag, phba->sli.last_iotag, *(((uint32_t *) &prspiocb->iocb) + 7)); @@ -2721,8 +2723,9 @@ lpfc_sli_iocbq_lookup_by_tag(struct lpfc_hba *phba, return cmd_iocb; } } + lpfc_printf_log(phba, KERN_ERR, LOG_SLI, - "0372 iotag x%x is out off range: max iotag (x%x)\n", + "0372 iotag x%x is out of range: max iotag (x%x)\n", iotag, phba->sli.last_iotag); return NULL; } @@ -6291,6 +6294,25 @@ lpfc_sli4_repost_els_sgl_list(struct lpfc_hba *phba) return 0; } +void +lpfc_set_host_data(struct lpfc_hba *phba, LPFC_MBOXQ_t *mbox) +{ + uint32_t len; + + len = sizeof(struct lpfc_mbx_set_host_data) - + sizeof(struct lpfc_sli4_cfg_mhdr); + lpfc_sli4_config(phba, mbox, LPFC_MBOX_SUBSYSTEM_COMMON, + LPFC_MBOX_OPCODE_SET_HOST_DATA, len, + LPFC_SLI4_MBX_EMBED); + + mbox->u.mqe.un.set_host_data.param_id = LPFC_SET_HOST_OS_DRIVER_VERSION; + mbox->u.mqe.un.set_host_data.param_len = 8; + snprintf(mbox->u.mqe.un.set_host_data.data, + LPFC_HOST_OS_DRIVER_VERSION_SIZE, + "Linux %s v"LPFC_DRIVER_VERSION, + (phba->hba_flag & HBA_FCOE_MODE) ? "FCoE" : "FC"); +} + /** * lpfc_sli4_hba_setup - SLI4 device intialization PCI function * @phba: Pointer to HBA context object. @@ -6542,6 +6564,15 @@ lpfc_sli4_hba_setup(struct lpfc_hba *phba) goto out_free_mbox; } + lpfc_set_host_data(phba, mboxq); + + rc = lpfc_sli_issue_mbox(phba, mboxq, MBX_POLL); + if (rc) { + lpfc_printf_log(phba, KERN_WARNING, LOG_MBOX | LOG_SLI, + "2134 Failed to set host os driver version %x", + rc); + } + /* Read the port's service parameters. */ rc = lpfc_read_sparam(phba, mboxq, vport->vpi); if (rc) { @@ -11781,6 +11812,8 @@ lpfc_sli4_els_wcqe_to_rspiocbq(struct lpfc_hba *phba, /* Look up the ELS command IOCB and create pseudo response IOCB */ cmdiocbq = lpfc_sli_iocbq_lookup_by_tag(phba, pring, bf_get(lpfc_wcqe_c_request_tag, wcqe)); + /* Put the iocb back on the txcmplq */ + lpfc_sli_ringtxcmpl_put(phba, pring, cmdiocbq); spin_unlock_irqrestore(&pring->ring_lock, iflags); if (unlikely(!cmdiocbq)) { diff --git a/drivers/scsi/lpfc/lpfc_version.h b/drivers/scsi/lpfc/lpfc_version.h index c9bf20e..50bfc43 100644 --- a/drivers/scsi/lpfc/lpfc_version.h +++ b/drivers/scsi/lpfc/lpfc_version.h @@ -18,7 +18,7 @@ * included with this package. * *******************************************************************/ -#define LPFC_DRIVER_VERSION "11.2.0.0." +#define LPFC_DRIVER_VERSION "11.2.0.2" #define LPFC_DRIVER_NAME "lpfc" /* Used for SLI 2/3 */ diff --git a/drivers/scsi/mac_scsi.c b/drivers/scsi/mac_scsi.c index a590089..ccb68d1 100644 --- a/drivers/scsi/mac_scsi.c +++ b/drivers/scsi/mac_scsi.c @@ -28,17 +28,15 @@ /* Definitions for the core NCR5380 driver. */ -#define NCR5380_implementation_fields unsigned char *pdma_base; \ - int pdma_residual +#define NCR5380_implementation_fields int pdma_residual -#define NCR5380_read(reg) macscsi_read(instance, reg) -#define NCR5380_write(reg, value) macscsi_write(instance, reg, value) +#define NCR5380_read(reg) in_8(hostdata->io + ((reg) << 4)) +#define NCR5380_write(reg, value) out_8(hostdata->io + ((reg) << 4), value) -#define NCR5380_dma_xfer_len(instance, cmd, phase) \ - macscsi_dma_xfer_len(instance, cmd) +#define NCR5380_dma_xfer_len macscsi_dma_xfer_len #define NCR5380_dma_recv_setup macscsi_pread #define NCR5380_dma_send_setup macscsi_pwrite -#define NCR5380_dma_residual(instance) (hostdata->pdma_residual) +#define NCR5380_dma_residual macscsi_dma_residual #define NCR5380_intr macscsi_intr #define NCR5380_queue_command macscsi_queue_command @@ -61,20 +59,6 @@ module_param(setup_hostid, int, 0); static int setup_toshiba_delay = -1; module_param(setup_toshiba_delay, int, 0); -/* - * NCR 5380 register access functions - */ - -static inline char macscsi_read(struct Scsi_Host *instance, int reg) -{ - return in_8(instance->base + (reg << 4)); -} - -static inline void macscsi_write(struct Scsi_Host *instance, int reg, int value) -{ - out_8(instance->base + (reg << 4), value); -} - #ifndef MODULE static int __init mac_scsi_setup(char *str) { @@ -167,16 +151,15 @@ __asm__ __volatile__ \ : "0"(s), "1"(d), "2"(n) \ : "d0") -static int macscsi_pread(struct Scsi_Host *instance, - unsigned char *dst, int len) +static inline int macscsi_pread(struct NCR5380_hostdata *hostdata, + unsigned char *dst, int len) { - struct NCR5380_hostdata *hostdata = shost_priv(instance); - unsigned char *s = hostdata->pdma_base + (INPUT_DATA_REG << 4); + unsigned char *s = hostdata->pdma_io + (INPUT_DATA_REG << 4); unsigned char *d = dst; int n = len; int transferred; - while (!NCR5380_poll_politely(instance, BUS_AND_STATUS_REG, + while (!NCR5380_poll_politely(hostdata, BUS_AND_STATUS_REG, BASR_DRQ | BASR_PHASE_MATCH, BASR_DRQ | BASR_PHASE_MATCH, HZ / 64)) { CP_IO_TO_MEM(s, d, n); @@ -189,23 +172,23 @@ static int macscsi_pread(struct Scsi_Host *instance, return 0; /* Target changed phase early? */ - if (NCR5380_poll_politely2(instance, STATUS_REG, SR_REQ, SR_REQ, + if (NCR5380_poll_politely2(hostdata, STATUS_REG, SR_REQ, SR_REQ, BUS_AND_STATUS_REG, BASR_ACK, BASR_ACK, HZ / 64) < 0) scmd_printk(KERN_ERR, hostdata->connected, "%s: !REQ and !ACK\n", __func__); if (!(NCR5380_read(BUS_AND_STATUS_REG) & BASR_PHASE_MATCH)) return 0; - dsprintk(NDEBUG_PSEUDO_DMA, instance, + dsprintk(NDEBUG_PSEUDO_DMA, hostdata->host, "%s: bus error (%d/%d)\n", __func__, transferred, len); - NCR5380_dprint(NDEBUG_PSEUDO_DMA, instance); + NCR5380_dprint(NDEBUG_PSEUDO_DMA, hostdata->host); d = dst + transferred; n = len - transferred; } scmd_printk(KERN_ERR, hostdata->connected, "%s: phase mismatch or !DRQ\n", __func__); - NCR5380_dprint(NDEBUG_PSEUDO_DMA, instance); + NCR5380_dprint(NDEBUG_PSEUDO_DMA, hostdata->host); return -1; } @@ -270,16 +253,15 @@ __asm__ __volatile__ \ : "0"(s), "1"(d), "2"(n) \ : "d0") -static int macscsi_pwrite(struct Scsi_Host *instance, - unsigned char *src, int len) +static inline int macscsi_pwrite(struct NCR5380_hostdata *hostdata, + unsigned char *src, int len) { - struct NCR5380_hostdata *hostdata = shost_priv(instance); unsigned char *s = src; - unsigned char *d = hostdata->pdma_base + (OUTPUT_DATA_REG << 4); + unsigned char *d = hostdata->pdma_io + (OUTPUT_DATA_REG << 4); int n = len; int transferred; - while (!NCR5380_poll_politely(instance, BUS_AND_STATUS_REG, + while (!NCR5380_poll_politely(hostdata, BUS_AND_STATUS_REG, BASR_DRQ | BASR_PHASE_MATCH, BASR_DRQ | BASR_PHASE_MATCH, HZ / 64)) { CP_MEM_TO_IO(s, d, n); @@ -288,7 +270,7 @@ static int macscsi_pwrite(struct Scsi_Host *instance, hostdata->pdma_residual = len - transferred; /* Target changed phase early? */ - if (NCR5380_poll_politely2(instance, STATUS_REG, SR_REQ, SR_REQ, + if (NCR5380_poll_politely2(hostdata, STATUS_REG, SR_REQ, SR_REQ, BUS_AND_STATUS_REG, BASR_ACK, BASR_ACK, HZ / 64) < 0) scmd_printk(KERN_ERR, hostdata->connected, "%s: !REQ and !ACK\n", __func__); @@ -297,7 +279,7 @@ static int macscsi_pwrite(struct Scsi_Host *instance, /* No bus error. */ if (n == 0) { - if (NCR5380_poll_politely(instance, TARGET_COMMAND_REG, + if (NCR5380_poll_politely(hostdata, TARGET_COMMAND_REG, TCR_LAST_BYTE_SENT, TCR_LAST_BYTE_SENT, HZ / 64) < 0) scmd_printk(KERN_ERR, hostdata->connected, @@ -305,25 +287,23 @@ static int macscsi_pwrite(struct Scsi_Host *instance, return 0; } - dsprintk(NDEBUG_PSEUDO_DMA, instance, + dsprintk(NDEBUG_PSEUDO_DMA, hostdata->host, "%s: bus error (%d/%d)\n", __func__, transferred, len); - NCR5380_dprint(NDEBUG_PSEUDO_DMA, instance); + NCR5380_dprint(NDEBUG_PSEUDO_DMA, hostdata->host); s = src + transferred; n = len - transferred; } scmd_printk(KERN_ERR, hostdata->connected, "%s: phase mismatch or !DRQ\n", __func__); - NCR5380_dprint(NDEBUG_PSEUDO_DMA, instance); + NCR5380_dprint(NDEBUG_PSEUDO_DMA, hostdata->host); return -1; } -static int macscsi_dma_xfer_len(struct Scsi_Host *instance, +static int macscsi_dma_xfer_len(struct NCR5380_hostdata *hostdata, struct scsi_cmnd *cmd) { - struct NCR5380_hostdata *hostdata = shost_priv(instance); - if (hostdata->flags & FLAG_NO_PSEUDO_DMA || cmd->SCp.this_residual < 16) return 0; @@ -331,6 +311,11 @@ static int macscsi_dma_xfer_len(struct Scsi_Host *instance, return cmd->SCp.this_residual; } +static int macscsi_dma_residual(struct NCR5380_hostdata *hostdata) +{ + return hostdata->pdma_residual; +} + #include "NCR5380.c" #define DRV_MODULE_NAME "mac_scsi" @@ -356,6 +341,7 @@ static struct scsi_host_template mac_scsi_template = { static int __init mac_scsi_probe(struct platform_device *pdev) { struct Scsi_Host *instance; + struct NCR5380_hostdata *hostdata; int error; int host_flags = 0; struct resource *irq, *pio_mem, *pdma_mem = NULL; @@ -388,17 +374,18 @@ static int __init mac_scsi_probe(struct platform_device *pdev) if (!instance) return -ENOMEM; - instance->base = pio_mem->start; if (irq) instance->irq = irq->start; else instance->irq = NO_IRQ; - if (pdma_mem && setup_use_pdma) { - struct NCR5380_hostdata *hostdata = shost_priv(instance); + hostdata = shost_priv(instance); + hostdata->base = pio_mem->start; + hostdata->io = (void *)pio_mem->start; - hostdata->pdma_base = (unsigned char *)pdma_mem->start; - } else + if (pdma_mem && setup_use_pdma) + hostdata->pdma_io = (void *)pdma_mem->start; + else host_flags |= FLAG_NO_PSEUDO_DMA; host_flags |= setup_toshiba_delay > 0 ? FLAG_TOSHIBA_DELAY : 0; diff --git a/drivers/scsi/megaraid/megaraid_sas.h b/drivers/scsi/megaraid/megaraid_sas.h index 3aaea71..fdd519c 100644 --- a/drivers/scsi/megaraid/megaraid_sas.h +++ b/drivers/scsi/megaraid/megaraid_sas.h @@ -35,8 +35,8 @@ /* * MegaRAID SAS Driver meta data */ -#define MEGASAS_VERSION "06.811.02.00-rc1" -#define MEGASAS_RELDATE "April 12, 2016" +#define MEGASAS_VERSION "06.812.07.00-rc1" +#define MEGASAS_RELDATE "August 22, 2016" /* * Device IDs @@ -1429,6 +1429,8 @@ enum FW_BOOT_CONTEXT { #define MR_MAX_REPLY_QUEUES_EXT_OFFSET_SHIFT 14 #define MR_MAX_MSIX_REG_ARRAY 16 #define MR_RDPQ_MODE_OFFSET 0X00800000 +#define MR_CAN_HANDLE_SYNC_CACHE_OFFSET 0X01000000 + /* * register set for both 1068 and 1078 controllers * structure extended for 1078 registers @@ -2118,7 +2120,6 @@ struct megasas_instance { u32 ctrl_context_pages; struct megasas_ctrl_info *ctrl_info; unsigned int msix_vectors; - struct msix_entry msixentry[MEGASAS_MAX_MSIX_QUEUES]; struct megasas_irq_context irq_context[MEGASAS_MAX_MSIX_QUEUES]; u64 map_id; u64 pd_seq_map_id; @@ -2140,6 +2141,7 @@ struct megasas_instance { u8 is_imr; u8 is_rdpq; bool dev_handle; + bool fw_sync_cache_support; }; struct MR_LD_VF_MAP { u32 size; diff --git a/drivers/scsi/megaraid/megaraid_sas_base.c b/drivers/scsi/megaraid/megaraid_sas_base.c index d8b1fbd..6484c38 100644 --- a/drivers/scsi/megaraid/megaraid_sas_base.c +++ b/drivers/scsi/megaraid/megaraid_sas_base.c @@ -1700,11 +1700,8 @@ megasas_queue_command(struct Scsi_Host *shost, struct scsi_cmnd *scmd) goto out_done; } - /* - * FW takes care of flush cache on its own for Virtual Disk. - * No need to send it down for VD. For JBOD send SYNCHRONIZE_CACHE to FW. - */ - if ((scmd->cmnd[0] == SYNCHRONIZE_CACHE) && MEGASAS_IS_LOGICAL(scmd)) { + if ((scmd->cmnd[0] == SYNCHRONIZE_CACHE) && MEGASAS_IS_LOGICAL(scmd) && + (!instance->fw_sync_cache_support)) { scmd->result = DID_OK << 16; goto out_done; } @@ -4840,7 +4837,7 @@ fail_alloc_cmds: } /* - * megasas_setup_irqs_msix - register legacy interrupts. + * megasas_setup_irqs_ioapic - register legacy interrupts. * @instance: Adapter soft state * * Do not enable interrupt, only setup ISRs. @@ -4855,8 +4852,9 @@ megasas_setup_irqs_ioapic(struct megasas_instance *instance) pdev = instance->pdev; instance->irq_context[0].instance = instance; instance->irq_context[0].MSIxIndex = 0; - if (request_irq(pdev->irq, instance->instancet->service_isr, - IRQF_SHARED, "megasas", &instance->irq_context[0])) { + if (request_irq(pci_irq_vector(pdev, 0), + instance->instancet->service_isr, IRQF_SHARED, + "megasas", &instance->irq_context[0])) { dev_err(&instance->pdev->dev, "Failed to register IRQ from %s %d\n", __func__, __LINE__); @@ -4877,28 +4875,23 @@ megasas_setup_irqs_ioapic(struct megasas_instance *instance) static int megasas_setup_irqs_msix(struct megasas_instance *instance, u8 is_probe) { - int i, j, cpu; + int i, j; struct pci_dev *pdev; pdev = instance->pdev; /* Try MSI-x */ - cpu = cpumask_first(cpu_online_mask); for (i = 0; i < instance->msix_vectors; i++) { instance->irq_context[i].instance = instance; instance->irq_context[i].MSIxIndex = i; - if (request_irq(instance->msixentry[i].vector, + if (request_irq(pci_irq_vector(pdev, i), instance->instancet->service_isr, 0, "megasas", &instance->irq_context[i])) { dev_err(&instance->pdev->dev, "Failed to register IRQ for vector %d.\n", i); - for (j = 0; j < i; j++) { - if (smp_affinity_enable) - irq_set_affinity_hint( - instance->msixentry[j].vector, NULL); - free_irq(instance->msixentry[j].vector, - &instance->irq_context[j]); - } + for (j = 0; j < i; j++) + free_irq(pci_irq_vector(pdev, j), + &instance->irq_context[j]); /* Retry irq register for IO_APIC*/ instance->msix_vectors = 0; if (is_probe) @@ -4906,14 +4899,6 @@ megasas_setup_irqs_msix(struct megasas_instance *instance, u8 is_probe) else return -1; } - if (smp_affinity_enable) { - if (irq_set_affinity_hint(instance->msixentry[i].vector, - get_cpu_mask(cpu))) - dev_err(&instance->pdev->dev, - "Failed to set affinity hint" - " for cpu %d\n", cpu); - cpu = cpumask_next(cpu, cpu_online_mask); - } } return 0; } @@ -4930,14 +4915,12 @@ megasas_destroy_irqs(struct megasas_instance *instance) { if (instance->msix_vectors) for (i = 0; i < instance->msix_vectors; i++) { - if (smp_affinity_enable) - irq_set_affinity_hint( - instance->msixentry[i].vector, NULL); - free_irq(instance->msixentry[i].vector, + free_irq(pci_irq_vector(instance->pdev, i), &instance->irq_context[i]); } else - free_irq(instance->pdev->irq, &instance->irq_context[0]); + free_irq(pci_irq_vector(instance->pdev, 0), + &instance->irq_context[0]); } /** @@ -5095,6 +5078,8 @@ static int megasas_init_fw(struct megasas_instance *instance) msix_enable = (instance->instancet->read_fw_status_reg(reg_set) & 0x4000000) >> 0x1a; if (msix_enable && !msix_disable) { + int irq_flags = PCI_IRQ_MSIX; + scratch_pad_2 = readl (&instance->reg_set->outbound_scratch_pad_2); /* Check max MSI-X vectors */ @@ -5131,15 +5116,18 @@ static int megasas_init_fw(struct megasas_instance *instance) /* Don't bother allocating more MSI-X vectors than cpus */ instance->msix_vectors = min(instance->msix_vectors, (unsigned int)num_online_cpus()); - for (i = 0; i < instance->msix_vectors; i++) - instance->msixentry[i].entry = i; - i = pci_enable_msix_range(instance->pdev, instance->msixentry, - 1, instance->msix_vectors); + if (smp_affinity_enable) + irq_flags |= PCI_IRQ_AFFINITY; + i = pci_alloc_irq_vectors(instance->pdev, 1, + instance->msix_vectors, irq_flags); if (i > 0) instance->msix_vectors = i; else instance->msix_vectors = 0; } + i = pci_alloc_irq_vectors(instance->pdev, 1, 1, PCI_IRQ_LEGACY); + if (i < 0) + goto fail_setup_irqs; dev_info(&instance->pdev->dev, "firmware supports msix\t: (%d)", fw_msix_count); @@ -5152,11 +5140,6 @@ static int megasas_init_fw(struct megasas_instance *instance) tasklet_init(&instance->isr_tasklet, instance->instancet->tasklet, (unsigned long)instance); - if (instance->msix_vectors ? - megasas_setup_irqs_msix(instance, 1) : - megasas_setup_irqs_ioapic(instance)) - goto fail_setup_irqs; - instance->ctrl_info = kzalloc(sizeof(struct megasas_ctrl_info), GFP_KERNEL); if (instance->ctrl_info == NULL) @@ -5172,6 +5155,10 @@ static int megasas_init_fw(struct megasas_instance *instance) if (instance->instancet->init_adapter(instance)) goto fail_init_adapter; + if (instance->msix_vectors ? + megasas_setup_irqs_msix(instance, 1) : + megasas_setup_irqs_ioapic(instance)) + goto fail_init_adapter; instance->instancet->enable_intr(instance); @@ -5315,7 +5302,7 @@ fail_init_adapter: megasas_destroy_irqs(instance); fail_setup_irqs: if (instance->msix_vectors) - pci_disable_msix(instance->pdev); + pci_free_irq_vectors(instance->pdev); instance->msix_vectors = 0; fail_ready_state: kfree(instance->ctrl_info); @@ -5584,7 +5571,6 @@ static int megasas_io_attach(struct megasas_instance *instance) /* * Export parameters required by SCSI mid-layer */ - host->irq = instance->pdev->irq; host->unique_id = instance->unique_id; host->can_queue = instance->max_scsi_cmds; host->this_id = instance->init_id; @@ -5947,7 +5933,7 @@ fail_io_attach: else megasas_release_mfi(instance); if (instance->msix_vectors) - pci_disable_msix(instance->pdev); + pci_free_irq_vectors(instance->pdev); fail_init_mfi: fail_alloc_dma_buf: if (instance->evt_detail) @@ -6105,7 +6091,7 @@ megasas_suspend(struct pci_dev *pdev, pm_message_t state) megasas_destroy_irqs(instance); if (instance->msix_vectors) - pci_disable_msix(instance->pdev); + pci_free_irq_vectors(instance->pdev); pci_save_state(pdev); pci_disable_device(pdev); @@ -6125,6 +6111,7 @@ megasas_resume(struct pci_dev *pdev) int rval; struct Scsi_Host *host; struct megasas_instance *instance; + int irq_flags = PCI_IRQ_LEGACY; instance = pci_get_drvdata(pdev); host = instance->host; @@ -6160,9 +6147,15 @@ megasas_resume(struct pci_dev *pdev) goto fail_ready_state; /* Now re-enable MSI-X */ - if (instance->msix_vectors && - pci_enable_msix_exact(instance->pdev, instance->msixentry, - instance->msix_vectors)) + if (instance->msix_vectors) { + irq_flags = PCI_IRQ_MSIX; + if (smp_affinity_enable) + irq_flags |= PCI_IRQ_AFFINITY; + } + rval = pci_alloc_irq_vectors(instance->pdev, 1, + instance->msix_vectors ? + instance->msix_vectors : 1, irq_flags); + if (rval < 0) goto fail_reenable_msix; if (instance->ctrl_context) { @@ -6245,6 +6238,34 @@ fail_reenable_msix: #define megasas_resume NULL #endif +static inline int +megasas_wait_for_adapter_operational(struct megasas_instance *instance) +{ + int wait_time = MEGASAS_RESET_WAIT_TIME * 2; + int i; + + if (atomic_read(&instance->adprecovery) == MEGASAS_HW_CRITICAL_ERROR) + return 1; + + for (i = 0; i < wait_time; i++) { + if (atomic_read(&instance->adprecovery) == MEGASAS_HBA_OPERATIONAL) + break; + + if (!(i % MEGASAS_RESET_NOTICE_INTERVAL)) + dev_notice(&instance->pdev->dev, "waiting for controller reset to finish\n"); + + msleep(1000); + } + + if (atomic_read(&instance->adprecovery) != MEGASAS_HBA_OPERATIONAL) { + dev_info(&instance->pdev->dev, "%s timed out while waiting for HBA to recover.\n", + __func__); + return 1; + } + + return 0; +} + /** * megasas_detach_one - PCI hot"un"plug entry point * @pdev: PCI device structure @@ -6269,9 +6290,14 @@ static void megasas_detach_one(struct pci_dev *pdev) if (instance->fw_crash_state != UNAVAILABLE) megasas_free_host_crash_buffer(instance); scsi_remove_host(instance->host); + + if (megasas_wait_for_adapter_operational(instance)) + goto skip_firing_dcmds; + megasas_flush_cache(instance); megasas_shutdown_controller(instance, MR_DCMD_CTRL_SHUTDOWN); +skip_firing_dcmds: /* cancel the delayed work if this work still in queue*/ if (instance->ev != NULL) { struct megasas_aen_event *ev = instance->ev; @@ -6302,7 +6328,7 @@ static void megasas_detach_one(struct pci_dev *pdev) megasas_destroy_irqs(instance); if (instance->msix_vectors) - pci_disable_msix(instance->pdev); + pci_free_irq_vectors(instance->pdev); if (instance->ctrl_context) { megasas_release_fusion(instance); @@ -6385,13 +6411,19 @@ static void megasas_shutdown(struct pci_dev *pdev) struct megasas_instance *instance = pci_get_drvdata(pdev); instance->unload = 1; + + if (megasas_wait_for_adapter_operational(instance)) + goto skip_firing_dcmds; + megasas_flush_cache(instance); megasas_shutdown_controller(instance, MR_DCMD_CTRL_SHUTDOWN); + +skip_firing_dcmds: instance->instancet->disable_intr(instance); megasas_destroy_irqs(instance); if (instance->msix_vectors) - pci_disable_msix(instance->pdev); + pci_free_irq_vectors(instance->pdev); } /** @@ -6752,8 +6784,7 @@ static int megasas_mgmt_ioctl_fw(struct file *file, unsigned long arg) if (atomic_read(&instance->adprecovery) != MEGASAS_HBA_OPERATIONAL) { spin_unlock_irqrestore(&instance->hba_lock, flags); - dev_err(&instance->pdev->dev, "timed out while" - "waiting for HBA to recover\n"); + dev_err(&instance->pdev->dev, "timed out while waiting for HBA to recover\n"); error = -ENODEV; goto out_up; } @@ -6821,8 +6852,7 @@ static int megasas_mgmt_ioctl_aen(struct file *file, unsigned long arg) spin_lock_irqsave(&instance->hba_lock, flags); if (atomic_read(&instance->adprecovery) != MEGASAS_HBA_OPERATIONAL) { spin_unlock_irqrestore(&instance->hba_lock, flags); - dev_err(&instance->pdev->dev, "timed out while waiting" - "for HBA to recover\n"); + dev_err(&instance->pdev->dev, "timed out while waiting for HBA to recover\n"); return -ENODEV; } spin_unlock_irqrestore(&instance->hba_lock, flags); diff --git a/drivers/scsi/megaraid/megaraid_sas_fp.c b/drivers/scsi/megaraid/megaraid_sas_fp.c index e413113..f237d00 100644 --- a/drivers/scsi/megaraid/megaraid_sas_fp.c +++ b/drivers/scsi/megaraid/megaraid_sas_fp.c @@ -782,7 +782,8 @@ static u8 mr_spanset_get_phy_params(struct megasas_instance *instance, u32 ld, (raid->regTypeReqOnRead != REGION_TYPE_UNUSED)))) pRAID_Context->regLockFlags = REGION_TYPE_EXCLUSIVE; else if (raid->level == 1) { - pd = MR_ArPdGet(arRef, physArm + 1, map); + physArm = physArm + 1; + pd = MR_ArPdGet(arRef, physArm, map); if (pd != MR_PD_INVALID) *pDevHandle = MR_PdDevHandleGet(pd, map); } @@ -879,7 +880,8 @@ u8 MR_GetPhyParams(struct megasas_instance *instance, u32 ld, u64 stripRow, pRAID_Context->regLockFlags = REGION_TYPE_EXCLUSIVE; else if (raid->level == 1) { /* Get alternate Pd. */ - pd = MR_ArPdGet(arRef, physArm + 1, map); + physArm = physArm + 1; + pd = MR_ArPdGet(arRef, physArm, map); if (pd != MR_PD_INVALID) /* Get dev handle from Pd */ *pDevHandle = MR_PdDevHandleGet(pd, map); diff --git a/drivers/scsi/megaraid/megaraid_sas_fusion.c b/drivers/scsi/megaraid/megaraid_sas_fusion.c index 52d8bbf..24778ba 100644 --- a/drivers/scsi/megaraid/megaraid_sas_fusion.c +++ b/drivers/scsi/megaraid/megaraid_sas_fusion.c @@ -748,6 +748,11 @@ megasas_ioc_init_fusion(struct megasas_instance *instance) goto fail_fw_init; } + instance->fw_sync_cache_support = (scratch_pad_2 & + MR_CAN_HANDLE_SYNC_CACHE_OFFSET) ? 1 : 0; + dev_info(&instance->pdev->dev, "FW supports sync cache\t: %s\n", + instance->fw_sync_cache_support ? "Yes" : "No"); + IOCInitMessage = dma_alloc_coherent(&instance->pdev->dev, sizeof(struct MPI2_IOC_INIT_REQUEST), @@ -2000,6 +2005,8 @@ megasas_build_syspd_fusion(struct megasas_instance *instance, io_request->DevHandle = pd_sync->seq[pd_index].devHandle; pRAID_Context->regLockFlags |= (MR_RL_FLAGS_SEQ_NUM_ENABLE|MR_RL_FLAGS_GRANT_DESTINATION_CUDA); + pRAID_Context->Type = MPI2_TYPE_CUDA; + pRAID_Context->nseg = 0x1; } else if (fusion->fast_path_io) { pRAID_Context->VirtualDiskTgtId = cpu_to_le16(device_id); pRAID_Context->configSeqNum = 0; @@ -2035,12 +2042,10 @@ megasas_build_syspd_fusion(struct megasas_instance *instance, pRAID_Context->timeoutValue = cpu_to_le16((os_timeout_value > timeout_limit) ? timeout_limit : os_timeout_value); - if (fusion->adapter_type == INVADER_SERIES) { - pRAID_Context->Type = MPI2_TYPE_CUDA; - pRAID_Context->nseg = 0x1; + if (fusion->adapter_type == INVADER_SERIES) io_request->IoFlags |= cpu_to_le16(MPI25_SAS_DEVICE0_FLAGS_ENABLED_FAST_PATH); - } + cmd->request_desc->SCSIIO.RequestFlags = (MPI2_REQ_DESCRIPT_FLAGS_FP_IO << MEGASAS_REQ_DESCRIPT_FLAGS_TYPE_SHIFT); @@ -2463,12 +2468,15 @@ irqreturn_t megasas_isr_fusion(int irq, void *devp) /* Start collecting crash, if DMA bit is done */ if ((fw_state == MFI_STATE_FAULT) && dma_state) schedule_work(&instance->crash_init); - else if (fw_state == MFI_STATE_FAULT) - schedule_work(&instance->work_init); + else if (fw_state == MFI_STATE_FAULT) { + if (instance->unload == 0) + schedule_work(&instance->work_init); + } } else if (fw_state == MFI_STATE_FAULT) { dev_warn(&instance->pdev->dev, "Iop2SysDoorbellInt" "for scsi%d\n", instance->host->host_no); - schedule_work(&instance->work_init); + if (instance->unload == 0) + schedule_work(&instance->work_init); } } @@ -2823,6 +2831,7 @@ int megasas_wait_for_outstanding_fusion(struct megasas_instance *instance, dev_err(&instance->pdev->dev, "pending commands remain after waiting, " "will reset adapter scsi%d.\n", instance->host->host_no); + *convert = 1; retval = 1; } out: diff --git a/drivers/scsi/mpt3sas/mpi/mpi2_cnfg.h b/drivers/scsi/mpt3sas/mpi/mpi2_cnfg.h index 95356a8..fa61baf 100644 --- a/drivers/scsi/mpt3sas/mpi/mpi2_cnfg.h +++ b/drivers/scsi/mpt3sas/mpi/mpi2_cnfg.h @@ -478,6 +478,13 @@ typedef struct _MPI2_CONFIG_REPLY { #define MPI26_MFGPAGE_DEVID_SAS3324_3 (0x00C2) #define MPI26_MFGPAGE_DEVID_SAS3324_4 (0x00C3) +#define MPI26_MFGPAGE_DEVID_SAS3516 (0x00AA) +#define MPI26_MFGPAGE_DEVID_SAS3516_1 (0x00AB) +#define MPI26_MFGPAGE_DEVID_SAS3416 (0x00AC) +#define MPI26_MFGPAGE_DEVID_SAS3508 (0x00AD) +#define MPI26_MFGPAGE_DEVID_SAS3508_1 (0x00AE) +#define MPI26_MFGPAGE_DEVID_SAS3408 (0x00AF) + /*Manufacturing Page 0 */ typedef struct _MPI2_CONFIG_PAGE_MAN_0 { diff --git a/drivers/scsi/mpt3sas/mpt3sas_base.c b/drivers/scsi/mpt3sas/mpt3sas_base.c index a1a5ceb..f00ef88 100644 --- a/drivers/scsi/mpt3sas/mpt3sas_base.c +++ b/drivers/scsi/mpt3sas/mpt3sas_base.c @@ -849,7 +849,7 @@ _base_async_event(struct MPT3SAS_ADAPTER *ioc, u8 msix_index, u32 reply) ack_request->EventContext = mpi_reply->EventContext; ack_request->VF_ID = 0; /* TODO */ ack_request->VP_ID = 0; - mpt3sas_base_put_smid_default(ioc, smid); + ioc->put_smid_default(ioc, smid); out: @@ -1078,7 +1078,7 @@ _base_interrupt(int irq, void *bus_id) * new reply host index value in ReplyPostIndex Field and msix_index * value in MSIxIndex field. */ - if (ioc->msix96_vector) + if (ioc->combined_reply_queue) writel(reply_q->reply_post_host_index | ((msix_index & 7) << MPI2_RPHI_MSIX_INDEX_SHIFT), ioc->replyPostRegisterIndex[msix_index/8]); @@ -1959,7 +1959,7 @@ _base_enable_msix(struct MPT3SAS_ADAPTER *ioc) { struct msix_entry *entries, *a; int r; - int i; + int i, local_max_msix_vectors; u8 try_msix = 0; if (msix_disable == -1 || msix_disable == 0) @@ -1979,13 +1979,15 @@ _base_enable_msix(struct MPT3SAS_ADAPTER *ioc) ioc->cpu_count, max_msix_vectors); if (!ioc->rdpq_array_enable && max_msix_vectors == -1) - max_msix_vectors = 8; + local_max_msix_vectors = 8; + else + local_max_msix_vectors = max_msix_vectors; - if (max_msix_vectors > 0) { - ioc->reply_queue_count = min_t(int, max_msix_vectors, + if (local_max_msix_vectors > 0) { + ioc->reply_queue_count = min_t(int, local_max_msix_vectors, ioc->reply_queue_count); ioc->msix_vector_count = ioc->reply_queue_count; - } else if (max_msix_vectors == 0) + } else if (local_max_msix_vectors == 0) goto try_ioapic; if (ioc->msix_vector_count < ioc->cpu_count) @@ -2050,7 +2052,7 @@ mpt3sas_base_unmap_resources(struct MPT3SAS_ADAPTER *ioc) _base_free_irq(ioc); _base_disable_msix(ioc); - if (ioc->msix96_vector) { + if (ioc->combined_reply_queue) { kfree(ioc->replyPostRegisterIndex); ioc->replyPostRegisterIndex = NULL; } @@ -2160,7 +2162,7 @@ mpt3sas_base_map_resources(struct MPT3SAS_ADAPTER *ioc) /* Use the Combined reply queue feature only for SAS3 C0 & higher * revision HBAs and also only when reply queue count is greater than 8 */ - if (ioc->msix96_vector && ioc->reply_queue_count > 8) { + if (ioc->combined_reply_queue && ioc->reply_queue_count > 8) { /* Determine the Supplemental Reply Post Host Index Registers * Addresse. Supplemental Reply Post Host Index Registers * starts at offset MPI25_SUP_REPLY_POST_HOST_INDEX_OFFSET and @@ -2168,7 +2170,7 @@ mpt3sas_base_map_resources(struct MPT3SAS_ADAPTER *ioc) * MPT3_SUP_REPLY_POST_HOST_INDEX_REG_OFFSET from previous one. */ ioc->replyPostRegisterIndex = kcalloc( - MPT3_SUP_REPLY_POST_HOST_INDEX_REG_COUNT, + ioc->combined_reply_index_count, sizeof(resource_size_t *), GFP_KERNEL); if (!ioc->replyPostRegisterIndex) { dfailprintk(ioc, printk(MPT3SAS_FMT @@ -2178,14 +2180,14 @@ mpt3sas_base_map_resources(struct MPT3SAS_ADAPTER *ioc) goto out_fail; } - for (i = 0; i < MPT3_SUP_REPLY_POST_HOST_INDEX_REG_COUNT; i++) { + for (i = 0; i < ioc->combined_reply_index_count; i++) { ioc->replyPostRegisterIndex[i] = (resource_size_t *) ((u8 *)&ioc->chip->Doorbell + MPI25_SUP_REPLY_POST_HOST_INDEX_OFFSET + (i * MPT3_SUP_REPLY_POST_HOST_INDEX_REG_OFFSET)); } } else - ioc->msix96_vector = 0; + ioc->combined_reply_queue = 0; if (ioc->is_warpdrive) { ioc->reply_post_host_index[0] = (resource_size_t __iomem *) @@ -2462,15 +2464,15 @@ _base_writeq(__u64 b, volatile void __iomem *addr, spinlock_t *writeq_lock) #endif /** - * mpt3sas_base_put_smid_scsi_io - send SCSI_IO request to firmware + * _base_put_smid_scsi_io - send SCSI_IO request to firmware * @ioc: per adapter object * @smid: system request message index * @handle: device handle * * Return nothing. */ -void -mpt3sas_base_put_smid_scsi_io(struct MPT3SAS_ADAPTER *ioc, u16 smid, u16 handle) +static void +_base_put_smid_scsi_io(struct MPT3SAS_ADAPTER *ioc, u16 smid, u16 handle) { Mpi2RequestDescriptorUnion_t descriptor; u64 *request = (u64 *)&descriptor; @@ -2486,15 +2488,15 @@ mpt3sas_base_put_smid_scsi_io(struct MPT3SAS_ADAPTER *ioc, u16 smid, u16 handle) } /** - * mpt3sas_base_put_smid_fast_path - send fast path request to firmware + * _base_put_smid_fast_path - send fast path request to firmware * @ioc: per adapter object * @smid: system request message index * @handle: device handle * * Return nothing. */ -void -mpt3sas_base_put_smid_fast_path(struct MPT3SAS_ADAPTER *ioc, u16 smid, +static void +_base_put_smid_fast_path(struct MPT3SAS_ADAPTER *ioc, u16 smid, u16 handle) { Mpi2RequestDescriptorUnion_t descriptor; @@ -2511,14 +2513,14 @@ mpt3sas_base_put_smid_fast_path(struct MPT3SAS_ADAPTER *ioc, u16 smid, } /** - * mpt3sas_base_put_smid_hi_priority - send Task Managment request to firmware + * _base_put_smid_hi_priority - send Task Management request to firmware * @ioc: per adapter object * @smid: system request message index * @msix_task: msix_task will be same as msix of IO incase of task abort else 0. * Return nothing. */ -void -mpt3sas_base_put_smid_hi_priority(struct MPT3SAS_ADAPTER *ioc, u16 smid, +static void +_base_put_smid_hi_priority(struct MPT3SAS_ADAPTER *ioc, u16 smid, u16 msix_task) { Mpi2RequestDescriptorUnion_t descriptor; @@ -2535,14 +2537,14 @@ mpt3sas_base_put_smid_hi_priority(struct MPT3SAS_ADAPTER *ioc, u16 smid, } /** - * mpt3sas_base_put_smid_default - Default, primarily used for config pages + * _base_put_smid_default - Default, primarily used for config pages * @ioc: per adapter object * @smid: system request message index * * Return nothing. */ -void -mpt3sas_base_put_smid_default(struct MPT3SAS_ADAPTER *ioc, u16 smid) +static void +_base_put_smid_default(struct MPT3SAS_ADAPTER *ioc, u16 smid) { Mpi2RequestDescriptorUnion_t descriptor; u64 *request = (u64 *)&descriptor; @@ -2557,6 +2559,95 @@ mpt3sas_base_put_smid_default(struct MPT3SAS_ADAPTER *ioc, u16 smid) } /** +* _base_put_smid_scsi_io_atomic - send SCSI_IO request to firmware using +* Atomic Request Descriptor +* @ioc: per adapter object +* @smid: system request message index +* @handle: device handle, unused in this function, for function type match +* +* Return nothing. +*/ +static void +_base_put_smid_scsi_io_atomic(struct MPT3SAS_ADAPTER *ioc, u16 smid, + u16 handle) +{ + Mpi26AtomicRequestDescriptor_t descriptor; + u32 *request = (u32 *)&descriptor; + + descriptor.RequestFlags = MPI2_REQ_DESCRIPT_FLAGS_SCSI_IO; + descriptor.MSIxIndex = _base_get_msix_index(ioc); + descriptor.SMID = cpu_to_le16(smid); + + writel(cpu_to_le32(*request), &ioc->chip->AtomicRequestDescriptorPost); +} + +/** + * _base_put_smid_fast_path_atomic - send fast path request to firmware + * using Atomic Request Descriptor + * @ioc: per adapter object + * @smid: system request message index + * @handle: device handle, unused in this function, for function type match + * Return nothing + */ +static void +_base_put_smid_fast_path_atomic(struct MPT3SAS_ADAPTER *ioc, u16 smid, + u16 handle) +{ + Mpi26AtomicRequestDescriptor_t descriptor; + u32 *request = (u32 *)&descriptor; + + descriptor.RequestFlags = MPI25_REQ_DESCRIPT_FLAGS_FAST_PATH_SCSI_IO; + descriptor.MSIxIndex = _base_get_msix_index(ioc); + descriptor.SMID = cpu_to_le16(smid); + + writel(cpu_to_le32(*request), &ioc->chip->AtomicRequestDescriptorPost); +} + +/** + * _base_put_smid_hi_priority_atomic - send Task Management request to + * firmware using Atomic Request Descriptor + * @ioc: per adapter object + * @smid: system request message index + * @msix_task: msix_task will be same as msix of IO incase of task abort else 0 + * + * Return nothing. + */ +static void +_base_put_smid_hi_priority_atomic(struct MPT3SAS_ADAPTER *ioc, u16 smid, + u16 msix_task) +{ + Mpi26AtomicRequestDescriptor_t descriptor; + u32 *request = (u32 *)&descriptor; + + descriptor.RequestFlags = MPI2_REQ_DESCRIPT_FLAGS_HIGH_PRIORITY; + descriptor.MSIxIndex = msix_task; + descriptor.SMID = cpu_to_le16(smid); + + writel(cpu_to_le32(*request), &ioc->chip->AtomicRequestDescriptorPost); +} + +/** + * _base_put_smid_default - Default, primarily used for config pages + * use Atomic Request Descriptor + * @ioc: per adapter object + * @smid: system request message index + * + * Return nothing. + */ +static void +_base_put_smid_default_atomic(struct MPT3SAS_ADAPTER *ioc, u16 smid) +{ + Mpi26AtomicRequestDescriptor_t descriptor; + u32 *request = (u32 *)&descriptor; + + descriptor.RequestFlags = MPI2_REQ_DESCRIPT_FLAGS_DEFAULT_TYPE; + descriptor.MSIxIndex = _base_get_msix_index(ioc); + descriptor.SMID = cpu_to_le16(smid); + + writel(cpu_to_le32(*request), &ioc->chip->AtomicRequestDescriptorPost); +} + +/** * _base_display_OEMs_branding - Display branding string * @ioc: per adapter object * @@ -4070,7 +4161,7 @@ mpt3sas_base_sas_iounit_control(struct MPT3SAS_ADAPTER *ioc, mpi_request->Operation == MPI2_SAS_OP_PHY_LINK_RESET) ioc->ioc_link_reset_in_progress = 1; init_completion(&ioc->base_cmds.done); - mpt3sas_base_put_smid_default(ioc, smid); + ioc->put_smid_default(ioc, smid); wait_for_completion_timeout(&ioc->base_cmds.done, msecs_to_jiffies(10000)); if ((mpi_request->Operation == MPI2_SAS_OP_PHY_HARD_RESET || @@ -4170,7 +4261,7 @@ mpt3sas_base_scsi_enclosure_processor(struct MPT3SAS_ADAPTER *ioc, ioc->base_cmds.smid = smid; memcpy(request, mpi_request, sizeof(Mpi2SepReply_t)); init_completion(&ioc->base_cmds.done); - mpt3sas_base_put_smid_default(ioc, smid); + ioc->put_smid_default(ioc, smid); wait_for_completion_timeout(&ioc->base_cmds.done, msecs_to_jiffies(10000)); if (!(ioc->base_cmds.status & MPT3_CMD_COMPLETE)) { @@ -4355,6 +4446,8 @@ _base_get_ioc_facts(struct MPT3SAS_ADAPTER *ioc) if ((facts->IOCCapabilities & MPI2_IOCFACTS_CAPABILITY_RDPQ_ARRAY_CAPABLE)) ioc->rdpq_array_capable = 1; + if (facts->IOCCapabilities & MPI26_IOCFACTS_CAPABILITY_ATOMIC_REQ) + ioc->atomic_desc_capable = 1; facts->FWVersion.Word = le32_to_cpu(mpi_reply.FWVersion.Word); facts->IOCRequestFrameSize = le16_to_cpu(mpi_reply.IOCRequestFrameSize); @@ -4582,7 +4675,7 @@ _base_send_port_enable(struct MPT3SAS_ADAPTER *ioc) mpi_request->Function = MPI2_FUNCTION_PORT_ENABLE; init_completion(&ioc->port_enable_cmds.done); - mpt3sas_base_put_smid_default(ioc, smid); + ioc->put_smid_default(ioc, smid); wait_for_completion_timeout(&ioc->port_enable_cmds.done, 300*HZ); if (!(ioc->port_enable_cmds.status & MPT3_CMD_COMPLETE)) { pr_err(MPT3SAS_FMT "%s: timeout\n", @@ -4645,7 +4738,7 @@ mpt3sas_port_enable(struct MPT3SAS_ADAPTER *ioc) memset(mpi_request, 0, sizeof(Mpi2PortEnableRequest_t)); mpi_request->Function = MPI2_FUNCTION_PORT_ENABLE; - mpt3sas_base_put_smid_default(ioc, smid); + ioc->put_smid_default(ioc, smid); return 0; } @@ -4764,7 +4857,7 @@ _base_event_notification(struct MPT3SAS_ADAPTER *ioc) mpi_request->EventMasks[i] = cpu_to_le32(ioc->event_masks[i]); init_completion(&ioc->base_cmds.done); - mpt3sas_base_put_smid_default(ioc, smid); + ioc->put_smid_default(ioc, smid); wait_for_completion_timeout(&ioc->base_cmds.done, 30*HZ); if (!(ioc->base_cmds.status & MPT3_CMD_COMPLETE)) { pr_err(MPT3SAS_FMT "%s: timeout\n", @@ -5138,7 +5231,7 @@ _base_make_ioc_operational(struct MPT3SAS_ADAPTER *ioc) /* initialize reply post host index */ list_for_each_entry(reply_q, &ioc->reply_queue_list, list) { - if (ioc->msix96_vector) + if (ioc->combined_reply_queue) writel((reply_q->msix_index & 7)<< MPI2_RPHI_MSIX_INDEX_SHIFT, ioc->replyPostRegisterIndex[reply_q->msix_index/8]); @@ -5280,9 +5373,23 @@ mpt3sas_base_attach(struct MPT3SAS_ADAPTER *ioc) ioc->build_sg = &_base_build_sg_ieee; ioc->build_zero_len_sge = &_base_build_zero_len_sge_ieee; ioc->sge_size_ieee = sizeof(Mpi2IeeeSgeSimple64_t); + break; } + if (ioc->atomic_desc_capable) { + ioc->put_smid_default = &_base_put_smid_default_atomic; + ioc->put_smid_scsi_io = &_base_put_smid_scsi_io_atomic; + ioc->put_smid_fast_path = &_base_put_smid_fast_path_atomic; + ioc->put_smid_hi_priority = &_base_put_smid_hi_priority_atomic; + } else { + ioc->put_smid_default = &_base_put_smid_default; + ioc->put_smid_scsi_io = &_base_put_smid_scsi_io; + ioc->put_smid_fast_path = &_base_put_smid_fast_path; + ioc->put_smid_hi_priority = &_base_put_smid_hi_priority; + } + + /* * These function pointers for other requests that don't * the require IEEE scatter gather elements. @@ -5332,6 +5439,21 @@ mpt3sas_base_attach(struct MPT3SAS_ADAPTER *ioc) goto out_free_resources; } + /* allocate memory for pending OS device add list */ + ioc->pend_os_device_add_sz = (ioc->facts.MaxDevHandle / 8); + if (ioc->facts.MaxDevHandle % 8) + ioc->pend_os_device_add_sz++; + ioc->pend_os_device_add = kzalloc(ioc->pend_os_device_add_sz, + GFP_KERNEL); + if (!ioc->pend_os_device_add) + goto out_free_resources; + + ioc->device_remove_in_progress_sz = ioc->pend_os_device_add_sz; + ioc->device_remove_in_progress = + kzalloc(ioc->device_remove_in_progress_sz, GFP_KERNEL); + if (!ioc->device_remove_in_progress) + goto out_free_resources; + ioc->fwfault_debug = mpt3sas_fwfault_debug; /* base internal command bits */ @@ -5414,6 +5536,8 @@ mpt3sas_base_attach(struct MPT3SAS_ADAPTER *ioc) kfree(ioc->reply_post_host_index); kfree(ioc->pd_handles); kfree(ioc->blocking_handles); + kfree(ioc->device_remove_in_progress); + kfree(ioc->pend_os_device_add); kfree(ioc->tm_cmds.reply); kfree(ioc->transport_cmds.reply); kfree(ioc->scsih_cmds.reply); @@ -5455,6 +5579,8 @@ mpt3sas_base_detach(struct MPT3SAS_ADAPTER *ioc) kfree(ioc->reply_post_host_index); kfree(ioc->pd_handles); kfree(ioc->blocking_handles); + kfree(ioc->device_remove_in_progress); + kfree(ioc->pend_os_device_add); kfree(ioc->pfacts); kfree(ioc->ctl_cmds.reply); kfree(ioc->ctl_cmds.sense); diff --git a/drivers/scsi/mpt3sas/mpt3sas_base.h b/drivers/scsi/mpt3sas/mpt3sas_base.h index 3e71bc1..8de0eda 100644 --- a/drivers/scsi/mpt3sas/mpt3sas_base.h +++ b/drivers/scsi/mpt3sas/mpt3sas_base.h @@ -73,9 +73,9 @@ #define MPT3SAS_DRIVER_NAME "mpt3sas" #define MPT3SAS_AUTHOR "Avago Technologies <MPT-FusionLinux.pdl@avagotech.com>" #define MPT3SAS_DESCRIPTION "LSI MPT Fusion SAS 3.0 Device Driver" -#define MPT3SAS_DRIVER_VERSION "13.100.00.00" -#define MPT3SAS_MAJOR_VERSION 13 -#define MPT3SAS_MINOR_VERSION 100 +#define MPT3SAS_DRIVER_VERSION "14.101.00.00" +#define MPT3SAS_MAJOR_VERSION 14 +#define MPT3SAS_MINOR_VERSION 101 #define MPT3SAS_BUILD_VERSION 0 #define MPT3SAS_RELEASE_VERSION 00 @@ -300,8 +300,9 @@ * There are twelve Supplemental Reply Post Host Index Registers * and each register is at offset 0x10 bytes from the previous one. */ -#define MPT3_SUP_REPLY_POST_HOST_INDEX_REG_COUNT 12 -#define MPT3_SUP_REPLY_POST_HOST_INDEX_REG_OFFSET (0x10) +#define MPT3_SUP_REPLY_POST_HOST_INDEX_REG_COUNT_G3 12 +#define MPT3_SUP_REPLY_POST_HOST_INDEX_REG_COUNT_G35 16 +#define MPT3_SUP_REPLY_POST_HOST_INDEX_REG_OFFSET (0x10) /* OEM Identifiers */ #define MFG10_OEM_ID_INVALID (0x00000000) @@ -375,7 +376,6 @@ struct MPT3SAS_TARGET { * per device private data */ #define MPT_DEVICE_FLAGS_INIT 0x01 -#define MPT_DEVICE_TLR_ON 0x02 #define MFG_PAGE10_HIDE_SSDS_MASK (0x00000003) #define MFG_PAGE10_HIDE_ALL_DISKS (0x00) @@ -736,7 +736,10 @@ typedef void (*MPT_BUILD_SG)(struct MPT3SAS_ADAPTER *ioc, void *psge, typedef void (*MPT_BUILD_ZERO_LEN_SGE)(struct MPT3SAS_ADAPTER *ioc, void *paddr); - +/* To support atomic and non atomic descriptors*/ +typedef void (*PUT_SMID_IO_FP_HIP) (struct MPT3SAS_ADAPTER *ioc, u16 smid, + u16 funcdep); +typedef void (*PUT_SMID_DEFAULT) (struct MPT3SAS_ADAPTER *ioc, u16 smid); /* IOC Facts and Port Facts converted from little endian to cpu */ union mpi3_version_union { @@ -1079,6 +1082,9 @@ struct MPT3SAS_ADAPTER { void *pd_handles; u16 pd_handles_sz; + void *pend_os_device_add; + u16 pend_os_device_add_sz; + /* config page */ u16 config_page_sz; void *config_page; @@ -1156,7 +1162,8 @@ struct MPT3SAS_ADAPTER { u8 reply_queue_count; struct list_head reply_queue_list; - u8 msix96_vector; + u8 combined_reply_queue; + u8 combined_reply_index_count; /* reply post register index */ resource_size_t **replyPostRegisterIndex; @@ -1187,6 +1194,15 @@ struct MPT3SAS_ADAPTER { struct SL_WH_EVENT_TRIGGERS_T diag_trigger_event; struct SL_WH_SCSI_TRIGGERS_T diag_trigger_scsi; struct SL_WH_MPI_TRIGGERS_T diag_trigger_mpi; + void *device_remove_in_progress; + u16 device_remove_in_progress_sz; + u8 is_gen35_ioc; + u8 atomic_desc_capable; + PUT_SMID_IO_FP_HIP put_smid_scsi_io; + PUT_SMID_IO_FP_HIP put_smid_fast_path; + PUT_SMID_IO_FP_HIP put_smid_hi_priority; + PUT_SMID_DEFAULT put_smid_default; + }; typedef u8 (*MPT_CALLBACK)(struct MPT3SAS_ADAPTER *ioc, u16 smid, u8 msix_index, @@ -1232,13 +1248,6 @@ u16 mpt3sas_base_get_smid_scsiio(struct MPT3SAS_ADAPTER *ioc, u8 cb_idx, u16 mpt3sas_base_get_smid(struct MPT3SAS_ADAPTER *ioc, u8 cb_idx); void mpt3sas_base_free_smid(struct MPT3SAS_ADAPTER *ioc, u16 smid); -void mpt3sas_base_put_smid_scsi_io(struct MPT3SAS_ADAPTER *ioc, u16 smid, - u16 handle); -void mpt3sas_base_put_smid_fast_path(struct MPT3SAS_ADAPTER *ioc, u16 smid, - u16 handle); -void mpt3sas_base_put_smid_hi_priority(struct MPT3SAS_ADAPTER *ioc, - u16 smid, u16 msix_task); -void mpt3sas_base_put_smid_default(struct MPT3SAS_ADAPTER *ioc, u16 smid); void mpt3sas_base_initialize_callback_handler(void); u8 mpt3sas_base_register_callback_handler(MPT_CALLBACK cb_func); void mpt3sas_base_release_callback_handler(u8 cb_idx); diff --git a/drivers/scsi/mpt3sas/mpt3sas_config.c b/drivers/scsi/mpt3sas/mpt3sas_config.c index cebfd73..dd62701 100644 --- a/drivers/scsi/mpt3sas/mpt3sas_config.c +++ b/drivers/scsi/mpt3sas/mpt3sas_config.c @@ -384,7 +384,7 @@ _config_request(struct MPT3SAS_ADAPTER *ioc, Mpi2ConfigRequest_t memcpy(config_request, mpi_request, sizeof(Mpi2ConfigRequest_t)); _config_display_some_debug(ioc, smid, "config_request", NULL); init_completion(&ioc->config_cmds.done); - mpt3sas_base_put_smid_default(ioc, smid); + ioc->put_smid_default(ioc, smid); wait_for_completion_timeout(&ioc->config_cmds.done, timeout*HZ); if (!(ioc->config_cmds.status & MPT3_CMD_COMPLETE)) { pr_err(MPT3SAS_FMT "%s: timeout\n", diff --git a/drivers/scsi/mpt3sas/mpt3sas_ctl.c b/drivers/scsi/mpt3sas/mpt3sas_ctl.c index 26cdc12..050bd78 100644 --- a/drivers/scsi/mpt3sas/mpt3sas_ctl.c +++ b/drivers/scsi/mpt3sas/mpt3sas_ctl.c @@ -654,6 +654,7 @@ _ctl_do_mpt_command(struct MPT3SAS_ADAPTER *ioc, struct mpt3_ioctl_command karg, size_t data_in_sz = 0; long ret; u16 wait_state_count; + u16 device_handle = MPT3SAS_INVALID_DEVICE_HANDLE; issue_reset = 0; @@ -738,10 +739,13 @@ _ctl_do_mpt_command(struct MPT3SAS_ADAPTER *ioc, struct mpt3_ioctl_command karg, data_in_sz = karg.data_in_size; if (mpi_request->Function == MPI2_FUNCTION_SCSI_IO_REQUEST || - mpi_request->Function == MPI2_FUNCTION_RAID_SCSI_IO_PASSTHROUGH) { - if (!le16_to_cpu(mpi_request->FunctionDependent1) || - le16_to_cpu(mpi_request->FunctionDependent1) > - ioc->facts.MaxDevHandle) { + mpi_request->Function == MPI2_FUNCTION_RAID_SCSI_IO_PASSTHROUGH || + mpi_request->Function == MPI2_FUNCTION_SCSI_TASK_MGMT || + mpi_request->Function == MPI2_FUNCTION_SATA_PASSTHROUGH) { + + device_handle = le16_to_cpu(mpi_request->FunctionDependent1); + if (!device_handle || (device_handle > + ioc->facts.MaxDevHandle)) { ret = -EINVAL; mpt3sas_base_free_smid(ioc, smid); goto out; @@ -797,14 +801,20 @@ _ctl_do_mpt_command(struct MPT3SAS_ADAPTER *ioc, struct mpt3_ioctl_command karg, scsiio_request->SenseBufferLowAddress = mpt3sas_base_get_sense_buffer_dma(ioc, smid); memset(ioc->ctl_cmds.sense, 0, SCSI_SENSE_BUFFERSIZE); + if (test_bit(device_handle, ioc->device_remove_in_progress)) { + dtmprintk(ioc, pr_info(MPT3SAS_FMT + "handle(0x%04x) :ioctl failed due to device removal in progress\n", + ioc->name, device_handle)); + mpt3sas_base_free_smid(ioc, smid); + ret = -EINVAL; + goto out; + } ioc->build_sg(ioc, psge, data_out_dma, data_out_sz, data_in_dma, data_in_sz); - if (mpi_request->Function == MPI2_FUNCTION_SCSI_IO_REQUEST) - mpt3sas_base_put_smid_scsi_io(ioc, smid, - le16_to_cpu(mpi_request->FunctionDependent1)); + ioc->put_smid_scsi_io(ioc, smid, device_handle); else - mpt3sas_base_put_smid_default(ioc, smid); + ioc->put_smid_default(ioc, smid); break; } case MPI2_FUNCTION_SCSI_TASK_MGMT: @@ -827,11 +837,19 @@ _ctl_do_mpt_command(struct MPT3SAS_ADAPTER *ioc, struct mpt3_ioctl_command karg, } } + if (test_bit(device_handle, ioc->device_remove_in_progress)) { + dtmprintk(ioc, pr_info(MPT3SAS_FMT + "handle(0x%04x) :ioctl failed due to device removal in progress\n", + ioc->name, device_handle)); + mpt3sas_base_free_smid(ioc, smid); + ret = -EINVAL; + goto out; + } mpt3sas_scsih_set_tm_flag(ioc, le16_to_cpu( tm_request->DevHandle)); ioc->build_sg_mpi(ioc, psge, data_out_dma, data_out_sz, data_in_dma, data_in_sz); - mpt3sas_base_put_smid_hi_priority(ioc, smid, 0); + ioc->put_smid_hi_priority(ioc, smid, 0); break; } case MPI2_FUNCTION_SMP_PASSTHROUGH: @@ -862,16 +880,30 @@ _ctl_do_mpt_command(struct MPT3SAS_ADAPTER *ioc, struct mpt3_ioctl_command karg, } ioc->build_sg(ioc, psge, data_out_dma, data_out_sz, data_in_dma, data_in_sz); - mpt3sas_base_put_smid_default(ioc, smid); + ioc->put_smid_default(ioc, smid); break; } case MPI2_FUNCTION_SATA_PASSTHROUGH: + { + if (test_bit(device_handle, ioc->device_remove_in_progress)) { + dtmprintk(ioc, pr_info(MPT3SAS_FMT + "handle(0x%04x) :ioctl failed due to device removal in progress\n", + ioc->name, device_handle)); + mpt3sas_base_free_smid(ioc, smid); + ret = -EINVAL; + goto out; + } + ioc->build_sg(ioc, psge, data_out_dma, data_out_sz, data_in_dma, + data_in_sz); + ioc->put_smid_default(ioc, smid); + break; + } case MPI2_FUNCTION_FW_DOWNLOAD: case MPI2_FUNCTION_FW_UPLOAD: { ioc->build_sg(ioc, psge, data_out_dma, data_out_sz, data_in_dma, data_in_sz); - mpt3sas_base_put_smid_default(ioc, smid); + ioc->put_smid_default(ioc, smid); break; } case MPI2_FUNCTION_TOOLBOX: @@ -886,7 +918,7 @@ _ctl_do_mpt_command(struct MPT3SAS_ADAPTER *ioc, struct mpt3_ioctl_command karg, ioc->build_sg_mpi(ioc, psge, data_out_dma, data_out_sz, data_in_dma, data_in_sz); } - mpt3sas_base_put_smid_default(ioc, smid); + ioc->put_smid_default(ioc, smid); break; } case MPI2_FUNCTION_SAS_IO_UNIT_CONTROL: @@ -905,7 +937,7 @@ _ctl_do_mpt_command(struct MPT3SAS_ADAPTER *ioc, struct mpt3_ioctl_command karg, default: ioc->build_sg_mpi(ioc, psge, data_out_dma, data_out_sz, data_in_dma, data_in_sz); - mpt3sas_base_put_smid_default(ioc, smid); + ioc->put_smid_default(ioc, smid); break; } @@ -1064,7 +1096,10 @@ _ctl_getiocinfo(struct MPT3SAS_ADAPTER *ioc, void __user *arg) break; case MPI25_VERSION: case MPI26_VERSION: - karg.adapter_type = MPT3_IOCTL_INTERFACE_SAS3; + if (ioc->is_gen35_ioc) + karg.adapter_type = MPT3_IOCTL_INTERFACE_SAS35; + else + karg.adapter_type = MPT3_IOCTL_INTERFACE_SAS3; strcat(karg.driver_version, MPT3SAS_DRIVER_VERSION); break; } @@ -1491,7 +1526,7 @@ _ctl_diag_register_2(struct MPT3SAS_ADAPTER *ioc, cpu_to_le32(ioc->product_specific[buffer_type][i]); init_completion(&ioc->ctl_cmds.done); - mpt3sas_base_put_smid_default(ioc, smid); + ioc->put_smid_default(ioc, smid); wait_for_completion_timeout(&ioc->ctl_cmds.done, MPT3_IOCTL_DEFAULT_TIMEOUT*HZ); @@ -1838,7 +1873,7 @@ mpt3sas_send_diag_release(struct MPT3SAS_ADAPTER *ioc, u8 buffer_type, mpi_request->VP_ID = 0; init_completion(&ioc->ctl_cmds.done); - mpt3sas_base_put_smid_default(ioc, smid); + ioc->put_smid_default(ioc, smid); wait_for_completion_timeout(&ioc->ctl_cmds.done, MPT3_IOCTL_DEFAULT_TIMEOUT*HZ); @@ -2105,7 +2140,7 @@ _ctl_diag_read_buffer(struct MPT3SAS_ADAPTER *ioc, void __user *arg) mpi_request->VP_ID = 0; init_completion(&ioc->ctl_cmds.done); - mpt3sas_base_put_smid_default(ioc, smid); + ioc->put_smid_default(ioc, smid); wait_for_completion_timeout(&ioc->ctl_cmds.done, MPT3_IOCTL_DEFAULT_TIMEOUT*HZ); diff --git a/drivers/scsi/mpt3sas/mpt3sas_ctl.h b/drivers/scsi/mpt3sas/mpt3sas_ctl.h index 8940835..f3e17a8 100644 --- a/drivers/scsi/mpt3sas/mpt3sas_ctl.h +++ b/drivers/scsi/mpt3sas/mpt3sas_ctl.h @@ -143,6 +143,7 @@ struct mpt3_ioctl_pci_info { #define MPT2_IOCTL_INTERFACE_SAS2 (0x04) #define MPT2_IOCTL_INTERFACE_SAS2_SSS6200 (0x05) #define MPT3_IOCTL_INTERFACE_SAS3 (0x06) +#define MPT3_IOCTL_INTERFACE_SAS35 (0x07) #define MPT2_IOCTL_VERSION_LENGTH (32) /** diff --git a/drivers/scsi/mpt3sas/mpt3sas_scsih.c b/drivers/scsi/mpt3sas/mpt3sas_scsih.c index 1c4744e..5c8f752 100644 --- a/drivers/scsi/mpt3sas/mpt3sas_scsih.c +++ b/drivers/scsi/mpt3sas/mpt3sas_scsih.c @@ -423,7 +423,7 @@ _scsih_get_sas_address(struct MPT3SAS_ADAPTER *ioc, u16 handle, return 0; } - /* we hit this becuase the given parent handle doesn't exist */ + /* we hit this because the given parent handle doesn't exist */ if (ioc_status == MPI2_IOCSTATUS_CONFIG_INVALID_PAGE) return -ENXIO; @@ -788,6 +788,11 @@ _scsih_sas_device_add(struct MPT3SAS_ADAPTER *ioc, list_add_tail(&sas_device->list, &ioc->sas_device_list); spin_unlock_irqrestore(&ioc->sas_device_lock, flags); + if (ioc->hide_drives) { + clear_bit(sas_device->handle, ioc->pend_os_device_add); + return; + } + if (!mpt3sas_transport_port_add(ioc, sas_device->handle, sas_device->sas_address_parent)) { _scsih_sas_device_remove(ioc, sas_device); @@ -803,7 +808,8 @@ _scsih_sas_device_add(struct MPT3SAS_ADAPTER *ioc, sas_device->sas_address_parent); _scsih_sas_device_remove(ioc, sas_device); } - } + } else + clear_bit(sas_device->handle, ioc->pend_os_device_add); } /** @@ -1517,7 +1523,7 @@ _scsih_display_sata_capabilities(struct MPT3SAS_ADAPTER *ioc, /* * raid transport support - * Enabled for SLES11 and newer, in older kernels the driver will panic when - * unloading the driver followed by a load - I beleive that the subroutine + * unloading the driver followed by a load - I believe that the subroutine * raid_class_release() is not cleaning up properly. */ @@ -2279,7 +2285,7 @@ mpt3sas_scsih_issue_tm(struct MPT3SAS_ADAPTER *ioc, u16 handle, uint channel, msix_task = scsi_lookup->msix_io; else msix_task = 0; - mpt3sas_base_put_smid_hi_priority(ioc, smid, msix_task); + ioc->put_smid_hi_priority(ioc, smid, msix_task); wait_for_completion_timeout(&ioc->tm_cmds.done, timeout*HZ); if (!(ioc->tm_cmds.status & MPT3_CMD_COMPLETE)) { pr_err(MPT3SAS_FMT "%s: timeout\n", @@ -2837,7 +2843,7 @@ _scsih_internal_device_block(struct scsi_device *sdev, if (r == -EINVAL) sdev_printk(KERN_WARNING, sdev, "device_block failed with return(%d) for handle(0x%04x)\n", - sas_device_priv_data->sas_target->handle, r); + r, sas_device_priv_data->sas_target->handle); } /** @@ -2867,20 +2873,20 @@ _scsih_internal_device_unblock(struct scsi_device *sdev, sdev_printk(KERN_WARNING, sdev, "device_unblock failed with return(%d) for handle(0x%04x) " "performing a block followed by an unblock\n", - sas_device_priv_data->sas_target->handle, r); + r, sas_device_priv_data->sas_target->handle); sas_device_priv_data->block = 1; r = scsi_internal_device_block(sdev); if (r) sdev_printk(KERN_WARNING, sdev, "retried device_block " "failed with return(%d) for handle(0x%04x)\n", - sas_device_priv_data->sas_target->handle, r); + r, sas_device_priv_data->sas_target->handle); sas_device_priv_data->block = 0; r = scsi_internal_device_unblock(sdev, SDEV_RUNNING); if (r) sdev_printk(KERN_WARNING, sdev, "retried device_unblock" " failed with return(%d) for handle(0x%04x)\n", - sas_device_priv_data->sas_target->handle, r); + r, sas_device_priv_data->sas_target->handle); } } @@ -2942,7 +2948,7 @@ _scsih_ublock_io_device(struct MPT3SAS_ADAPTER *ioc, u64 sas_address) * @ioc: per adapter object * @handle: device handle * - * During device pull we need to appropiately set the sdev state. + * During device pull we need to appropriately set the sdev state. */ static void _scsih_block_io_all_device(struct MPT3SAS_ADAPTER *ioc) @@ -2971,7 +2977,7 @@ _scsih_block_io_all_device(struct MPT3SAS_ADAPTER *ioc) * @ioc: per adapter object * @handle: device handle * - * During device pull we need to appropiately set the sdev state. + * During device pull we need to appropriately set the sdev state. */ static void _scsih_block_io_device(struct MPT3SAS_ADAPTER *ioc, u16 handle) @@ -3138,6 +3144,8 @@ _scsih_tm_tr_send(struct MPT3SAS_ADAPTER *ioc, u16 handle) if (test_bit(handle, ioc->pd_handles)) return; + clear_bit(handle, ioc->pend_os_device_add); + spin_lock_irqsave(&ioc->sas_device_lock, flags); sas_device = __mpt3sas_get_sdev_by_handle(ioc, handle); if (sas_device && sas_device->starget && @@ -3192,7 +3200,8 @@ _scsih_tm_tr_send(struct MPT3SAS_ADAPTER *ioc, u16 handle) mpi_request->Function = MPI2_FUNCTION_SCSI_TASK_MGMT; mpi_request->DevHandle = cpu_to_le16(handle); mpi_request->TaskType = MPI2_SCSITASKMGMT_TASKTYPE_TARGET_RESET; - mpt3sas_base_put_smid_hi_priority(ioc, smid, 0); + set_bit(handle, ioc->device_remove_in_progress); + ioc->put_smid_hi_priority(ioc, smid, 0); mpt3sas_trigger_master(ioc, MASTER_TRIGGER_DEVICE_REMOVAL); out: @@ -3291,7 +3300,7 @@ _scsih_tm_tr_complete(struct MPT3SAS_ADAPTER *ioc, u16 smid, u8 msix_index, mpi_request->Function = MPI2_FUNCTION_SAS_IO_UNIT_CONTROL; mpi_request->Operation = MPI2_SAS_OP_REMOVE_DEVICE; mpi_request->DevHandle = mpi_request_tm->DevHandle; - mpt3sas_base_put_smid_default(ioc, smid_sas_ctrl); + ioc->put_smid_default(ioc, smid_sas_ctrl); return _scsih_check_for_pending_tm(ioc, smid); } @@ -3326,6 +3335,11 @@ _scsih_sas_control_complete(struct MPT3SAS_ADAPTER *ioc, u16 smid, ioc->name, le16_to_cpu(mpi_reply->DevHandle), smid, le16_to_cpu(mpi_reply->IOCStatus), le32_to_cpu(mpi_reply->IOCLogInfo))); + if (le16_to_cpu(mpi_reply->IOCStatus) == + MPI2_IOCSTATUS_SUCCESS) { + clear_bit(le16_to_cpu(mpi_reply->DevHandle), + ioc->device_remove_in_progress); + } } else { pr_err(MPT3SAS_FMT "mpi_reply not valid at %s:%d/%s()!\n", ioc->name, __FILE__, __LINE__, __func__); @@ -3381,7 +3395,7 @@ _scsih_tm_tr_volume_send(struct MPT3SAS_ADAPTER *ioc, u16 handle) mpi_request->Function = MPI2_FUNCTION_SCSI_TASK_MGMT; mpi_request->DevHandle = cpu_to_le16(handle); mpi_request->TaskType = MPI2_SCSITASKMGMT_TASKTYPE_TARGET_RESET; - mpt3sas_base_put_smid_hi_priority(ioc, smid, 0); + ioc->put_smid_hi_priority(ioc, smid, 0); } /** @@ -3473,7 +3487,7 @@ _scsih_issue_delayed_event_ack(struct MPT3SAS_ADAPTER *ioc, u16 smid, u16 event, ack_request->EventContext = event_context; ack_request->VF_ID = 0; /* TODO */ ack_request->VP_ID = 0; - mpt3sas_base_put_smid_default(ioc, smid); + ioc->put_smid_default(ioc, smid); } /** @@ -3530,7 +3544,7 @@ _scsih_issue_delayed_sas_io_unit_ctrl(struct MPT3SAS_ADAPTER *ioc, mpi_request->Function = MPI2_FUNCTION_SAS_IO_UNIT_CONTROL; mpi_request->Operation = MPI2_SAS_OP_REMOVE_DEVICE; mpi_request->DevHandle = handle; - mpt3sas_base_put_smid_default(ioc, smid); + ioc->put_smid_default(ioc, smid); } /** @@ -3930,7 +3944,7 @@ _scsih_flush_running_cmds(struct MPT3SAS_ADAPTER *ioc) * _scsih_setup_eedp - setup MPI request for EEDP transfer * @ioc: per adapter object * @scmd: pointer to scsi command object - * @mpi_request: pointer to the SCSI_IO reqest message frame + * @mpi_request: pointer to the SCSI_IO request message frame * * Supporting protection 1 and 3. * @@ -3983,6 +3997,9 @@ _scsih_setup_eedp(struct MPT3SAS_ADAPTER *ioc, struct scsi_cmnd *scmd, mpi_request_3v->EEDPBlockSize = cpu_to_le16(scmd->device->sector_size); + + if (ioc->is_gen35_ioc) + eedp_flags |= MPI25_SCSIIO_EEDPFLAGS_APPTAG_DISABLE_MODE; mpi_request->EEDPFlags = cpu_to_le16(eedp_flags); } @@ -4084,7 +4101,7 @@ scsih_qcmd(struct Scsi_Host *shost, struct scsi_cmnd *scmd) scmd->result = DID_NO_CONNECT << 16; scmd->scsi_done(scmd); return 0; - /* device busy with task managment */ + /* device busy with task management */ } else if (sas_target_priv_data->tm_busy || sas_device_priv_data->block) return SCSI_MLQUEUE_DEVICE_BUSY; @@ -4154,12 +4171,12 @@ scsih_qcmd(struct Scsi_Host *shost, struct scsi_cmnd *scmd) if (sas_target_priv_data->flags & MPT_TARGET_FASTPATH_IO) { mpi_request->IoFlags = cpu_to_le16(scmd->cmd_len | MPI25_SCSIIO_IOFLAGS_FAST_PATH); - mpt3sas_base_put_smid_fast_path(ioc, smid, handle); + ioc->put_smid_fast_path(ioc, smid, handle); } else - mpt3sas_base_put_smid_scsi_io(ioc, smid, + ioc->put_smid_scsi_io(ioc, smid, le16_to_cpu(mpi_request->DevHandle)); } else - mpt3sas_base_put_smid_default(ioc, smid); + ioc->put_smid_default(ioc, smid); return 0; out: @@ -4658,7 +4675,7 @@ _scsih_io_done(struct MPT3SAS_ADAPTER *ioc, u16 smid, u8 msix_index, u32 reply) memcpy(mpi_request->CDB.CDB32, scmd->cmnd, scmd->cmd_len); mpi_request->DevHandle = cpu_to_le16(sas_device_priv_data->sas_target->handle); - mpt3sas_base_put_smid_scsi_io(ioc, smid, + ioc->put_smid_scsi_io(ioc, smid, sas_device_priv_data->sas_target->handle); return 0; } @@ -5383,10 +5400,10 @@ _scsih_check_device(struct MPT3SAS_ADAPTER *ioc, sas_device->handle, handle); sas_target_priv_data->handle = handle; sas_device->handle = handle; - if (sas_device_pg0.Flags & + if (le16_to_cpu(sas_device_pg0.Flags) & MPI2_SAS_DEVICE0_FLAGS_ENCL_LEVEL_VALID) { sas_device->enclosure_level = - le16_to_cpu(sas_device_pg0.EnclosureLevel); + sas_device_pg0.EnclosureLevel; memcpy(sas_device->connector_name, sas_device_pg0.ConnectorName, 4); sas_device->connector_name[4] = '\0'; @@ -5465,6 +5482,7 @@ _scsih_add_device(struct MPT3SAS_ADAPTER *ioc, u16 handle, u8 phy_num, device_info = le32_to_cpu(sas_device_pg0.DeviceInfo); if (!(_scsih_is_end_device(device_info))) return -1; + set_bit(handle, ioc->pend_os_device_add); sas_address = le64_to_cpu(sas_device_pg0.SASAddress); /* check if device is present */ @@ -5483,6 +5501,7 @@ _scsih_add_device(struct MPT3SAS_ADAPTER *ioc, u16 handle, u8 phy_num, sas_device = mpt3sas_get_sdev_by_addr(ioc, sas_address); if (sas_device) { + clear_bit(handle, ioc->pend_os_device_add); sas_device_put(sas_device); return -1; } @@ -5513,9 +5532,10 @@ _scsih_add_device(struct MPT3SAS_ADAPTER *ioc, u16 handle, u8 phy_num, sas_device->fast_path = (le16_to_cpu(sas_device_pg0.Flags) & MPI25_SAS_DEVICE0_FLAGS_FAST_PATH_CAPABLE) ? 1 : 0; - if (sas_device_pg0.Flags & MPI2_SAS_DEVICE0_FLAGS_ENCL_LEVEL_VALID) { + if (le16_to_cpu(sas_device_pg0.Flags) + & MPI2_SAS_DEVICE0_FLAGS_ENCL_LEVEL_VALID) { sas_device->enclosure_level = - le16_to_cpu(sas_device_pg0.EnclosureLevel); + sas_device_pg0.EnclosureLevel; memcpy(sas_device->connector_name, sas_device_pg0.ConnectorName, 4); sas_device->connector_name[4] = '\0'; @@ -5806,6 +5826,9 @@ _scsih_sas_topology_change_event(struct MPT3SAS_ADAPTER *ioc, _scsih_check_device(ioc, sas_address, handle, phy_number, link_rate); + if (!test_bit(handle, ioc->pend_os_device_add)) + break; + case MPI2_EVENT_SAS_TOPO_RC_TARG_ADDED: @@ -6267,7 +6290,7 @@ _scsih_ir_fastpath(struct MPT3SAS_ADAPTER *ioc, u16 handle, u8 phys_disk_num) handle, phys_disk_num)); init_completion(&ioc->scsih_cmds.done); - mpt3sas_base_put_smid_default(ioc, smid); + ioc->put_smid_default(ioc, smid); wait_for_completion_timeout(&ioc->scsih_cmds.done, 10*HZ); if (!(ioc->scsih_cmds.status & MPT3_CMD_COMPLETE)) { @@ -6320,7 +6343,7 @@ _scsih_reprobe_lun(struct scsi_device *sdev, void *no_uld_attach) { sdev->no_uld_attach = no_uld_attach ? 1 : 0; sdev_printk(KERN_INFO, sdev, "%s raid component\n", - sdev->no_uld_attach ? "hidding" : "exposing"); + sdev->no_uld_attach ? "hiding" : "exposing"); WARN_ON(scsi_device_reprobe(sdev)); } @@ -7050,7 +7073,7 @@ Mpi2SasDevicePage0_t *sas_device_pg0) if (sas_device_pg0->Flags & MPI2_SAS_DEVICE0_FLAGS_ENCL_LEVEL_VALID) { sas_device->enclosure_level = - le16_to_cpu(sas_device_pg0->EnclosureLevel); + sas_device_pg0->EnclosureLevel; memcpy(&sas_device->connector_name[0], &sas_device_pg0->ConnectorName[0], 4); } else { @@ -7112,6 +7135,7 @@ _scsih_search_responding_sas_devices(struct MPT3SAS_ADAPTER *ioc) sas_device_pg0.SASAddress = le64_to_cpu(sas_device_pg0.SASAddress); sas_device_pg0.Slot = le16_to_cpu(sas_device_pg0.Slot); + sas_device_pg0.Flags = le16_to_cpu(sas_device_pg0.Flags); _scsih_mark_responding_sas_device(ioc, &sas_device_pg0); } @@ -7723,6 +7747,9 @@ mpt3sas_scsih_reset_handler(struct MPT3SAS_ADAPTER *ioc, int reset_phase) complete(&ioc->tm_cmds.done); } + memset(ioc->pend_os_device_add, 0, ioc->pend_os_device_add_sz); + memset(ioc->device_remove_in_progress, 0, + ioc->device_remove_in_progress_sz); _scsih_fw_event_cleanup_queue(ioc); _scsih_flush_running_cmds(ioc); break; @@ -8113,7 +8140,7 @@ _scsih_ir_shutdown(struct MPT3SAS_ADAPTER *ioc) if (!ioc->hide_ir_msg) pr_info(MPT3SAS_FMT "IR shutdown (sending)\n", ioc->name); init_completion(&ioc->scsih_cmds.done); - mpt3sas_base_put_smid_default(ioc, smid); + ioc->put_smid_default(ioc, smid); wait_for_completion_timeout(&ioc->scsih_cmds.done, 10*HZ); if (!(ioc->scsih_cmds.status & MPT3_CMD_COMPLETE)) { @@ -8654,6 +8681,12 @@ _scsih_determine_hba_mpi_version(struct pci_dev *pdev) case MPI26_MFGPAGE_DEVID_SAS3324_2: case MPI26_MFGPAGE_DEVID_SAS3324_3: case MPI26_MFGPAGE_DEVID_SAS3324_4: + case MPI26_MFGPAGE_DEVID_SAS3508: + case MPI26_MFGPAGE_DEVID_SAS3508_1: + case MPI26_MFGPAGE_DEVID_SAS3408: + case MPI26_MFGPAGE_DEVID_SAS3516: + case MPI26_MFGPAGE_DEVID_SAS3516_1: + case MPI26_MFGPAGE_DEVID_SAS3416: return MPI26_VERSION; } return 0; @@ -8722,10 +8755,29 @@ _scsih_probe(struct pci_dev *pdev, const struct pci_device_id *id) ioc->hba_mpi_version_belonged = hba_mpi_version; ioc->id = mpt3_ids++; sprintf(ioc->driver_name, "%s", MPT3SAS_DRIVER_NAME); + switch (pdev->device) { + case MPI26_MFGPAGE_DEVID_SAS3508: + case MPI26_MFGPAGE_DEVID_SAS3508_1: + case MPI26_MFGPAGE_DEVID_SAS3408: + case MPI26_MFGPAGE_DEVID_SAS3516: + case MPI26_MFGPAGE_DEVID_SAS3516_1: + case MPI26_MFGPAGE_DEVID_SAS3416: + ioc->is_gen35_ioc = 1; + break; + default: + ioc->is_gen35_ioc = 0; + } if ((ioc->hba_mpi_version_belonged == MPI25_VERSION && pdev->revision >= SAS3_PCI_DEVICE_C0_REVISION) || - (ioc->hba_mpi_version_belonged == MPI26_VERSION)) - ioc->msix96_vector = 1; + (ioc->hba_mpi_version_belonged == MPI26_VERSION)) { + ioc->combined_reply_queue = 1; + if (ioc->is_gen35_ioc) + ioc->combined_reply_index_count = + MPT3_SUP_REPLY_POST_HOST_INDEX_REG_COUNT_G35; + else + ioc->combined_reply_index_count = + MPT3_SUP_REPLY_POST_HOST_INDEX_REG_COUNT_G3; + } break; default: return -ENODEV; @@ -9128,6 +9180,19 @@ static const struct pci_device_id mpt3sas_pci_table[] = { PCI_ANY_ID, PCI_ANY_ID }, { MPI2_MFGPAGE_VENDORID_LSI, MPI26_MFGPAGE_DEVID_SAS3324_4, PCI_ANY_ID, PCI_ANY_ID }, + /* Ventura, Crusader, Harpoon & Tomcat ~ 3516, 3416, 3508 & 3408*/ + { MPI2_MFGPAGE_VENDORID_LSI, MPI26_MFGPAGE_DEVID_SAS3508, + PCI_ANY_ID, PCI_ANY_ID }, + { MPI2_MFGPAGE_VENDORID_LSI, MPI26_MFGPAGE_DEVID_SAS3508_1, + PCI_ANY_ID, PCI_ANY_ID }, + { MPI2_MFGPAGE_VENDORID_LSI, MPI26_MFGPAGE_DEVID_SAS3408, + PCI_ANY_ID, PCI_ANY_ID }, + { MPI2_MFGPAGE_VENDORID_LSI, MPI26_MFGPAGE_DEVID_SAS3516, + PCI_ANY_ID, PCI_ANY_ID }, + { MPI2_MFGPAGE_VENDORID_LSI, MPI26_MFGPAGE_DEVID_SAS3516_1, + PCI_ANY_ID, PCI_ANY_ID }, + { MPI2_MFGPAGE_VENDORID_LSI, MPI26_MFGPAGE_DEVID_SAS3416, + PCI_ANY_ID, PCI_ANY_ID }, {0} /* Terminating entry */ }; MODULE_DEVICE_TABLE(pci, mpt3sas_pci_table); @@ -9168,7 +9233,7 @@ scsih_init(void) /* queuecommand callback hander */ scsi_io_cb_idx = mpt3sas_base_register_callback_handler(_scsih_io_done); - /* task managment callback handler */ + /* task management callback handler */ tm_cb_idx = mpt3sas_base_register_callback_handler(_scsih_tm_done); /* base internal commands callback handler */ diff --git a/drivers/scsi/mpt3sas/mpt3sas_transport.c b/drivers/scsi/mpt3sas/mpt3sas_transport.c index b74faf1..7f1d578 100644 --- a/drivers/scsi/mpt3sas/mpt3sas_transport.c +++ b/drivers/scsi/mpt3sas/mpt3sas_transport.c @@ -392,7 +392,7 @@ _transport_expander_report_manufacture(struct MPT3SAS_ADAPTER *ioc, "report_manufacture - send to sas_addr(0x%016llx)\n", ioc->name, (unsigned long long)sas_address)); init_completion(&ioc->transport_cmds.done); - mpt3sas_base_put_smid_default(ioc, smid); + ioc->put_smid_default(ioc, smid); wait_for_completion_timeout(&ioc->transport_cmds.done, 10*HZ); if (!(ioc->transport_cmds.status & MPT3_CMD_COMPLETE)) { @@ -1198,7 +1198,7 @@ _transport_get_expander_phy_error_log(struct MPT3SAS_ADAPTER *ioc, ioc->name, (unsigned long long)phy->identify.sas_address, phy->number)); init_completion(&ioc->transport_cmds.done); - mpt3sas_base_put_smid_default(ioc, smid); + ioc->put_smid_default(ioc, smid); wait_for_completion_timeout(&ioc->transport_cmds.done, 10*HZ); if (!(ioc->transport_cmds.status & MPT3_CMD_COMPLETE)) { @@ -1514,7 +1514,7 @@ _transport_expander_phy_control(struct MPT3SAS_ADAPTER *ioc, ioc->name, (unsigned long long)phy->identify.sas_address, phy->number, phy_operation)); init_completion(&ioc->transport_cmds.done); - mpt3sas_base_put_smid_default(ioc, smid); + ioc->put_smid_default(ioc, smid); wait_for_completion_timeout(&ioc->transport_cmds.done, 10*HZ); if (!(ioc->transport_cmds.status & MPT3_CMD_COMPLETE)) { @@ -2032,7 +2032,7 @@ _transport_smp_handler(struct Scsi_Host *shost, struct sas_rphy *rphy, "%s - sending smp request\n", ioc->name, __func__)); init_completion(&ioc->transport_cmds.done); - mpt3sas_base_put_smid_default(ioc, smid); + ioc->put_smid_default(ioc, smid); wait_for_completion_timeout(&ioc->transport_cmds.done, 10*HZ); if (!(ioc->transport_cmds.status & MPT3_CMD_COMPLETE)) { diff --git a/drivers/scsi/mvsas/mv_94xx.c b/drivers/scsi/mvsas/mv_94xx.c index 4c57d9a..7de5d8d 100644 --- a/drivers/scsi/mvsas/mv_94xx.c +++ b/drivers/scsi/mvsas/mv_94xx.c @@ -668,7 +668,7 @@ static void mvs_94xx_command_active(struct mvs_info *mvi, u32 slot_idx) { u32 tmp; tmp = mvs_cr32(mvi, MVS_COMMAND_ACTIVE+(slot_idx >> 3)); - if (tmp && 1 << (slot_idx % 32)) { + if (tmp & 1 << (slot_idx % 32)) { mv_printk("command active %08X, slot [%x].\n", tmp, slot_idx); mvs_cw32(mvi, MVS_COMMAND_ACTIVE + (slot_idx >> 3), 1 << (slot_idx % 32)); diff --git a/drivers/scsi/pmcraid.c b/drivers/scsi/pmcraid.c index 845affa..337982c 100644 --- a/drivers/scsi/pmcraid.c +++ b/drivers/scsi/pmcraid.c @@ -3787,11 +3787,11 @@ static long pmcraid_ioctl_passthrough( direction); if (rc) { pmcraid_err("couldn't build passthrough ioadls\n"); - goto out_free_buffer; + goto out_free_cmd; } } else if (request_size < 0) { rc = -EINVAL; - goto out_free_buffer; + goto out_free_cmd; } /* If data is being written into the device, copy the data from user @@ -3908,6 +3908,8 @@ out_handle_response: out_free_sglist: pmcraid_release_passthrough_ioadls(cmd, request_size, direction); + +out_free_cmd: pmcraid_return_cmd(cmd); out_free_buffer: @@ -6018,8 +6020,10 @@ static int __init pmcraid_init(void) error = pmcraid_netlink_init(); - if (error) + if (error) { + class_destroy(pmcraid_class); goto out_unreg_chrdev; + } error = pci_register_driver(&pmcraid_driver); diff --git a/drivers/scsi/qla2xxx/qla_bsg.c b/drivers/scsi/qla2xxx/qla_bsg.c index 643014f..1bf8061 100644 --- a/drivers/scsi/qla2xxx/qla_bsg.c +++ b/drivers/scsi/qla2xxx/qla_bsg.c @@ -1,4 +1,4 @@ -/* + /* * QLogic Fibre Channel HBA Driver * Copyright (c) 2003-2014 QLogic Corporation * @@ -9,6 +9,7 @@ #include <linux/kthread.h> #include <linux/vmalloc.h> #include <linux/delay.h> +#include <linux/bsg-lib.h> /* BSG support for ELS/CT pass through */ void @@ -16,10 +17,12 @@ qla2x00_bsg_job_done(void *data, void *ptr, int res) { srb_t *sp = (srb_t *)ptr; struct scsi_qla_host *vha = (scsi_qla_host_t *)data; - struct fc_bsg_job *bsg_job = sp->u.bsg_job; + struct bsg_job *bsg_job = sp->u.bsg_job; + struct fc_bsg_reply *bsg_reply = bsg_job->reply; - bsg_job->reply->result = res; - bsg_job->job_done(bsg_job); + bsg_reply->result = res; + bsg_job_done(bsg_job, bsg_reply->result, + bsg_reply->reply_payload_rcv_len); sp->free(vha, sp); } @@ -28,13 +31,15 @@ qla2x00_bsg_sp_free(void *data, void *ptr) { srb_t *sp = (srb_t *)ptr; struct scsi_qla_host *vha = sp->fcport->vha; - struct fc_bsg_job *bsg_job = sp->u.bsg_job; + struct bsg_job *bsg_job = sp->u.bsg_job; + struct fc_bsg_request *bsg_request = bsg_job->request; + struct qla_hw_data *ha = vha->hw; struct qla_mt_iocb_rqst_fx00 *piocb_rqst; if (sp->type == SRB_FXIOCB_BCMD) { piocb_rqst = (struct qla_mt_iocb_rqst_fx00 *) - &bsg_job->request->rqst_data.h_vendor.vendor_cmd[1]; + &bsg_request->rqst_data.h_vendor.vendor_cmd[1]; if (piocb_rqst->flags & SRB_FXDISC_REQ_DMA_VALID) dma_unmap_sg(&ha->pdev->dev, @@ -116,9 +121,11 @@ qla24xx_fcp_prio_cfg_valid(scsi_qla_host_t *vha, } static int -qla24xx_proc_fcp_prio_cfg_cmd(struct fc_bsg_job *bsg_job) +qla24xx_proc_fcp_prio_cfg_cmd(struct bsg_job *bsg_job) { - struct Scsi_Host *host = bsg_job->shost; + struct Scsi_Host *host = fc_bsg_to_shost(bsg_job); + struct fc_bsg_request *bsg_request = bsg_job->request; + struct fc_bsg_reply *bsg_reply = bsg_job->reply; scsi_qla_host_t *vha = shost_priv(host); struct qla_hw_data *ha = vha->hw; int ret = 0; @@ -131,7 +138,7 @@ qla24xx_proc_fcp_prio_cfg_cmd(struct fc_bsg_job *bsg_job) } /* Get the sub command */ - oper = bsg_job->request->rqst_data.h_vendor.vendor_cmd[1]; + oper = bsg_request->rqst_data.h_vendor.vendor_cmd[1]; /* Only set config is allowed if config memory is not allocated */ if (!ha->fcp_prio_cfg && (oper != QLFC_FCP_PRIO_SET_CONFIG)) { @@ -145,10 +152,10 @@ qla24xx_proc_fcp_prio_cfg_cmd(struct fc_bsg_job *bsg_job) ha->fcp_prio_cfg->attributes &= ~FCP_PRIO_ATTR_ENABLE; qla24xx_update_all_fcp_prio(vha); - bsg_job->reply->result = DID_OK; + bsg_reply->result = DID_OK; } else { ret = -EINVAL; - bsg_job->reply->result = (DID_ERROR << 16); + bsg_reply->result = (DID_ERROR << 16); goto exit_fcp_prio_cfg; } break; @@ -160,10 +167,10 @@ qla24xx_proc_fcp_prio_cfg_cmd(struct fc_bsg_job *bsg_job) ha->fcp_prio_cfg->attributes |= FCP_PRIO_ATTR_ENABLE; qla24xx_update_all_fcp_prio(vha); - bsg_job->reply->result = DID_OK; + bsg_reply->result = DID_OK; } else { ret = -EINVAL; - bsg_job->reply->result = (DID_ERROR << 16); + bsg_reply->result = (DID_ERROR << 16); goto exit_fcp_prio_cfg; } } @@ -173,12 +180,12 @@ qla24xx_proc_fcp_prio_cfg_cmd(struct fc_bsg_job *bsg_job) len = bsg_job->reply_payload.payload_len; if (!len || len > FCP_PRIO_CFG_SIZE) { ret = -EINVAL; - bsg_job->reply->result = (DID_ERROR << 16); + bsg_reply->result = (DID_ERROR << 16); goto exit_fcp_prio_cfg; } - bsg_job->reply->result = DID_OK; - bsg_job->reply->reply_payload_rcv_len = + bsg_reply->result = DID_OK; + bsg_reply->reply_payload_rcv_len = sg_copy_from_buffer( bsg_job->reply_payload.sg_list, bsg_job->reply_payload.sg_cnt, ha->fcp_prio_cfg, @@ -189,7 +196,7 @@ qla24xx_proc_fcp_prio_cfg_cmd(struct fc_bsg_job *bsg_job) case QLFC_FCP_PRIO_SET_CONFIG: len = bsg_job->request_payload.payload_len; if (!len || len > FCP_PRIO_CFG_SIZE) { - bsg_job->reply->result = (DID_ERROR << 16); + bsg_reply->result = (DID_ERROR << 16); ret = -EINVAL; goto exit_fcp_prio_cfg; } @@ -200,7 +207,7 @@ qla24xx_proc_fcp_prio_cfg_cmd(struct fc_bsg_job *bsg_job) ql_log(ql_log_warn, vha, 0x7050, "Unable to allocate memory for fcp prio " "config data (%x).\n", FCP_PRIO_CFG_SIZE); - bsg_job->reply->result = (DID_ERROR << 16); + bsg_reply->result = (DID_ERROR << 16); ret = -ENOMEM; goto exit_fcp_prio_cfg; } @@ -215,7 +222,7 @@ qla24xx_proc_fcp_prio_cfg_cmd(struct fc_bsg_job *bsg_job) if (!qla24xx_fcp_prio_cfg_valid(vha, (struct qla_fcp_prio_cfg *) ha->fcp_prio_cfg, 1)) { - bsg_job->reply->result = (DID_ERROR << 16); + bsg_reply->result = (DID_ERROR << 16); ret = -EINVAL; /* If buffer was invalidatic int * fcp_prio_cfg is of no use @@ -229,7 +236,7 @@ qla24xx_proc_fcp_prio_cfg_cmd(struct fc_bsg_job *bsg_job) if (ha->fcp_prio_cfg->attributes & FCP_PRIO_ATTR_ENABLE) ha->flags.fcp_prio_enabled = 1; qla24xx_update_all_fcp_prio(vha); - bsg_job->reply->result = DID_OK; + bsg_reply->result = DID_OK; break; default: ret = -EINVAL; @@ -237,13 +244,15 @@ qla24xx_proc_fcp_prio_cfg_cmd(struct fc_bsg_job *bsg_job) } exit_fcp_prio_cfg: if (!ret) - bsg_job->job_done(bsg_job); + bsg_job_done(bsg_job, bsg_reply->result, + bsg_reply->reply_payload_rcv_len); return ret; } static int -qla2x00_process_els(struct fc_bsg_job *bsg_job) +qla2x00_process_els(struct bsg_job *bsg_job) { + struct fc_bsg_request *bsg_request = bsg_job->request; struct fc_rport *rport; fc_port_t *fcport = NULL; struct Scsi_Host *host; @@ -255,15 +264,15 @@ qla2x00_process_els(struct fc_bsg_job *bsg_job) int rval = (DRIVER_ERROR << 16); uint16_t nextlid = 0; - if (bsg_job->request->msgcode == FC_BSG_RPT_ELS) { - rport = bsg_job->rport; + if (bsg_request->msgcode == FC_BSG_RPT_ELS) { + rport = fc_bsg_to_rport(bsg_job); fcport = *(fc_port_t **) rport->dd_data; host = rport_to_shost(rport); vha = shost_priv(host); ha = vha->hw; type = "FC_BSG_RPT_ELS"; } else { - host = bsg_job->shost; + host = fc_bsg_to_shost(bsg_job); vha = shost_priv(host); ha = vha->hw; type = "FC_BSG_HST_ELS_NOLOGIN"; @@ -296,7 +305,7 @@ qla2x00_process_els(struct fc_bsg_job *bsg_job) } /* ELS request for rport */ - if (bsg_job->request->msgcode == FC_BSG_RPT_ELS) { + if (bsg_request->msgcode == FC_BSG_RPT_ELS) { /* make sure the rport is logged in, * if not perform fabric login */ @@ -322,11 +331,11 @@ qla2x00_process_els(struct fc_bsg_job *bsg_job) /* Initialize all required fields of fcport */ fcport->vha = vha; fcport->d_id.b.al_pa = - bsg_job->request->rqst_data.h_els.port_id[0]; + bsg_request->rqst_data.h_els.port_id[0]; fcport->d_id.b.area = - bsg_job->request->rqst_data.h_els.port_id[1]; + bsg_request->rqst_data.h_els.port_id[1]; fcport->d_id.b.domain = - bsg_job->request->rqst_data.h_els.port_id[2]; + bsg_request->rqst_data.h_els.port_id[2]; fcport->loop_id = (fcport->d_id.b.al_pa == 0xFD) ? NPH_FABRIC_CONTROLLER : NPH_F_PORT; @@ -366,11 +375,11 @@ qla2x00_process_els(struct fc_bsg_job *bsg_job) } sp->type = - (bsg_job->request->msgcode == FC_BSG_RPT_ELS ? - SRB_ELS_CMD_RPT : SRB_ELS_CMD_HST); + (bsg_request->msgcode == FC_BSG_RPT_ELS ? + SRB_ELS_CMD_RPT : SRB_ELS_CMD_HST); sp->name = - (bsg_job->request->msgcode == FC_BSG_RPT_ELS ? - "bsg_els_rpt" : "bsg_els_hst"); + (bsg_request->msgcode == FC_BSG_RPT_ELS ? + "bsg_els_rpt" : "bsg_els_hst"); sp->u.bsg_job = bsg_job; sp->free = qla2x00_bsg_sp_free; sp->done = qla2x00_bsg_job_done; @@ -378,7 +387,7 @@ qla2x00_process_els(struct fc_bsg_job *bsg_job) ql_dbg(ql_dbg_user, vha, 0x700a, "bsg rqst type: %s els type: %x - loop-id=%x " "portid=%-2x%02x%02x.\n", type, - bsg_job->request->rqst_data.h_els.command_code, fcport->loop_id, + bsg_request->rqst_data.h_els.command_code, fcport->loop_id, fcport->d_id.b.domain, fcport->d_id.b.area, fcport->d_id.b.al_pa); rval = qla2x00_start_sp(sp); @@ -399,7 +408,7 @@ done_unmap_sg: goto done_free_fcport; done_free_fcport: - if (bsg_job->request->msgcode == FC_BSG_RPT_ELS) + if (bsg_request->msgcode == FC_BSG_RPT_ELS) kfree(fcport); done: return rval; @@ -420,10 +429,11 @@ qla24xx_calc_ct_iocbs(uint16_t dsds) } static int -qla2x00_process_ct(struct fc_bsg_job *bsg_job) +qla2x00_process_ct(struct bsg_job *bsg_job) { srb_t *sp; - struct Scsi_Host *host = bsg_job->shost; + struct fc_bsg_request *bsg_request = bsg_job->request; + struct Scsi_Host *host = fc_bsg_to_shost(bsg_job); scsi_qla_host_t *vha = shost_priv(host); struct qla_hw_data *ha = vha->hw; int rval = (DRIVER_ERROR << 16); @@ -469,7 +479,7 @@ qla2x00_process_ct(struct fc_bsg_job *bsg_job) } loop_id = - (bsg_job->request->rqst_data.h_ct.preamble_word1 & 0xFF000000) + (bsg_request->rqst_data.h_ct.preamble_word1 & 0xFF000000) >> 24; switch (loop_id) { case 0xFC: @@ -500,9 +510,9 @@ qla2x00_process_ct(struct fc_bsg_job *bsg_job) /* Initialize all required fields of fcport */ fcport->vha = vha; - fcport->d_id.b.al_pa = bsg_job->request->rqst_data.h_ct.port_id[0]; - fcport->d_id.b.area = bsg_job->request->rqst_data.h_ct.port_id[1]; - fcport->d_id.b.domain = bsg_job->request->rqst_data.h_ct.port_id[2]; + fcport->d_id.b.al_pa = bsg_request->rqst_data.h_ct.port_id[0]; + fcport->d_id.b.area = bsg_request->rqst_data.h_ct.port_id[1]; + fcport->d_id.b.domain = bsg_request->rqst_data.h_ct.port_id[2]; fcport->loop_id = loop_id; /* Alloc SRB structure */ @@ -524,7 +534,7 @@ qla2x00_process_ct(struct fc_bsg_job *bsg_job) ql_dbg(ql_dbg_user, vha, 0x7016, "bsg rqst type: %s else type: %x - " "loop-id=%x portid=%02x%02x%02x.\n", type, - (bsg_job->request->rqst_data.h_ct.preamble_word2 >> 16), + (bsg_request->rqst_data.h_ct.preamble_word2 >> 16), fcport->loop_id, fcport->d_id.b.domain, fcport->d_id.b.area, fcport->d_id.b.al_pa); @@ -697,9 +707,11 @@ done_set_internal: } static int -qla2x00_process_loopback(struct fc_bsg_job *bsg_job) +qla2x00_process_loopback(struct bsg_job *bsg_job) { - struct Scsi_Host *host = bsg_job->shost; + struct fc_bsg_request *bsg_request = bsg_job->request; + struct fc_bsg_reply *bsg_reply = bsg_job->reply; + struct Scsi_Host *host = fc_bsg_to_shost(bsg_job); scsi_qla_host_t *vha = shost_priv(host); struct qla_hw_data *ha = vha->hw; int rval; @@ -780,9 +792,9 @@ qla2x00_process_loopback(struct fc_bsg_job *bsg_job) elreq.rcv_dma = rsp_data_dma; elreq.transfer_size = req_data_len; - elreq.options = bsg_job->request->rqst_data.h_vendor.vendor_cmd[1]; + elreq.options = bsg_request->rqst_data.h_vendor.vendor_cmd[1]; elreq.iteration_count = - bsg_job->request->rqst_data.h_vendor.vendor_cmd[2]; + bsg_request->rqst_data.h_vendor.vendor_cmd[2]; if (atomic_read(&vha->loop_state) == LOOP_READY && (ha->current_topology == ISP_CFG_F || @@ -896,12 +908,12 @@ qla2x00_process_loopback(struct fc_bsg_job *bsg_job) "Vendor request %s failed.\n", type); rval = 0; - bsg_job->reply->result = (DID_ERROR << 16); - bsg_job->reply->reply_payload_rcv_len = 0; + bsg_reply->result = (DID_ERROR << 16); + bsg_reply->reply_payload_rcv_len = 0; } else { ql_dbg(ql_dbg_user, vha, 0x702d, "Vendor request %s completed.\n", type); - bsg_job->reply->result = (DID_OK << 16); + bsg_reply->result = (DID_OK << 16); sg_copy_from_buffer(bsg_job->reply_payload.sg_list, bsg_job->reply_payload.sg_cnt, rsp_data, rsp_data_len); @@ -930,14 +942,17 @@ done_unmap_req_sg: bsg_job->request_payload.sg_list, bsg_job->request_payload.sg_cnt, DMA_TO_DEVICE); if (!rval) - bsg_job->job_done(bsg_job); + bsg_job_done(bsg_job, bsg_reply->result, + bsg_reply->reply_payload_rcv_len); return rval; } static int -qla84xx_reset(struct fc_bsg_job *bsg_job) +qla84xx_reset(struct bsg_job *bsg_job) { - struct Scsi_Host *host = bsg_job->shost; + struct fc_bsg_request *bsg_request = bsg_job->request; + struct Scsi_Host *host = fc_bsg_to_shost(bsg_job); + struct fc_bsg_reply *bsg_reply = bsg_job->reply; scsi_qla_host_t *vha = shost_priv(host); struct qla_hw_data *ha = vha->hw; int rval = 0; @@ -948,7 +963,7 @@ qla84xx_reset(struct fc_bsg_job *bsg_job) return -EINVAL; } - flag = bsg_job->request->rqst_data.h_vendor.vendor_cmd[1]; + flag = bsg_request->rqst_data.h_vendor.vendor_cmd[1]; rval = qla84xx_reset_chip(vha, flag == A84_ISSUE_RESET_DIAG_FW); @@ -960,17 +975,20 @@ qla84xx_reset(struct fc_bsg_job *bsg_job) } else { ql_dbg(ql_dbg_user, vha, 0x7031, "Vendor request 84xx reset completed.\n"); - bsg_job->reply->result = DID_OK; - bsg_job->job_done(bsg_job); + bsg_reply->result = DID_OK; + bsg_job_done(bsg_job, bsg_reply->result, + bsg_reply->reply_payload_rcv_len); } return rval; } static int -qla84xx_updatefw(struct fc_bsg_job *bsg_job) +qla84xx_updatefw(struct bsg_job *bsg_job) { - struct Scsi_Host *host = bsg_job->shost; + struct fc_bsg_request *bsg_request = bsg_job->request; + struct fc_bsg_reply *bsg_reply = bsg_job->reply; + struct Scsi_Host *host = fc_bsg_to_shost(bsg_job); scsi_qla_host_t *vha = shost_priv(host); struct qla_hw_data *ha = vha->hw; struct verify_chip_entry_84xx *mn = NULL; @@ -1027,7 +1045,7 @@ qla84xx_updatefw(struct fc_bsg_job *bsg_job) goto done_free_fw_buf; } - flag = bsg_job->request->rqst_data.h_vendor.vendor_cmd[1]; + flag = bsg_request->rqst_data.h_vendor.vendor_cmd[1]; fw_ver = le32_to_cpu(*((uint32_t *)((uint32_t *)fw_buf + 2))); memset(mn, 0, sizeof(struct access_chip_84xx)); @@ -1059,7 +1077,7 @@ qla84xx_updatefw(struct fc_bsg_job *bsg_job) "Vendor request 84xx updatefw completed.\n"); bsg_job->reply_len = sizeof(struct fc_bsg_reply); - bsg_job->reply->result = DID_OK; + bsg_reply->result = DID_OK; } dma_pool_free(ha->s_dma_pool, mn, mn_dma); @@ -1072,14 +1090,17 @@ done_unmap_sg: bsg_job->request_payload.sg_cnt, DMA_TO_DEVICE); if (!rval) - bsg_job->job_done(bsg_job); + bsg_job_done(bsg_job, bsg_reply->result, + bsg_reply->reply_payload_rcv_len); return rval; } static int -qla84xx_mgmt_cmd(struct fc_bsg_job *bsg_job) +qla84xx_mgmt_cmd(struct bsg_job *bsg_job) { - struct Scsi_Host *host = bsg_job->shost; + struct fc_bsg_request *bsg_request = bsg_job->request; + struct fc_bsg_reply *bsg_reply = bsg_job->reply; + struct Scsi_Host *host = fc_bsg_to_shost(bsg_job); scsi_qla_host_t *vha = shost_priv(host); struct qla_hw_data *ha = vha->hw; struct access_chip_84xx *mn = NULL; @@ -1107,7 +1128,7 @@ qla84xx_mgmt_cmd(struct fc_bsg_job *bsg_job) memset(mn, 0, sizeof(struct access_chip_84xx)); mn->entry_type = ACCESS_CHIP_IOCB_TYPE; mn->entry_count = 1; - ql84_mgmt = (void *)bsg_job->request + sizeof(struct fc_bsg_request); + ql84_mgmt = (void *)bsg_request + sizeof(struct fc_bsg_request); switch (ql84_mgmt->mgmt.cmd) { case QLA84_MGMT_READ_MEM: case QLA84_MGMT_GET_INFO: @@ -1239,11 +1260,11 @@ qla84xx_mgmt_cmd(struct fc_bsg_job *bsg_job) "Vendor request 84xx mgmt completed.\n"); bsg_job->reply_len = sizeof(struct fc_bsg_reply); - bsg_job->reply->result = DID_OK; + bsg_reply->result = DID_OK; if ((ql84_mgmt->mgmt.cmd == QLA84_MGMT_READ_MEM) || (ql84_mgmt->mgmt.cmd == QLA84_MGMT_GET_INFO)) { - bsg_job->reply->reply_payload_rcv_len = + bsg_reply->reply_payload_rcv_len = bsg_job->reply_payload.payload_len; sg_copy_from_buffer(bsg_job->reply_payload.sg_list, @@ -1267,14 +1288,17 @@ exit_mgmt: dma_pool_free(ha->s_dma_pool, mn, mn_dma); if (!rval) - bsg_job->job_done(bsg_job); + bsg_job_done(bsg_job, bsg_reply->result, + bsg_reply->reply_payload_rcv_len); return rval; } static int -qla24xx_iidma(struct fc_bsg_job *bsg_job) +qla24xx_iidma(struct bsg_job *bsg_job) { - struct Scsi_Host *host = bsg_job->shost; + struct fc_bsg_request *bsg_request = bsg_job->request; + struct fc_bsg_reply *bsg_reply = bsg_job->reply; + struct Scsi_Host *host = fc_bsg_to_shost(bsg_job); scsi_qla_host_t *vha = shost_priv(host); int rval = 0; struct qla_port_param *port_param = NULL; @@ -1288,7 +1312,7 @@ qla24xx_iidma(struct fc_bsg_job *bsg_job) return -EINVAL; } - port_param = (void *)bsg_job->request + sizeof(struct fc_bsg_request); + port_param = (void *)bsg_request + sizeof(struct fc_bsg_request); if (port_param->fc_scsi_addr.dest_type != EXT_DEF_TYPE_WWPN) { ql_log(ql_log_warn, vha, 0x7048, "Invalid destination type.\n"); @@ -1343,24 +1367,26 @@ qla24xx_iidma(struct fc_bsg_job *bsg_job) bsg_job->reply_len = sizeof(struct fc_bsg_reply) + sizeof(struct qla_port_param); - rsp_ptr = ((uint8_t *)bsg_job->reply) + + rsp_ptr = ((uint8_t *)bsg_reply) + sizeof(struct fc_bsg_reply); memcpy(rsp_ptr, port_param, sizeof(struct qla_port_param)); } - bsg_job->reply->result = DID_OK; - bsg_job->job_done(bsg_job); + bsg_reply->result = DID_OK; + bsg_job_done(bsg_job, bsg_reply->result, + bsg_reply->reply_payload_rcv_len); } return rval; } static int -qla2x00_optrom_setup(struct fc_bsg_job *bsg_job, scsi_qla_host_t *vha, +qla2x00_optrom_setup(struct bsg_job *bsg_job, scsi_qla_host_t *vha, uint8_t is_update) { + struct fc_bsg_request *bsg_request = bsg_job->request; uint32_t start = 0; int valid = 0; struct qla_hw_data *ha = vha->hw; @@ -1368,7 +1394,7 @@ qla2x00_optrom_setup(struct fc_bsg_job *bsg_job, scsi_qla_host_t *vha, if (unlikely(pci_channel_offline(ha->pdev))) return -EINVAL; - start = bsg_job->request->rqst_data.h_vendor.vendor_cmd[1]; + start = bsg_request->rqst_data.h_vendor.vendor_cmd[1]; if (start > ha->optrom_size) { ql_log(ql_log_warn, vha, 0x7055, "start %d > optrom_size %d.\n", start, ha->optrom_size); @@ -1427,9 +1453,10 @@ qla2x00_optrom_setup(struct fc_bsg_job *bsg_job, scsi_qla_host_t *vha, } static int -qla2x00_read_optrom(struct fc_bsg_job *bsg_job) +qla2x00_read_optrom(struct bsg_job *bsg_job) { - struct Scsi_Host *host = bsg_job->shost; + struct fc_bsg_reply *bsg_reply = bsg_job->reply; + struct Scsi_Host *host = fc_bsg_to_shost(bsg_job); scsi_qla_host_t *vha = shost_priv(host); struct qla_hw_data *ha = vha->hw; int rval = 0; @@ -1451,20 +1478,22 @@ qla2x00_read_optrom(struct fc_bsg_job *bsg_job) bsg_job->reply_payload.sg_cnt, ha->optrom_buffer, ha->optrom_region_size); - bsg_job->reply->reply_payload_rcv_len = ha->optrom_region_size; - bsg_job->reply->result = DID_OK; + bsg_reply->reply_payload_rcv_len = ha->optrom_region_size; + bsg_reply->result = DID_OK; vfree(ha->optrom_buffer); ha->optrom_buffer = NULL; ha->optrom_state = QLA_SWAITING; mutex_unlock(&ha->optrom_mutex); - bsg_job->job_done(bsg_job); + bsg_job_done(bsg_job, bsg_reply->result, + bsg_reply->reply_payload_rcv_len); return rval; } static int -qla2x00_update_optrom(struct fc_bsg_job *bsg_job) +qla2x00_update_optrom(struct bsg_job *bsg_job) { - struct Scsi_Host *host = bsg_job->shost; + struct fc_bsg_reply *bsg_reply = bsg_job->reply; + struct Scsi_Host *host = fc_bsg_to_shost(bsg_job); scsi_qla_host_t *vha = shost_priv(host); struct qla_hw_data *ha = vha->hw; int rval = 0; @@ -1486,19 +1515,21 @@ qla2x00_update_optrom(struct fc_bsg_job *bsg_job) ha->isp_ops->write_optrom(vha, ha->optrom_buffer, ha->optrom_region_start, ha->optrom_region_size); - bsg_job->reply->result = DID_OK; + bsg_reply->result = DID_OK; vfree(ha->optrom_buffer); ha->optrom_buffer = NULL; ha->optrom_state = QLA_SWAITING; mutex_unlock(&ha->optrom_mutex); - bsg_job->job_done(bsg_job); + bsg_job_done(bsg_job, bsg_reply->result, + bsg_reply->reply_payload_rcv_len); return rval; } static int -qla2x00_update_fru_versions(struct fc_bsg_job *bsg_job) +qla2x00_update_fru_versions(struct bsg_job *bsg_job) { - struct Scsi_Host *host = bsg_job->shost; + struct fc_bsg_reply *bsg_reply = bsg_job->reply; + struct Scsi_Host *host = fc_bsg_to_shost(bsg_job); scsi_qla_host_t *vha = shost_priv(host); struct qla_hw_data *ha = vha->hw; int rval = 0; @@ -1509,7 +1540,7 @@ qla2x00_update_fru_versions(struct fc_bsg_job *bsg_job) dma_addr_t sfp_dma; void *sfp = dma_pool_alloc(ha->s_dma_pool, GFP_KERNEL, &sfp_dma); if (!sfp) { - bsg_job->reply->reply_data.vendor_reply.vendor_rsp[0] = + bsg_reply->reply_data.vendor_reply.vendor_rsp[0] = EXT_STATUS_NO_MEMORY; goto done; } @@ -1525,30 +1556,32 @@ qla2x00_update_fru_versions(struct fc_bsg_job *bsg_job) image->field_address.device, image->field_address.offset, sizeof(image->field_info), image->field_address.option); if (rval) { - bsg_job->reply->reply_data.vendor_reply.vendor_rsp[0] = + bsg_reply->reply_data.vendor_reply.vendor_rsp[0] = EXT_STATUS_MAILBOX; goto dealloc; } image++; } - bsg_job->reply->reply_data.vendor_reply.vendor_rsp[0] = 0; + bsg_reply->reply_data.vendor_reply.vendor_rsp[0] = 0; dealloc: dma_pool_free(ha->s_dma_pool, sfp, sfp_dma); done: bsg_job->reply_len = sizeof(struct fc_bsg_reply); - bsg_job->reply->result = DID_OK << 16; - bsg_job->job_done(bsg_job); + bsg_reply->result = DID_OK << 16; + bsg_job_done(bsg_job, bsg_reply->result, + bsg_reply->reply_payload_rcv_len); return 0; } static int -qla2x00_read_fru_status(struct fc_bsg_job *bsg_job) +qla2x00_read_fru_status(struct bsg_job *bsg_job) { - struct Scsi_Host *host = bsg_job->shost; + struct fc_bsg_reply *bsg_reply = bsg_job->reply; + struct Scsi_Host *host = fc_bsg_to_shost(bsg_job); scsi_qla_host_t *vha = shost_priv(host); struct qla_hw_data *ha = vha->hw; int rval = 0; @@ -1557,7 +1590,7 @@ qla2x00_read_fru_status(struct fc_bsg_job *bsg_job) dma_addr_t sfp_dma; uint8_t *sfp = dma_pool_alloc(ha->s_dma_pool, GFP_KERNEL, &sfp_dma); if (!sfp) { - bsg_job->reply->reply_data.vendor_reply.vendor_rsp[0] = + bsg_reply->reply_data.vendor_reply.vendor_rsp[0] = EXT_STATUS_NO_MEMORY; goto done; } @@ -1571,7 +1604,7 @@ qla2x00_read_fru_status(struct fc_bsg_job *bsg_job) sr->status_reg = *sfp; if (rval) { - bsg_job->reply->reply_data.vendor_reply.vendor_rsp[0] = + bsg_reply->reply_data.vendor_reply.vendor_rsp[0] = EXT_STATUS_MAILBOX; goto dealloc; } @@ -1579,24 +1612,26 @@ qla2x00_read_fru_status(struct fc_bsg_job *bsg_job) sg_copy_from_buffer(bsg_job->reply_payload.sg_list, bsg_job->reply_payload.sg_cnt, sr, sizeof(*sr)); - bsg_job->reply->reply_data.vendor_reply.vendor_rsp[0] = 0; + bsg_reply->reply_data.vendor_reply.vendor_rsp[0] = 0; dealloc: dma_pool_free(ha->s_dma_pool, sfp, sfp_dma); done: bsg_job->reply_len = sizeof(struct fc_bsg_reply); - bsg_job->reply->reply_payload_rcv_len = sizeof(*sr); - bsg_job->reply->result = DID_OK << 16; - bsg_job->job_done(bsg_job); + bsg_reply->reply_payload_rcv_len = sizeof(*sr); + bsg_reply->result = DID_OK << 16; + bsg_job_done(bsg_job, bsg_reply->result, + bsg_reply->reply_payload_rcv_len); return 0; } static int -qla2x00_write_fru_status(struct fc_bsg_job *bsg_job) +qla2x00_write_fru_status(struct bsg_job *bsg_job) { - struct Scsi_Host *host = bsg_job->shost; + struct fc_bsg_reply *bsg_reply = bsg_job->reply; + struct Scsi_Host *host = fc_bsg_to_shost(bsg_job); scsi_qla_host_t *vha = shost_priv(host); struct qla_hw_data *ha = vha->hw; int rval = 0; @@ -1605,7 +1640,7 @@ qla2x00_write_fru_status(struct fc_bsg_job *bsg_job) dma_addr_t sfp_dma; uint8_t *sfp = dma_pool_alloc(ha->s_dma_pool, GFP_KERNEL, &sfp_dma); if (!sfp) { - bsg_job->reply->reply_data.vendor_reply.vendor_rsp[0] = + bsg_reply->reply_data.vendor_reply.vendor_rsp[0] = EXT_STATUS_NO_MEMORY; goto done; } @@ -1619,28 +1654,30 @@ qla2x00_write_fru_status(struct fc_bsg_job *bsg_job) sizeof(sr->status_reg), sr->field_address.option); if (rval) { - bsg_job->reply->reply_data.vendor_reply.vendor_rsp[0] = + bsg_reply->reply_data.vendor_reply.vendor_rsp[0] = EXT_STATUS_MAILBOX; goto dealloc; } - bsg_job->reply->reply_data.vendor_reply.vendor_rsp[0] = 0; + bsg_reply->reply_data.vendor_reply.vendor_rsp[0] = 0; dealloc: dma_pool_free(ha->s_dma_pool, sfp, sfp_dma); done: bsg_job->reply_len = sizeof(struct fc_bsg_reply); - bsg_job->reply->result = DID_OK << 16; - bsg_job->job_done(bsg_job); + bsg_reply->result = DID_OK << 16; + bsg_job_done(bsg_job, bsg_reply->result, + bsg_reply->reply_payload_rcv_len); return 0; } static int -qla2x00_write_i2c(struct fc_bsg_job *bsg_job) +qla2x00_write_i2c(struct bsg_job *bsg_job) { - struct Scsi_Host *host = bsg_job->shost; + struct fc_bsg_reply *bsg_reply = bsg_job->reply; + struct Scsi_Host *host = fc_bsg_to_shost(bsg_job); scsi_qla_host_t *vha = shost_priv(host); struct qla_hw_data *ha = vha->hw; int rval = 0; @@ -1649,7 +1686,7 @@ qla2x00_write_i2c(struct fc_bsg_job *bsg_job) dma_addr_t sfp_dma; uint8_t *sfp = dma_pool_alloc(ha->s_dma_pool, GFP_KERNEL, &sfp_dma); if (!sfp) { - bsg_job->reply->reply_data.vendor_reply.vendor_rsp[0] = + bsg_reply->reply_data.vendor_reply.vendor_rsp[0] = EXT_STATUS_NO_MEMORY; goto done; } @@ -1662,28 +1699,30 @@ qla2x00_write_i2c(struct fc_bsg_job *bsg_job) i2c->device, i2c->offset, i2c->length, i2c->option); if (rval) { - bsg_job->reply->reply_data.vendor_reply.vendor_rsp[0] = + bsg_reply->reply_data.vendor_reply.vendor_rsp[0] = EXT_STATUS_MAILBOX; goto dealloc; } - bsg_job->reply->reply_data.vendor_reply.vendor_rsp[0] = 0; + bsg_reply->reply_data.vendor_reply.vendor_rsp[0] = 0; dealloc: dma_pool_free(ha->s_dma_pool, sfp, sfp_dma); done: bsg_job->reply_len = sizeof(struct fc_bsg_reply); - bsg_job->reply->result = DID_OK << 16; - bsg_job->job_done(bsg_job); + bsg_reply->result = DID_OK << 16; + bsg_job_done(bsg_job, bsg_reply->result, + bsg_reply->reply_payload_rcv_len); return 0; } static int -qla2x00_read_i2c(struct fc_bsg_job *bsg_job) +qla2x00_read_i2c(struct bsg_job *bsg_job) { - struct Scsi_Host *host = bsg_job->shost; + struct fc_bsg_reply *bsg_reply = bsg_job->reply; + struct Scsi_Host *host = fc_bsg_to_shost(bsg_job); scsi_qla_host_t *vha = shost_priv(host); struct qla_hw_data *ha = vha->hw; int rval = 0; @@ -1692,7 +1731,7 @@ qla2x00_read_i2c(struct fc_bsg_job *bsg_job) dma_addr_t sfp_dma; uint8_t *sfp = dma_pool_alloc(ha->s_dma_pool, GFP_KERNEL, &sfp_dma); if (!sfp) { - bsg_job->reply->reply_data.vendor_reply.vendor_rsp[0] = + bsg_reply->reply_data.vendor_reply.vendor_rsp[0] = EXT_STATUS_NO_MEMORY; goto done; } @@ -1704,7 +1743,7 @@ qla2x00_read_i2c(struct fc_bsg_job *bsg_job) i2c->device, i2c->offset, i2c->length, i2c->option); if (rval) { - bsg_job->reply->reply_data.vendor_reply.vendor_rsp[0] = + bsg_reply->reply_data.vendor_reply.vendor_rsp[0] = EXT_STATUS_MAILBOX; goto dealloc; } @@ -1713,24 +1752,26 @@ qla2x00_read_i2c(struct fc_bsg_job *bsg_job) sg_copy_from_buffer(bsg_job->reply_payload.sg_list, bsg_job->reply_payload.sg_cnt, i2c, sizeof(*i2c)); - bsg_job->reply->reply_data.vendor_reply.vendor_rsp[0] = 0; + bsg_reply->reply_data.vendor_reply.vendor_rsp[0] = 0; dealloc: dma_pool_free(ha->s_dma_pool, sfp, sfp_dma); done: bsg_job->reply_len = sizeof(struct fc_bsg_reply); - bsg_job->reply->reply_payload_rcv_len = sizeof(*i2c); - bsg_job->reply->result = DID_OK << 16; - bsg_job->job_done(bsg_job); + bsg_reply->reply_payload_rcv_len = sizeof(*i2c); + bsg_reply->result = DID_OK << 16; + bsg_job_done(bsg_job, bsg_reply->result, + bsg_reply->reply_payload_rcv_len); return 0; } static int -qla24xx_process_bidir_cmd(struct fc_bsg_job *bsg_job) +qla24xx_process_bidir_cmd(struct bsg_job *bsg_job) { - struct Scsi_Host *host = bsg_job->shost; + struct fc_bsg_reply *bsg_reply = bsg_job->reply; + struct Scsi_Host *host = fc_bsg_to_shost(bsg_job); scsi_qla_host_t *vha = shost_priv(host); struct qla_hw_data *ha = vha->hw; uint32_t rval = EXT_STATUS_OK; @@ -1895,19 +1936,21 @@ done: /* Return an error vendor specific response * and complete the bsg request */ - bsg_job->reply->reply_data.vendor_reply.vendor_rsp[0] = rval; + bsg_reply->reply_data.vendor_reply.vendor_rsp[0] = rval; bsg_job->reply_len = sizeof(struct fc_bsg_reply); - bsg_job->reply->reply_payload_rcv_len = 0; - bsg_job->reply->result = (DID_OK) << 16; - bsg_job->job_done(bsg_job); + bsg_reply->reply_payload_rcv_len = 0; + bsg_reply->result = (DID_OK) << 16; + bsg_job_done(bsg_job, bsg_reply->result, + bsg_reply->reply_payload_rcv_len); /* Always return success, vendor rsp carries correct status */ return 0; } static int -qlafx00_mgmt_cmd(struct fc_bsg_job *bsg_job) +qlafx00_mgmt_cmd(struct bsg_job *bsg_job) { - struct Scsi_Host *host = bsg_job->shost; + struct fc_bsg_request *bsg_request = bsg_job->request; + struct Scsi_Host *host = fc_bsg_to_shost(bsg_job); scsi_qla_host_t *vha = shost_priv(host); struct qla_hw_data *ha = vha->hw; int rval = (DRIVER_ERROR << 16); @@ -1919,7 +1962,7 @@ qlafx00_mgmt_cmd(struct fc_bsg_job *bsg_job) /* Copy the IOCB specific information */ piocb_rqst = (struct qla_mt_iocb_rqst_fx00 *) - &bsg_job->request->rqst_data.h_vendor.vendor_cmd[1]; + &bsg_request->rqst_data.h_vendor.vendor_cmd[1]; /* Dump the vendor information */ ql_dump_buffer(ql_dbg_user + ql_dbg_verbose , vha, 0x70cf, @@ -2027,9 +2070,10 @@ done: } static int -qla26xx_serdes_op(struct fc_bsg_job *bsg_job) +qla26xx_serdes_op(struct bsg_job *bsg_job) { - struct Scsi_Host *host = bsg_job->shost; + struct fc_bsg_reply *bsg_reply = bsg_job->reply; + struct Scsi_Host *host = fc_bsg_to_shost(bsg_job); scsi_qla_host_t *vha = shost_priv(host); int rval = 0; struct qla_serdes_reg sr; @@ -2042,13 +2086,13 @@ qla26xx_serdes_op(struct fc_bsg_job *bsg_job) switch (sr.cmd) { case INT_SC_SERDES_WRITE_REG: rval = qla2x00_write_serdes_word(vha, sr.addr, sr.val); - bsg_job->reply->reply_payload_rcv_len = 0; + bsg_reply->reply_payload_rcv_len = 0; break; case INT_SC_SERDES_READ_REG: rval = qla2x00_read_serdes_word(vha, sr.addr, &sr.val); sg_copy_from_buffer(bsg_job->reply_payload.sg_list, bsg_job->reply_payload.sg_cnt, &sr, sizeof(sr)); - bsg_job->reply->reply_payload_rcv_len = sizeof(sr); + bsg_reply->reply_payload_rcv_len = sizeof(sr); break; default: ql_dbg(ql_dbg_user, vha, 0x708c, @@ -2057,19 +2101,21 @@ qla26xx_serdes_op(struct fc_bsg_job *bsg_job) break; } - bsg_job->reply->reply_data.vendor_reply.vendor_rsp[0] = + bsg_reply->reply_data.vendor_reply.vendor_rsp[0] = rval ? EXT_STATUS_MAILBOX : 0; bsg_job->reply_len = sizeof(struct fc_bsg_reply); - bsg_job->reply->result = DID_OK << 16; - bsg_job->job_done(bsg_job); + bsg_reply->result = DID_OK << 16; + bsg_job_done(bsg_job, bsg_reply->result, + bsg_reply->reply_payload_rcv_len); return 0; } static int -qla8044_serdes_op(struct fc_bsg_job *bsg_job) +qla8044_serdes_op(struct bsg_job *bsg_job) { - struct Scsi_Host *host = bsg_job->shost; + struct fc_bsg_reply *bsg_reply = bsg_job->reply; + struct Scsi_Host *host = fc_bsg_to_shost(bsg_job); scsi_qla_host_t *vha = shost_priv(host); int rval = 0; struct qla_serdes_reg_ex sr; @@ -2082,13 +2128,13 @@ qla8044_serdes_op(struct fc_bsg_job *bsg_job) switch (sr.cmd) { case INT_SC_SERDES_WRITE_REG: rval = qla8044_write_serdes_word(vha, sr.addr, sr.val); - bsg_job->reply->reply_payload_rcv_len = 0; + bsg_reply->reply_payload_rcv_len = 0; break; case INT_SC_SERDES_READ_REG: rval = qla8044_read_serdes_word(vha, sr.addr, &sr.val); sg_copy_from_buffer(bsg_job->reply_payload.sg_list, bsg_job->reply_payload.sg_cnt, &sr, sizeof(sr)); - bsg_job->reply->reply_payload_rcv_len = sizeof(sr); + bsg_reply->reply_payload_rcv_len = sizeof(sr); break; default: ql_dbg(ql_dbg_user, vha, 0x70cf, @@ -2097,19 +2143,21 @@ qla8044_serdes_op(struct fc_bsg_job *bsg_job) break; } - bsg_job->reply->reply_data.vendor_reply.vendor_rsp[0] = + bsg_reply->reply_data.vendor_reply.vendor_rsp[0] = rval ? EXT_STATUS_MAILBOX : 0; bsg_job->reply_len = sizeof(struct fc_bsg_reply); - bsg_job->reply->result = DID_OK << 16; - bsg_job->job_done(bsg_job); + bsg_reply->result = DID_OK << 16; + bsg_job_done(bsg_job, bsg_reply->result, + bsg_reply->reply_payload_rcv_len); return 0; } static int -qla27xx_get_flash_upd_cap(struct fc_bsg_job *bsg_job) +qla27xx_get_flash_upd_cap(struct bsg_job *bsg_job) { - struct Scsi_Host *host = bsg_job->shost; + struct fc_bsg_reply *bsg_reply = bsg_job->reply; + struct Scsi_Host *host = fc_bsg_to_shost(bsg_job); scsi_qla_host_t *vha = shost_priv(host); struct qla_hw_data *ha = vha->hw; struct qla_flash_update_caps cap; @@ -2125,21 +2173,23 @@ qla27xx_get_flash_upd_cap(struct fc_bsg_job *bsg_job) sg_copy_from_buffer(bsg_job->reply_payload.sg_list, bsg_job->reply_payload.sg_cnt, &cap, sizeof(cap)); - bsg_job->reply->reply_payload_rcv_len = sizeof(cap); + bsg_reply->reply_payload_rcv_len = sizeof(cap); - bsg_job->reply->reply_data.vendor_reply.vendor_rsp[0] = + bsg_reply->reply_data.vendor_reply.vendor_rsp[0] = EXT_STATUS_OK; bsg_job->reply_len = sizeof(struct fc_bsg_reply); - bsg_job->reply->result = DID_OK << 16; - bsg_job->job_done(bsg_job); + bsg_reply->result = DID_OK << 16; + bsg_job_done(bsg_job, bsg_reply->result, + bsg_reply->reply_payload_rcv_len); return 0; } static int -qla27xx_set_flash_upd_cap(struct fc_bsg_job *bsg_job) +qla27xx_set_flash_upd_cap(struct bsg_job *bsg_job) { - struct Scsi_Host *host = bsg_job->shost; + struct fc_bsg_reply *bsg_reply = bsg_job->reply; + struct Scsi_Host *host = fc_bsg_to_shost(bsg_job); scsi_qla_host_t *vha = shost_priv(host); struct qla_hw_data *ha = vha->hw; uint64_t online_fw_attr = 0; @@ -2158,32 +2208,34 @@ qla27xx_set_flash_upd_cap(struct fc_bsg_job *bsg_job) (uint64_t)ha->fw_attributes; if (online_fw_attr != cap.capabilities) { - bsg_job->reply->reply_data.vendor_reply.vendor_rsp[0] = + bsg_reply->reply_data.vendor_reply.vendor_rsp[0] = EXT_STATUS_INVALID_PARAM; return -EINVAL; } if (cap.outage_duration < MAX_LOOP_TIMEOUT) { - bsg_job->reply->reply_data.vendor_reply.vendor_rsp[0] = + bsg_reply->reply_data.vendor_reply.vendor_rsp[0] = EXT_STATUS_INVALID_PARAM; return -EINVAL; } - bsg_job->reply->reply_payload_rcv_len = 0; + bsg_reply->reply_payload_rcv_len = 0; - bsg_job->reply->reply_data.vendor_reply.vendor_rsp[0] = + bsg_reply->reply_data.vendor_reply.vendor_rsp[0] = EXT_STATUS_OK; bsg_job->reply_len = sizeof(struct fc_bsg_reply); - bsg_job->reply->result = DID_OK << 16; - bsg_job->job_done(bsg_job); + bsg_reply->result = DID_OK << 16; + bsg_job_done(bsg_job, bsg_reply->result, + bsg_reply->reply_payload_rcv_len); return 0; } static int -qla27xx_get_bbcr_data(struct fc_bsg_job *bsg_job) +qla27xx_get_bbcr_data(struct bsg_job *bsg_job) { - struct Scsi_Host *host = bsg_job->shost; + struct fc_bsg_reply *bsg_reply = bsg_job->reply; + struct Scsi_Host *host = fc_bsg_to_shost(bsg_job); scsi_qla_host_t *vha = shost_priv(host); struct qla_hw_data *ha = vha->hw; struct qla_bbcr_data bbcr; @@ -2227,27 +2279,30 @@ qla27xx_get_bbcr_data(struct fc_bsg_job *bsg_job) done: sg_copy_from_buffer(bsg_job->reply_payload.sg_list, bsg_job->reply_payload.sg_cnt, &bbcr, sizeof(bbcr)); - bsg_job->reply->reply_payload_rcv_len = sizeof(bbcr); + bsg_reply->reply_payload_rcv_len = sizeof(bbcr); - bsg_job->reply->reply_data.vendor_reply.vendor_rsp[0] = EXT_STATUS_OK; + bsg_reply->reply_data.vendor_reply.vendor_rsp[0] = EXT_STATUS_OK; bsg_job->reply_len = sizeof(struct fc_bsg_reply); - bsg_job->reply->result = DID_OK << 16; - bsg_job->job_done(bsg_job); + bsg_reply->result = DID_OK << 16; + bsg_job_done(bsg_job, bsg_reply->result, + bsg_reply->reply_payload_rcv_len); return 0; } static int -qla2x00_get_priv_stats(struct fc_bsg_job *bsg_job) +qla2x00_get_priv_stats(struct bsg_job *bsg_job) { - struct Scsi_Host *host = bsg_job->shost; + struct fc_bsg_request *bsg_request = bsg_job->request; + struct fc_bsg_reply *bsg_reply = bsg_job->reply; + struct Scsi_Host *host = fc_bsg_to_shost(bsg_job); scsi_qla_host_t *vha = shost_priv(host); struct qla_hw_data *ha = vha->hw; struct scsi_qla_host *base_vha = pci_get_drvdata(ha->pdev); struct link_statistics *stats = NULL; dma_addr_t stats_dma; int rval; - uint32_t *cmd = bsg_job->request->rqst_data.h_vendor.vendor_cmd; + uint32_t *cmd = bsg_request->rqst_data.h_vendor.vendor_cmd; uint options = cmd[0] == QL_VND_GET_PRIV_STATS_EX ? cmd[1] : 0; if (test_bit(UNLOADING, &vha->dpc_flags)) @@ -2281,13 +2336,14 @@ qla2x00_get_priv_stats(struct fc_bsg_job *bsg_job) bsg_job->reply_payload.sg_cnt, stats, sizeof(*stats)); } - bsg_job->reply->reply_payload_rcv_len = sizeof(*stats); - bsg_job->reply->reply_data.vendor_reply.vendor_rsp[0] = + bsg_reply->reply_payload_rcv_len = sizeof(*stats); + bsg_reply->reply_data.vendor_reply.vendor_rsp[0] = rval ? EXT_STATUS_MAILBOX : EXT_STATUS_OK; - bsg_job->reply_len = sizeof(*bsg_job->reply); - bsg_job->reply->result = DID_OK << 16; - bsg_job->job_done(bsg_job); + bsg_job->reply_len = sizeof(*bsg_reply); + bsg_reply->result = DID_OK << 16; + bsg_job_done(bsg_job, bsg_reply->result, + bsg_reply->reply_payload_rcv_len); dma_free_coherent(&ha->pdev->dev, sizeof(*stats), stats, stats_dma); @@ -2296,9 +2352,10 @@ qla2x00_get_priv_stats(struct fc_bsg_job *bsg_job) } static int -qla2x00_do_dport_diagnostics(struct fc_bsg_job *bsg_job) +qla2x00_do_dport_diagnostics(struct bsg_job *bsg_job) { - struct Scsi_Host *host = bsg_job->shost; + struct fc_bsg_reply *bsg_reply = bsg_job->reply; + struct Scsi_Host *host = fc_bsg_to_shost(bsg_job); scsi_qla_host_t *vha = shost_priv(host); int rval; struct qla_dport_diag *dd; @@ -2323,13 +2380,14 @@ qla2x00_do_dport_diagnostics(struct fc_bsg_job *bsg_job) bsg_job->reply_payload.sg_cnt, dd, sizeof(*dd)); } - bsg_job->reply->reply_payload_rcv_len = sizeof(*dd); - bsg_job->reply->reply_data.vendor_reply.vendor_rsp[0] = + bsg_reply->reply_payload_rcv_len = sizeof(*dd); + bsg_reply->reply_data.vendor_reply.vendor_rsp[0] = rval ? EXT_STATUS_MAILBOX : EXT_STATUS_OK; - bsg_job->reply_len = sizeof(*bsg_job->reply); - bsg_job->reply->result = DID_OK << 16; - bsg_job->job_done(bsg_job); + bsg_job->reply_len = sizeof(*bsg_reply); + bsg_reply->result = DID_OK << 16; + bsg_job_done(bsg_job, bsg_reply->result, + bsg_reply->reply_payload_rcv_len); kfree(dd); @@ -2337,9 +2395,11 @@ qla2x00_do_dport_diagnostics(struct fc_bsg_job *bsg_job) } static int -qla2x00_process_vendor_specific(struct fc_bsg_job *bsg_job) +qla2x00_process_vendor_specific(struct bsg_job *bsg_job) { - switch (bsg_job->request->rqst_data.h_vendor.vendor_cmd[0]) { + struct fc_bsg_request *bsg_request = bsg_job->request; + + switch (bsg_request->rqst_data.h_vendor.vendor_cmd[0]) { case QL_VND_LOOPBACK: return qla2x00_process_loopback(bsg_job); @@ -2413,36 +2473,38 @@ qla2x00_process_vendor_specific(struct fc_bsg_job *bsg_job) } int -qla24xx_bsg_request(struct fc_bsg_job *bsg_job) +qla24xx_bsg_request(struct bsg_job *bsg_job) { + struct fc_bsg_request *bsg_request = bsg_job->request; + struct fc_bsg_reply *bsg_reply = bsg_job->reply; int ret = -EINVAL; struct fc_rport *rport; struct Scsi_Host *host; scsi_qla_host_t *vha; /* In case no data transferred. */ - bsg_job->reply->reply_payload_rcv_len = 0; + bsg_reply->reply_payload_rcv_len = 0; - if (bsg_job->request->msgcode == FC_BSG_RPT_ELS) { - rport = bsg_job->rport; + if (bsg_request->msgcode == FC_BSG_RPT_ELS) { + rport = fc_bsg_to_rport(bsg_job); host = rport_to_shost(rport); vha = shost_priv(host); } else { - host = bsg_job->shost; + host = fc_bsg_to_shost(bsg_job); vha = shost_priv(host); } if (qla2x00_reset_active(vha)) { ql_dbg(ql_dbg_user, vha, 0x709f, "BSG: ISP abort active/needed -- cmd=%d.\n", - bsg_job->request->msgcode); + bsg_request->msgcode); return -EBUSY; } ql_dbg(ql_dbg_user, vha, 0x7000, - "Entered %s msgcode=0x%x.\n", __func__, bsg_job->request->msgcode); + "Entered %s msgcode=0x%x.\n", __func__, bsg_request->msgcode); - switch (bsg_job->request->msgcode) { + switch (bsg_request->msgcode) { case FC_BSG_RPT_ELS: case FC_BSG_HST_ELS_NOLOGIN: ret = qla2x00_process_els(bsg_job); @@ -2464,9 +2526,10 @@ qla24xx_bsg_request(struct fc_bsg_job *bsg_job) } int -qla24xx_bsg_timeout(struct fc_bsg_job *bsg_job) +qla24xx_bsg_timeout(struct bsg_job *bsg_job) { - scsi_qla_host_t *vha = shost_priv(bsg_job->shost); + struct fc_bsg_reply *bsg_reply = bsg_job->reply; + scsi_qla_host_t *vha = shost_priv(fc_bsg_to_shost(bsg_job)); struct qla_hw_data *ha = vha->hw; srb_t *sp; int cnt, que; @@ -2494,13 +2557,13 @@ qla24xx_bsg_timeout(struct fc_bsg_job *bsg_job) "mbx abort_command " "failed.\n"); bsg_job->req->errors = - bsg_job->reply->result = -EIO; + bsg_reply->result = -EIO; } else { ql_dbg(ql_dbg_user, vha, 0x708a, "mbx abort_command " "success.\n"); bsg_job->req->errors = - bsg_job->reply->result = 0; + bsg_reply->result = 0; } spin_lock_irqsave(&ha->hardware_lock, flags); goto done; @@ -2510,7 +2573,7 @@ qla24xx_bsg_timeout(struct fc_bsg_job *bsg_job) } spin_unlock_irqrestore(&ha->hardware_lock, flags); ql_log(ql_log_info, vha, 0x708b, "SRB not found to abort.\n"); - bsg_job->req->errors = bsg_job->reply->result = -ENXIO; + bsg_job->req->errors = bsg_reply->result = -ENXIO; return 0; done: diff --git a/drivers/scsi/qla2xxx/qla_def.h b/drivers/scsi/qla2xxx/qla_def.h index 73b12e4..5236e3f 100644 --- a/drivers/scsi/qla2xxx/qla_def.h +++ b/drivers/scsi/qla2xxx/qla_def.h @@ -403,7 +403,7 @@ typedef struct srb { int iocbs; union { struct srb_iocb iocb_cmd; - struct fc_bsg_job *bsg_job; + struct bsg_job *bsg_job; struct srb_cmd scmd; } u; void (*done)(void *, void *, int); diff --git a/drivers/scsi/qla2xxx/qla_gbl.h b/drivers/scsi/qla2xxx/qla_gbl.h index 6ca0081..c51d9f3 100644 --- a/drivers/scsi/qla2xxx/qla_gbl.h +++ b/drivers/scsi/qla2xxx/qla_gbl.h @@ -733,8 +733,8 @@ extern int qla82xx_read_temperature(scsi_qla_host_t *); extern int qla8044_read_temperature(scsi_qla_host_t *); /* BSG related functions */ -extern int qla24xx_bsg_request(struct fc_bsg_job *); -extern int qla24xx_bsg_timeout(struct fc_bsg_job *); +extern int qla24xx_bsg_request(struct bsg_job *); +extern int qla24xx_bsg_timeout(struct bsg_job *); extern int qla84xx_reset_chip(scsi_qla_host_t *, uint16_t); extern int qla2x00_issue_iocb_timeout(scsi_qla_host_t *, void *, dma_addr_t, size_t, uint32_t); diff --git a/drivers/scsi/qla2xxx/qla_iocb.c b/drivers/scsi/qla2xxx/qla_iocb.c index b41265a..221ad89 100644 --- a/drivers/scsi/qla2xxx/qla_iocb.c +++ b/drivers/scsi/qla2xxx/qla_iocb.c @@ -2197,7 +2197,8 @@ qla24xx_els_logo_iocb(srb_t *sp, struct els_entry_24xx *els_iocb) static void qla24xx_els_iocb(srb_t *sp, struct els_entry_24xx *els_iocb) { - struct fc_bsg_job *bsg_job = sp->u.bsg_job; + struct bsg_job *bsg_job = sp->u.bsg_job; + struct fc_bsg_request *bsg_request = bsg_job->request; els_iocb->entry_type = ELS_IOCB_TYPE; els_iocb->entry_count = 1; @@ -2212,8 +2213,8 @@ qla24xx_els_iocb(srb_t *sp, struct els_entry_24xx *els_iocb) els_iocb->opcode = sp->type == SRB_ELS_CMD_RPT ? - bsg_job->request->rqst_data.r_els.els_code : - bsg_job->request->rqst_data.h_els.command_code; + bsg_request->rqst_data.r_els.els_code : + bsg_request->rqst_data.h_els.command_code; els_iocb->port_id[0] = sp->fcport->d_id.b.al_pa; els_iocb->port_id[1] = sp->fcport->d_id.b.area; els_iocb->port_id[2] = sp->fcport->d_id.b.domain; @@ -2250,7 +2251,7 @@ qla2x00_ct_iocb(srb_t *sp, ms_iocb_entry_t *ct_iocb) uint16_t tot_dsds; scsi_qla_host_t *vha = sp->fcport->vha; struct qla_hw_data *ha = vha->hw; - struct fc_bsg_job *bsg_job = sp->u.bsg_job; + struct bsg_job *bsg_job = sp->u.bsg_job; int loop_iterartion = 0; int entry_count = 1; @@ -2327,7 +2328,7 @@ qla24xx_ct_iocb(srb_t *sp, struct ct_entry_24xx *ct_iocb) uint16_t tot_dsds; scsi_qla_host_t *vha = sp->fcport->vha; struct qla_hw_data *ha = vha->hw; - struct fc_bsg_job *bsg_job = sp->u.bsg_job; + struct bsg_job *bsg_job = sp->u.bsg_job; int loop_iterartion = 0; int entry_count = 1; @@ -2833,7 +2834,7 @@ qla25xx_build_bidir_iocb(srb_t *sp, struct scsi_qla_host *vha, struct scatterlist *sg; int index; int entry_count = 1; - struct fc_bsg_job *bsg_job = sp->u.bsg_job; + struct bsg_job *bsg_job = sp->u.bsg_job; /*Update entry type to indicate bidir command */ *((uint32_t *)(&cmd_pkt->entry_type)) = diff --git a/drivers/scsi/qla2xxx/qla_isr.c b/drivers/scsi/qla2xxx/qla_isr.c index 068c4e4..19f1848 100644 --- a/drivers/scsi/qla2xxx/qla_isr.c +++ b/drivers/scsi/qla2xxx/qla_isr.c @@ -1356,7 +1356,8 @@ qla2x00_ct_entry(scsi_qla_host_t *vha, struct req_que *req, const char func[] = "CT_IOCB"; const char *type; srb_t *sp; - struct fc_bsg_job *bsg_job; + struct bsg_job *bsg_job; + struct fc_bsg_reply *bsg_reply; uint16_t comp_status; int res; @@ -1365,6 +1366,7 @@ qla2x00_ct_entry(scsi_qla_host_t *vha, struct req_que *req, return; bsg_job = sp->u.bsg_job; + bsg_reply = bsg_job->reply; type = "ct pass-through"; @@ -1373,32 +1375,32 @@ qla2x00_ct_entry(scsi_qla_host_t *vha, struct req_que *req, /* return FC_CTELS_STATUS_OK and leave the decoding of the ELS/CT * fc payload to the caller */ - bsg_job->reply->reply_data.ctels_reply.status = FC_CTELS_STATUS_OK; + bsg_reply->reply_data.ctels_reply.status = FC_CTELS_STATUS_OK; bsg_job->reply_len = sizeof(struct fc_bsg_reply); if (comp_status != CS_COMPLETE) { if (comp_status == CS_DATA_UNDERRUN) { res = DID_OK << 16; - bsg_job->reply->reply_payload_rcv_len = + bsg_reply->reply_payload_rcv_len = le16_to_cpu(((sts_entry_t *)pkt)->rsp_info_len); ql_log(ql_log_warn, vha, 0x5048, "CT pass-through-%s error " "comp_status-status=0x%x total_byte = 0x%x.\n", type, comp_status, - bsg_job->reply->reply_payload_rcv_len); + bsg_reply->reply_payload_rcv_len); } else { ql_log(ql_log_warn, vha, 0x5049, "CT pass-through-%s error " "comp_status-status=0x%x.\n", type, comp_status); res = DID_ERROR << 16; - bsg_job->reply->reply_payload_rcv_len = 0; + bsg_reply->reply_payload_rcv_len = 0; } ql_dump_buffer(ql_dbg_async + ql_dbg_buffer, vha, 0x5035, (uint8_t *)pkt, sizeof(*pkt)); } else { res = DID_OK << 16; - bsg_job->reply->reply_payload_rcv_len = + bsg_reply->reply_payload_rcv_len = bsg_job->reply_payload.payload_len; bsg_job->reply_len = 0; } @@ -1413,7 +1415,8 @@ qla24xx_els_ct_entry(scsi_qla_host_t *vha, struct req_que *req, const char func[] = "ELS_CT_IOCB"; const char *type; srb_t *sp; - struct fc_bsg_job *bsg_job; + struct bsg_job *bsg_job; + struct fc_bsg_reply *bsg_reply; uint16_t comp_status; uint32_t fw_status[3]; uint8_t* fw_sts_ptr; @@ -1423,6 +1426,7 @@ qla24xx_els_ct_entry(scsi_qla_host_t *vha, struct req_que *req, if (!sp) return; bsg_job = sp->u.bsg_job; + bsg_reply = bsg_job->reply; type = NULL; switch (sp->type) { @@ -1452,13 +1456,13 @@ qla24xx_els_ct_entry(scsi_qla_host_t *vha, struct req_que *req, /* return FC_CTELS_STATUS_OK and leave the decoding of the ELS/CT * fc payload to the caller */ - bsg_job->reply->reply_data.ctels_reply.status = FC_CTELS_STATUS_OK; + bsg_reply->reply_data.ctels_reply.status = FC_CTELS_STATUS_OK; bsg_job->reply_len = sizeof(struct fc_bsg_reply) + sizeof(fw_status); if (comp_status != CS_COMPLETE) { if (comp_status == CS_DATA_UNDERRUN) { res = DID_OK << 16; - bsg_job->reply->reply_payload_rcv_len = + bsg_reply->reply_payload_rcv_len = le16_to_cpu(((struct els_sts_entry_24xx *)pkt)->total_byte_count); ql_dbg(ql_dbg_user, vha, 0x503f, @@ -1480,7 +1484,7 @@ qla24xx_els_ct_entry(scsi_qla_host_t *vha, struct req_que *req, le16_to_cpu(((struct els_sts_entry_24xx *) pkt)->error_subcode_2)); res = DID_ERROR << 16; - bsg_job->reply->reply_payload_rcv_len = 0; + bsg_reply->reply_payload_rcv_len = 0; fw_sts_ptr = ((uint8_t*)bsg_job->req->sense) + sizeof(struct fc_bsg_reply); memcpy( fw_sts_ptr, fw_status, sizeof(fw_status)); } @@ -1489,7 +1493,7 @@ qla24xx_els_ct_entry(scsi_qla_host_t *vha, struct req_que *req, } else { res = DID_OK << 16; - bsg_job->reply->reply_payload_rcv_len = bsg_job->reply_payload.payload_len; + bsg_reply->reply_payload_rcv_len = bsg_job->reply_payload.payload_len; bsg_job->reply_len = 0; } @@ -1904,7 +1908,9 @@ qla25xx_process_bidir_status_iocb(scsi_qla_host_t *vha, void *pkt, uint16_t scsi_status; uint16_t thread_id; uint32_t rval = EXT_STATUS_OK; - struct fc_bsg_job *bsg_job = NULL; + struct bsg_job *bsg_job = NULL; + struct fc_bsg_request *bsg_request; + struct fc_bsg_reply *bsg_reply; sts_entry_t *sts; struct sts_entry_24xx *sts24; sts = (sts_entry_t *) pkt; @@ -1919,11 +1925,7 @@ qla25xx_process_bidir_status_iocb(scsi_qla_host_t *vha, void *pkt, } sp = req->outstanding_cmds[index]; - if (sp) { - /* Free outstanding command slot. */ - req->outstanding_cmds[index] = NULL; - bsg_job = sp->u.bsg_job; - } else { + if (!sp) { ql_log(ql_log_warn, vha, 0x70b0, "Req:%d: Invalid ISP SCSI completion handle(0x%x)\n", req->id, index); @@ -1932,6 +1934,12 @@ qla25xx_process_bidir_status_iocb(scsi_qla_host_t *vha, void *pkt, return; } + /* Free outstanding command slot. */ + req->outstanding_cmds[index] = NULL; + bsg_job = sp->u.bsg_job; + bsg_request = bsg_job->request; + bsg_reply = bsg_job->reply; + if (IS_FWI2_CAPABLE(ha)) { comp_status = le16_to_cpu(sts24->comp_status); scsi_status = le16_to_cpu(sts24->scsi_status) & SS_MASK; @@ -1940,14 +1948,14 @@ qla25xx_process_bidir_status_iocb(scsi_qla_host_t *vha, void *pkt, scsi_status = le16_to_cpu(sts->scsi_status) & SS_MASK; } - thread_id = bsg_job->request->rqst_data.h_vendor.vendor_cmd[1]; + thread_id = bsg_request->rqst_data.h_vendor.vendor_cmd[1]; switch (comp_status) { case CS_COMPLETE: if (scsi_status == 0) { - bsg_job->reply->reply_payload_rcv_len = + bsg_reply->reply_payload_rcv_len = bsg_job->reply_payload.payload_len; vha->qla_stats.input_bytes += - bsg_job->reply->reply_payload_rcv_len; + bsg_reply->reply_payload_rcv_len; vha->qla_stats.input_requests++; rval = EXT_STATUS_OK; } @@ -2028,11 +2036,11 @@ qla25xx_process_bidir_status_iocb(scsi_qla_host_t *vha, void *pkt, rval = EXT_STATUS_ERR; break; } - bsg_job->reply->reply_payload_rcv_len = 0; + bsg_reply->reply_payload_rcv_len = 0; done: /* Return the vendor specific reply to API */ - bsg_job->reply->reply_data.vendor_reply.vendor_rsp[0] = rval; + bsg_reply->reply_data.vendor_reply.vendor_rsp[0] = rval; bsg_job->reply_len = sizeof(struct fc_bsg_reply); /* Always return DID_OK, bsg will send the vendor specific response * in this case only */ diff --git a/drivers/scsi/qla2xxx/qla_mr.c b/drivers/scsi/qla2xxx/qla_mr.c index 15dff70..02f1de1 100644 --- a/drivers/scsi/qla2xxx/qla_mr.c +++ b/drivers/scsi/qla2xxx/qla_mr.c @@ -10,6 +10,7 @@ #include <linux/pci.h> #include <linux/ratelimit.h> #include <linux/vmalloc.h> +#include <linux/bsg-lib.h> #include <scsi/scsi_tcq.h> #include <linux/utsname.h> @@ -2206,7 +2207,8 @@ qlafx00_ioctl_iosb_entry(scsi_qla_host_t *vha, struct req_que *req, { const char func[] = "IOSB_IOCB"; srb_t *sp; - struct fc_bsg_job *bsg_job; + struct bsg_job *bsg_job; + struct fc_bsg_reply *bsg_reply; struct srb_iocb *iocb_job; int res; struct qla_mt_iocb_rsp_fx00 fstatus; @@ -2226,6 +2228,7 @@ qlafx00_ioctl_iosb_entry(scsi_qla_host_t *vha, struct req_que *req, pkt->dataword_r; } else { bsg_job = sp->u.bsg_job; + bsg_reply = bsg_job->reply; memset(&fstatus, 0, sizeof(struct qla_mt_iocb_rsp_fx00)); @@ -2257,8 +2260,8 @@ qlafx00_ioctl_iosb_entry(scsi_qla_host_t *vha, struct req_que *req, sp->fcport->vha, 0x5074, (uint8_t *)fw_sts_ptr, sizeof(struct qla_mt_iocb_rsp_fx00)); - res = bsg_job->reply->result = DID_OK << 16; - bsg_job->reply->reply_payload_rcv_len = + res = bsg_reply->result = DID_OK << 16; + bsg_reply->reply_payload_rcv_len = bsg_job->reply_payload.payload_len; } sp->done(vha, sp, res); @@ -3252,7 +3255,8 @@ qlafx00_fxdisc_iocb(srb_t *sp, struct fxdisc_entry_fx00 *pfxiocb) { struct srb_iocb *fxio = &sp->u.iocb_cmd; struct qla_mt_iocb_rqst_fx00 *piocb_rqst; - struct fc_bsg_job *bsg_job; + struct bsg_job *bsg_job; + struct fc_bsg_request *bsg_request; struct fxdisc_entry_fx00 fx_iocb; uint8_t entry_cnt = 1; @@ -3301,8 +3305,9 @@ qlafx00_fxdisc_iocb(srb_t *sp, struct fxdisc_entry_fx00 *pfxiocb) } else { struct scatterlist *sg; bsg_job = sp->u.bsg_job; + bsg_request = bsg_job->request; piocb_rqst = (struct qla_mt_iocb_rqst_fx00 *) - &bsg_job->request->rqst_data.h_vendor.vendor_cmd[1]; + &bsg_request->rqst_data.h_vendor.vendor_cmd[1]; fx_iocb.func_num = piocb_rqst->func_type; fx_iocb.adapid = piocb_rqst->adapid; diff --git a/drivers/scsi/qla4xxx/ql4_def.h b/drivers/scsi/qla4xxx/ql4_def.h index a7cfc27..aeebefb 100644 --- a/drivers/scsi/qla4xxx/ql4_def.h +++ b/drivers/scsi/qla4xxx/ql4_def.h @@ -409,18 +409,9 @@ struct qla4_8xxx_legacy_intr_set { /* MSI-X Support */ -#define QLA_MSIX_DEFAULT 0x00 -#define QLA_MSIX_RSP_Q 0x01 - +#define QLA_MSIX_DEFAULT 0 +#define QLA_MSIX_RSP_Q 1 #define QLA_MSIX_ENTRIES 2 -#define QLA_MIDX_DEFAULT 0 -#define QLA_MIDX_RSP_Q 1 - -struct ql4_msix_entry { - int have_irq; - uint16_t msix_vector; - uint16_t msix_entry; -}; /* * ISP Operations @@ -572,9 +563,6 @@ struct scsi_qla_host { #define AF_IRQ_ATTACHED 10 /* 0x00000400 */ #define AF_DISABLE_ACB_COMPLETE 11 /* 0x00000800 */ #define AF_HA_REMOVAL 12 /* 0x00001000 */ -#define AF_INTx_ENABLED 15 /* 0x00008000 */ -#define AF_MSI_ENABLED 16 /* 0x00010000 */ -#define AF_MSIX_ENABLED 17 /* 0x00020000 */ #define AF_MBOX_COMMAND_NOPOLL 18 /* 0x00040000 */ #define AF_FW_RECOVERY 19 /* 0x00080000 */ #define AF_EEH_BUSY 20 /* 0x00100000 */ @@ -762,8 +750,6 @@ struct scsi_qla_host { struct isp_operations *isp_ops; struct ql82xx_hw_data hw; - struct ql4_msix_entry msix_entries[QLA_MSIX_ENTRIES]; - uint32_t nx_dev_init_timeout; uint32_t nx_reset_timeout; void *fw_dump; diff --git a/drivers/scsi/qla4xxx/ql4_glbl.h b/drivers/scsi/qla4xxx/ql4_glbl.h index 2559144..bce96a5 100644 --- a/drivers/scsi/qla4xxx/ql4_glbl.h +++ b/drivers/scsi/qla4xxx/ql4_glbl.h @@ -134,7 +134,6 @@ int qla4_8xxx_get_flash_info(struct scsi_qla_host *ha); void qla4_82xx_enable_intrs(struct scsi_qla_host *ha); void qla4_82xx_disable_intrs(struct scsi_qla_host *ha); int qla4_8xxx_enable_msix(struct scsi_qla_host *ha); -void qla4_8xxx_disable_msix(struct scsi_qla_host *ha); irqreturn_t qla4_8xxx_msi_handler(int irq, void *dev_id); irqreturn_t qla4_8xxx_default_intr_handler(int irq, void *dev_id); irqreturn_t qla4_8xxx_msix_rsp_q(int irq, void *dev_id); diff --git a/drivers/scsi/qla4xxx/ql4_isr.c b/drivers/scsi/qla4xxx/ql4_isr.c index 4f9c0f2..d2cd33d 100644 --- a/drivers/scsi/qla4xxx/ql4_isr.c +++ b/drivers/scsi/qla4xxx/ql4_isr.c @@ -1107,7 +1107,7 @@ static void qla4_82xx_spurious_interrupt(struct scsi_qla_host *ha, DEBUG2(ql4_printk(KERN_INFO, ha, "Spurious Interrupt\n")); if (is_qla8022(ha)) { writel(0, &ha->qla4_82xx_reg->host_int); - if (test_bit(AF_INTx_ENABLED, &ha->flags)) + if (!ha->pdev->msi_enabled && !ha->pdev->msix_enabled) qla4_82xx_wr_32(ha, ha->nx_legacy_intr.tgt_mask_reg, 0xfbff); } @@ -1564,19 +1564,18 @@ int qla4xxx_request_irqs(struct scsi_qla_host *ha) try_msi: /* Trying MSI */ - ret = pci_enable_msi(ha->pdev); - if (!ret) { + ret = pci_alloc_irq_vectors(ha->pdev, 1, 1, PCI_IRQ_MSI); + if (ret > 0) { ret = request_irq(ha->pdev->irq, qla4_8xxx_msi_handler, 0, DRIVER_NAME, ha); if (!ret) { DEBUG2(ql4_printk(KERN_INFO, ha, "MSI: Enabled.\n")); - set_bit(AF_MSI_ENABLED, &ha->flags); goto irq_attached; } else { ql4_printk(KERN_WARNING, ha, "MSI: Failed to reserve interrupt %d " "already in use.\n", ha->pdev->irq); - pci_disable_msi(ha->pdev); + pci_free_irq_vectors(ha->pdev); } } @@ -1592,7 +1591,6 @@ try_intx: IRQF_SHARED, DRIVER_NAME, ha); if (!ret) { DEBUG2(ql4_printk(KERN_INFO, ha, "INTx: Enabled.\n")); - set_bit(AF_INTx_ENABLED, &ha->flags); goto irq_attached; } else { @@ -1614,14 +1612,11 @@ irq_not_attached: void qla4xxx_free_irqs(struct scsi_qla_host *ha) { - if (test_and_clear_bit(AF_IRQ_ATTACHED, &ha->flags)) { - if (test_bit(AF_MSIX_ENABLED, &ha->flags)) { - qla4_8xxx_disable_msix(ha); - } else if (test_and_clear_bit(AF_MSI_ENABLED, &ha->flags)) { - free_irq(ha->pdev->irq, ha); - pci_disable_msi(ha->pdev); - } else if (test_and_clear_bit(AF_INTx_ENABLED, &ha->flags)) { - free_irq(ha->pdev->irq, ha); - } - } + if (!test_and_clear_bit(AF_IRQ_ATTACHED, &ha->flags)) + return; + + if (ha->pdev->msix_enabled) + free_irq(pci_irq_vector(ha->pdev, 1), ha); + free_irq(pci_irq_vector(ha->pdev, 0), ha); + pci_free_irq_vectors(ha->pdev); } diff --git a/drivers/scsi/qla4xxx/ql4_mbx.c b/drivers/scsi/qla4xxx/ql4_mbx.c index c291fdf..1da04f3 100644 --- a/drivers/scsi/qla4xxx/ql4_mbx.c +++ b/drivers/scsi/qla4xxx/ql4_mbx.c @@ -2032,10 +2032,7 @@ int qla4xxx_set_param_ddbentry(struct scsi_qla_host *ha, ptid = (uint16_t *)&fw_ddb_entry->isid[1]; *ptid = cpu_to_le16((uint16_t)ddb_entry->sess->target_id); - DEBUG2(ql4_printk(KERN_INFO, ha, "ISID [%02x%02x%02x%02x%02x%02x]\n", - fw_ddb_entry->isid[5], fw_ddb_entry->isid[4], - fw_ddb_entry->isid[3], fw_ddb_entry->isid[2], - fw_ddb_entry->isid[1], fw_ddb_entry->isid[0])); + DEBUG2(ql4_printk(KERN_INFO, ha, "ISID [%pmR]\n", fw_ddb_entry->isid)); iscsi_opts = le16_to_cpu(fw_ddb_entry->iscsi_options); memset(fw_ddb_entry->iscsi_alias, 0, sizeof(fw_ddb_entry->iscsi_alias)); diff --git a/drivers/scsi/qla4xxx/ql4_nx.c b/drivers/scsi/qla4xxx/ql4_nx.c index 06ddd13..e91abb3 100644 --- a/drivers/scsi/qla4xxx/ql4_nx.c +++ b/drivers/scsi/qla4xxx/ql4_nx.c @@ -3945,7 +3945,7 @@ void qla4_82xx_process_mbox_intr(struct scsi_qla_host *ha, int out_count) ha->isp_ops->interrupt_service_routine(ha, intr_status); if (test_bit(AF_INTERRUPTS_ON, &ha->flags) && - test_bit(AF_INTx_ENABLED, &ha->flags)) + (!ha->pdev->msi_enabled && !ha->pdev->msix_enabled)) qla4_82xx_wr_32(ha, ha->nx_legacy_intr.tgt_mask_reg, 0xfbff); } @@ -4094,12 +4094,8 @@ int qla4_8xxx_get_sys_info(struct scsi_qla_host *ha) ha->phy_port_num = sys_info->port_num; ha->iscsi_pci_func_cnt = sys_info->iscsi_pci_func_cnt; - DEBUG2(printk("scsi%ld: %s: " - "mac %02x:%02x:%02x:%02x:%02x:%02x " - "serial %s\n", ha->host_no, __func__, - ha->my_mac[0], ha->my_mac[1], ha->my_mac[2], - ha->my_mac[3], ha->my_mac[4], ha->my_mac[5], - ha->serial_number)); + DEBUG2(printk("scsi%ld: %s: mac %pM serial %s\n", + ha->host_no, __func__, ha->my_mac, ha->serial_number)); status = QLA_SUCCESS; @@ -4178,78 +4174,37 @@ qla4_82xx_disable_intrs(struct scsi_qla_host *ha) spin_unlock_irq(&ha->hardware_lock); } -struct ql4_init_msix_entry { - uint16_t entry; - uint16_t index; - const char *name; - irq_handler_t handler; -}; - -static struct ql4_init_msix_entry qla4_8xxx_msix_entries[QLA_MSIX_ENTRIES] = { - { QLA_MSIX_DEFAULT, QLA_MIDX_DEFAULT, - "qla4xxx (default)", - (irq_handler_t)qla4_8xxx_default_intr_handler }, - { QLA_MSIX_RSP_Q, QLA_MIDX_RSP_Q, - "qla4xxx (rsp_q)", (irq_handler_t)qla4_8xxx_msix_rsp_q }, -}; - -void -qla4_8xxx_disable_msix(struct scsi_qla_host *ha) -{ - int i; - struct ql4_msix_entry *qentry; - - for (i = 0; i < QLA_MSIX_ENTRIES; i++) { - qentry = &ha->msix_entries[qla4_8xxx_msix_entries[i].index]; - if (qentry->have_irq) { - free_irq(qentry->msix_vector, ha); - DEBUG2(ql4_printk(KERN_INFO, ha, "%s: %s\n", - __func__, qla4_8xxx_msix_entries[i].name)); - } - } - pci_disable_msix(ha->pdev); - clear_bit(AF_MSIX_ENABLED, &ha->flags); -} - int qla4_8xxx_enable_msix(struct scsi_qla_host *ha) { - int i, ret; - struct msix_entry entries[QLA_MSIX_ENTRIES]; - struct ql4_msix_entry *qentry; - - for (i = 0; i < QLA_MSIX_ENTRIES; i++) - entries[i].entry = qla4_8xxx_msix_entries[i].entry; + int ret; - ret = pci_enable_msix_exact(ha->pdev, entries, ARRAY_SIZE(entries)); - if (ret) { + ret = pci_alloc_irq_vectors(ha->pdev, QLA_MSIX_ENTRIES, + QLA_MSIX_ENTRIES, PCI_IRQ_MSIX); + if (ret < 0) { ql4_printk(KERN_WARNING, ha, "MSI-X: Failed to enable support -- %d/%d\n", QLA_MSIX_ENTRIES, ret); - goto msix_out; - } - set_bit(AF_MSIX_ENABLED, &ha->flags); - - for (i = 0; i < QLA_MSIX_ENTRIES; i++) { - qentry = &ha->msix_entries[qla4_8xxx_msix_entries[i].index]; - qentry->msix_vector = entries[i].vector; - qentry->msix_entry = entries[i].entry; - qentry->have_irq = 0; - ret = request_irq(qentry->msix_vector, - qla4_8xxx_msix_entries[i].handler, 0, - qla4_8xxx_msix_entries[i].name, ha); - if (ret) { - ql4_printk(KERN_WARNING, ha, - "MSI-X: Unable to register handler -- %x/%d.\n", - qla4_8xxx_msix_entries[i].index, ret); - qla4_8xxx_disable_msix(ha); - goto msix_out; - } - qentry->have_irq = 1; - DEBUG2(ql4_printk(KERN_INFO, ha, "%s: %s\n", - __func__, qla4_8xxx_msix_entries[i].name)); + return ret; } -msix_out: + + ret = request_irq(pci_irq_vector(ha->pdev, 0), + qla4_8xxx_default_intr_handler, 0, "qla4xxx (default)", + ha); + if (ret) + goto out_free_vectors; + + ret = request_irq(pci_irq_vector(ha->pdev, 1), + qla4_8xxx_msix_rsp_q, 0, "qla4xxx (rsp_q)", ha); + if (ret) + goto out_free_default_irq; + + return 0; + +out_free_default_irq: + free_irq(pci_irq_vector(ha->pdev, 0), ha); +out_free_vectors: + pci_free_irq_vectors(ha->pdev); return ret; } diff --git a/drivers/scsi/qla4xxx/ql4_os.c b/drivers/scsi/qla4xxx/ql4_os.c index 01c3610..9fbb33f 100644 --- a/drivers/scsi/qla4xxx/ql4_os.c +++ b/drivers/scsi/qla4xxx/ql4_os.c @@ -6304,13 +6304,9 @@ static int qla4xxx_compare_tuple_ddb(struct scsi_qla_host *ha, * ISID would not match firmware generated ISID. */ if (is_isid_compare) { - DEBUG2(ql4_printk(KERN_INFO, ha, "%s: old ISID [%02x%02x%02x" - "%02x%02x%02x] New ISID [%02x%02x%02x%02x%02x%02x]\n", - __func__, old_tddb->isid[5], old_tddb->isid[4], - old_tddb->isid[3], old_tddb->isid[2], old_tddb->isid[1], - old_tddb->isid[0], new_tddb->isid[5], new_tddb->isid[4], - new_tddb->isid[3], new_tddb->isid[2], new_tddb->isid[1], - new_tddb->isid[0])); + DEBUG2(ql4_printk(KERN_INFO, ha, + "%s: old ISID [%pmR] New ISID [%pmR]\n", + __func__, old_tddb->isid, new_tddb->isid)); if (memcmp(&old_tddb->isid[0], &new_tddb->isid[0], sizeof(old_tddb->isid))) @@ -7925,10 +7921,7 @@ qla4xxx_sysfs_ddb_get_param(struct iscsi_bus_flash_session *fnode_sess, rc = sprintf(buf, "%u\n", fnode_conn->keepalive_timeout); break; case ISCSI_FLASHNODE_ISID: - rc = sprintf(buf, "%02x%02x%02x%02x%02x%02x\n", - fnode_sess->isid[0], fnode_sess->isid[1], - fnode_sess->isid[2], fnode_sess->isid[3], - fnode_sess->isid[4], fnode_sess->isid[5]); + rc = sprintf(buf, "%pm\n", fnode_sess->isid); break; case ISCSI_FLASHNODE_TSID: rc = sprintf(buf, "%u\n", fnode_sess->tsid); diff --git a/drivers/scsi/scsi_devinfo.c b/drivers/scsi/scsi_devinfo.c index 2464569..28fea83 100644 --- a/drivers/scsi/scsi_devinfo.c +++ b/drivers/scsi/scsi_devinfo.c @@ -220,8 +220,6 @@ static struct { {"NAKAMICH", "MJ-5.16S", NULL, BLIST_FORCELUN | BLIST_SINGLELUN}, {"NEC", "PD-1 ODX654P", NULL, BLIST_FORCELUN | BLIST_SINGLELUN}, {"NEC", "iStorage", NULL, BLIST_REPORTLUN2}, - {"NETAPP", "LUN C-Mode", NULL, BLIST_SYNC_ALUA}, - {"NETAPP", "INF-01-00", NULL, BLIST_SYNC_ALUA}, {"NRC", "MBR-7", NULL, BLIST_FORCELUN | BLIST_SINGLELUN}, {"NRC", "MBR-7.4", NULL, BLIST_FORCELUN | BLIST_SINGLELUN}, {"PIONEER", "CD-ROM DRM-600", NULL, BLIST_FORCELUN | BLIST_SINGLELUN}, diff --git a/drivers/scsi/scsi_lib.c b/drivers/scsi/scsi_lib.c index 9a8ccff..c35b6de 100644 --- a/drivers/scsi/scsi_lib.c +++ b/drivers/scsi/scsi_lib.c @@ -1998,6 +1998,15 @@ static void scsi_exit_request(void *data, struct request *rq, kfree(cmd->sense_buffer); } +static int scsi_map_queues(struct blk_mq_tag_set *set) +{ + struct Scsi_Host *shost = container_of(set, struct Scsi_Host, tag_set); + + if (shost->hostt->map_queues) + return shost->hostt->map_queues(shost); + return blk_mq_map_queues(set); +} + static u64 scsi_calculate_bounce_limit(struct Scsi_Host *shost) { struct device *host_dev; @@ -2090,6 +2099,7 @@ static struct blk_mq_ops scsi_mq_ops = { .timeout = scsi_timeout, .init_request = scsi_init_request, .exit_request = scsi_exit_request, + .map_queues = scsi_map_queues, }; struct request_queue *scsi_mq_alloc_queue(struct scsi_device *sdev) @@ -2732,6 +2742,39 @@ void sdev_evt_send_simple(struct scsi_device *sdev, EXPORT_SYMBOL_GPL(sdev_evt_send_simple); /** + * scsi_request_fn_active() - number of kernel threads inside scsi_request_fn() + * @sdev: SCSI device to count the number of scsi_request_fn() callers for. + */ +static int scsi_request_fn_active(struct scsi_device *sdev) +{ + struct request_queue *q = sdev->request_queue; + int request_fn_active; + + WARN_ON_ONCE(sdev->host->use_blk_mq); + + spin_lock_irq(q->queue_lock); + request_fn_active = q->request_fn_active; + spin_unlock_irq(q->queue_lock); + + return request_fn_active; +} + +/** + * scsi_wait_for_queuecommand() - wait for ongoing queuecommand() calls + * @sdev: SCSI device pointer. + * + * Wait until the ongoing shost->hostt->queuecommand() calls that are + * invoked from scsi_request_fn() have finished. + */ +static void scsi_wait_for_queuecommand(struct scsi_device *sdev) +{ + WARN_ON_ONCE(sdev->host->use_blk_mq); + + while (scsi_request_fn_active(sdev)) + msleep(20); +} + +/** * scsi_device_quiesce - Block user issued commands. * @sdev: scsi device to quiesce. * @@ -2815,8 +2858,7 @@ EXPORT_SYMBOL(scsi_target_resume); * @sdev: device to block * * Block request made by scsi lld's to temporarily stop all - * scsi commands on the specified device. Called from interrupt - * or normal process context. + * scsi commands on the specified device. May sleep. * * Returns zero if successful or error if not * @@ -2825,6 +2867,10 @@ EXPORT_SYMBOL(scsi_target_resume); * (which must be a legal transition). When the device is in this * state, all commands are deferred until the scsi lld reenables * the device with scsi_device_unblock or device_block_tmo fires. + * + * To do: avoid that scsi_send_eh_cmnd() calls queuecommand() after + * scsi_internal_device_block() has blocked a SCSI device and also + * remove the rport mutex lock and unlock calls from srp_queuecommand(). */ int scsi_internal_device_block(struct scsi_device *sdev) @@ -2852,6 +2898,7 @@ scsi_internal_device_block(struct scsi_device *sdev) spin_lock_irqsave(q->queue_lock, flags); blk_stop_queue(q); spin_unlock_irqrestore(q->queue_lock, flags); + scsi_wait_for_queuecommand(sdev); } return 0; diff --git a/drivers/scsi/scsi_transport_fc.c b/drivers/scsi/scsi_transport_fc.c index 0f3a386..03577bd 100644 --- a/drivers/scsi/scsi_transport_fc.c +++ b/drivers/scsi/scsi_transport_fc.c @@ -30,6 +30,7 @@ #include <linux/slab.h> #include <linux/delay.h> #include <linux/kernel.h> +#include <linux/bsg-lib.h> #include <scsi/scsi_device.h> #include <scsi/scsi_host.h> #include <scsi/scsi_transport.h> @@ -2592,7 +2593,7 @@ fc_rport_final_delete(struct work_struct *work) /** - * fc_rport_create - allocates and creates a remote FC port. + * fc_remote_port_create - allocates and creates a remote FC port. * @shost: scsi host the remote port is connected to. * @channel: Channel on shost port connected to. * @ids: The world wide names, fc address, and FC4 port @@ -2605,8 +2606,8 @@ fc_rport_final_delete(struct work_struct *work) * This routine assumes no locks are held on entry. */ static struct fc_rport * -fc_rport_create(struct Scsi_Host *shost, int channel, - struct fc_rport_identifiers *ids) +fc_remote_port_create(struct Scsi_Host *shost, int channel, + struct fc_rport_identifiers *ids) { struct fc_host_attrs *fc_host = shost_to_fc_host(shost); struct fc_internal *fci = to_fc_internal(shost->transportt); @@ -2914,7 +2915,7 @@ fc_remote_port_add(struct Scsi_Host *shost, int channel, spin_unlock_irqrestore(shost->host_lock, flags); /* No consistent binding found - create new remote port entry */ - rport = fc_rport_create(shost, channel, ids); + rport = fc_remote_port_create(shost, channel, ids); return rport; } @@ -3554,81 +3555,6 @@ fc_vport_sched_delete(struct work_struct *work) * BSG support */ - -/** - * fc_destroy_bsgjob - routine to teardown/delete a fc bsg job - * @job: fc_bsg_job that is to be torn down - */ -static void -fc_destroy_bsgjob(struct fc_bsg_job *job) -{ - unsigned long flags; - - spin_lock_irqsave(&job->job_lock, flags); - if (job->ref_cnt) { - spin_unlock_irqrestore(&job->job_lock, flags); - return; - } - spin_unlock_irqrestore(&job->job_lock, flags); - - put_device(job->dev); /* release reference for the request */ - - kfree(job->request_payload.sg_list); - kfree(job->reply_payload.sg_list); - kfree(job); -} - -/** - * fc_bsg_jobdone - completion routine for bsg requests that the LLD has - * completed - * @job: fc_bsg_job that is complete - */ -static void -fc_bsg_jobdone(struct fc_bsg_job *job) -{ - struct request *req = job->req; - struct request *rsp = req->next_rq; - int err; - - err = job->req->errors = job->reply->result; - - if (err < 0) - /* we're only returning the result field in the reply */ - job->req->sense_len = sizeof(uint32_t); - else - job->req->sense_len = job->reply_len; - - /* we assume all request payload was transferred, residual == 0 */ - req->resid_len = 0; - - if (rsp) { - WARN_ON(job->reply->reply_payload_rcv_len > rsp->resid_len); - - /* set reply (bidi) residual */ - rsp->resid_len -= min(job->reply->reply_payload_rcv_len, - rsp->resid_len); - } - blk_complete_request(req); -} - -/** - * fc_bsg_softirq_done - softirq done routine for destroying the bsg requests - * @rq: BSG request that holds the job to be destroyed - */ -static void fc_bsg_softirq_done(struct request *rq) -{ - struct fc_bsg_job *job = rq->special; - unsigned long flags; - - spin_lock_irqsave(&job->job_lock, flags); - job->state_flags |= FC_RQST_STATE_DONE; - job->ref_cnt--; - spin_unlock_irqrestore(&job->job_lock, flags); - - blk_end_request_all(rq, rq->errors); - fc_destroy_bsgjob(job); -} - /** * fc_bsg_job_timeout - handler for when a bsg request timesout * @req: request that timed out @@ -3636,27 +3562,22 @@ static void fc_bsg_softirq_done(struct request *rq) static enum blk_eh_timer_return fc_bsg_job_timeout(struct request *req) { - struct fc_bsg_job *job = (void *) req->special; - struct Scsi_Host *shost = job->shost; + struct bsg_job *job = (void *) req->special; + struct Scsi_Host *shost = fc_bsg_to_shost(job); + struct fc_rport *rport = fc_bsg_to_rport(job); struct fc_internal *i = to_fc_internal(shost->transportt); - unsigned long flags; - int err = 0, done = 0; + int err = 0, inflight = 0; - if (job->rport && job->rport->port_state == FC_PORTSTATE_BLOCKED) + if (rport && rport->port_state == FC_PORTSTATE_BLOCKED) return BLK_EH_RESET_TIMER; - spin_lock_irqsave(&job->job_lock, flags); - if (job->state_flags & FC_RQST_STATE_DONE) - done = 1; - else - job->ref_cnt++; - spin_unlock_irqrestore(&job->job_lock, flags); + inflight = bsg_job_get(job); - if (!done && i->f->bsg_timeout) { + if (inflight && i->f->bsg_timeout) { /* call LLDD to abort the i/o as it has timed out */ err = i->f->bsg_timeout(job); if (err == -EAGAIN) { - job->ref_cnt--; + bsg_job_put(job); return BLK_EH_RESET_TIMER; } else if (err) printk(KERN_ERR "ERROR: FC BSG request timeout - LLD " @@ -3664,126 +3585,33 @@ fc_bsg_job_timeout(struct request *req) } /* the blk_end_sync_io() doesn't check the error */ - if (done) + if (!inflight) return BLK_EH_NOT_HANDLED; else return BLK_EH_HANDLED; } -static int -fc_bsg_map_buffer(struct fc_bsg_buffer *buf, struct request *req) -{ - size_t sz = (sizeof(struct scatterlist) * req->nr_phys_segments); - - BUG_ON(!req->nr_phys_segments); - - buf->sg_list = kzalloc(sz, GFP_KERNEL); - if (!buf->sg_list) - return -ENOMEM; - sg_init_table(buf->sg_list, req->nr_phys_segments); - buf->sg_cnt = blk_rq_map_sg(req->q, req, buf->sg_list); - buf->payload_len = blk_rq_bytes(req); - return 0; -} - - -/** - * fc_req_to_bsgjob - Allocate/create the fc_bsg_job structure for the - * bsg request - * @shost: SCSI Host corresponding to the bsg object - * @rport: (optional) FC Remote Port corresponding to the bsg object - * @req: BSG request that needs a job structure - */ -static int -fc_req_to_bsgjob(struct Scsi_Host *shost, struct fc_rport *rport, - struct request *req) -{ - struct fc_internal *i = to_fc_internal(shost->transportt); - struct request *rsp = req->next_rq; - struct fc_bsg_job *job; - int ret; - - BUG_ON(req->special); - - job = kzalloc(sizeof(struct fc_bsg_job) + i->f->dd_bsg_size, - GFP_KERNEL); - if (!job) - return -ENOMEM; - - /* - * Note: this is a bit silly. - * The request gets formatted as a SGIO v4 ioctl request, which - * then gets reformatted as a blk request, which then gets - * reformatted as a fc bsg request. And on completion, we have - * to wrap return results such that SGIO v4 thinks it was a scsi - * status. I hope this was all worth it. - */ - - req->special = job; - job->shost = shost; - job->rport = rport; - job->req = req; - if (i->f->dd_bsg_size) - job->dd_data = (void *)&job[1]; - spin_lock_init(&job->job_lock); - job->request = (struct fc_bsg_request *)req->cmd; - job->request_len = req->cmd_len; - job->reply = req->sense; - job->reply_len = SCSI_SENSE_BUFFERSIZE; /* Size of sense buffer - * allocated */ - if (req->bio) { - ret = fc_bsg_map_buffer(&job->request_payload, req); - if (ret) - goto failjob_rls_job; - } - if (rsp && rsp->bio) { - ret = fc_bsg_map_buffer(&job->reply_payload, rsp); - if (ret) - goto failjob_rls_rqst_payload; - } - job->job_done = fc_bsg_jobdone; - if (rport) - job->dev = &rport->dev; - else - job->dev = &shost->shost_gendev; - get_device(job->dev); /* take a reference for the request */ - - job->ref_cnt = 1; - - return 0; - - -failjob_rls_rqst_payload: - kfree(job->request_payload.sg_list); -failjob_rls_job: - kfree(job); - return -ENOMEM; -} - - -enum fc_dispatch_result { - FC_DISPATCH_BREAK, /* on return, q is locked, break from q loop */ - FC_DISPATCH_LOCKED, /* on return, q is locked, continue on */ - FC_DISPATCH_UNLOCKED, /* on return, q is unlocked, continue on */ -}; - - /** * fc_bsg_host_dispatch - process fc host bsg requests and dispatch to LLDD - * @q: fc host request queue * @shost: scsi host rport attached to * @job: bsg job to be processed */ -static enum fc_dispatch_result -fc_bsg_host_dispatch(struct request_queue *q, struct Scsi_Host *shost, - struct fc_bsg_job *job) +static int fc_bsg_host_dispatch(struct Scsi_Host *shost, struct bsg_job *job) { struct fc_internal *i = to_fc_internal(shost->transportt); + struct fc_bsg_request *bsg_request = job->request; + struct fc_bsg_reply *bsg_reply = job->reply; int cmdlen = sizeof(uint32_t); /* start with length of msgcode */ int ret; + /* check if we really have all the request data needed */ + if (job->request_len < cmdlen) { + ret = -ENOMSG; + goto fail_host_msg; + } + /* Validate the host command */ - switch (job->request->msgcode) { + switch (bsg_request->msgcode) { case FC_BSG_HST_ADD_RPORT: cmdlen += sizeof(struct fc_bsg_host_add_rport); break; @@ -3815,7 +3643,7 @@ fc_bsg_host_dispatch(struct request_queue *q, struct Scsi_Host *shost, case FC_BSG_HST_VENDOR: cmdlen += sizeof(struct fc_bsg_host_vendor); if ((shost->hostt->vendor_id == 0L) || - (job->request->rqst_data.h_vendor.vendor_id != + (bsg_request->rqst_data.h_vendor.vendor_id != shost->hostt->vendor_id)) { ret = -ESRCH; goto fail_host_msg; @@ -3827,24 +3655,19 @@ fc_bsg_host_dispatch(struct request_queue *q, struct Scsi_Host *shost, goto fail_host_msg; } - /* check if we really have all the request data needed */ - if (job->request_len < cmdlen) { - ret = -ENOMSG; - goto fail_host_msg; - } - ret = i->f->bsg_request(job); if (!ret) - return FC_DISPATCH_UNLOCKED; + return 0; fail_host_msg: /* return the errno failure code as the only status */ BUG_ON(job->reply_len < sizeof(uint32_t)); - job->reply->reply_payload_rcv_len = 0; - job->reply->result = ret; + bsg_reply->reply_payload_rcv_len = 0; + bsg_reply->result = ret; job->reply_len = sizeof(uint32_t); - fc_bsg_jobdone(job); - return FC_DISPATCH_UNLOCKED; + bsg_job_done(job, bsg_reply->result, + bsg_reply->reply_payload_rcv_len); + return 0; } @@ -3855,34 +3678,38 @@ fail_host_msg: static void fc_bsg_goose_queue(struct fc_rport *rport) { - if (!rport->rqst_q) + struct request_queue *q = rport->rqst_q; + unsigned long flags; + + if (!q) return; - /* - * This get/put dance makes no sense - */ - get_device(&rport->dev); - blk_run_queue_async(rport->rqst_q); - put_device(&rport->dev); + spin_lock_irqsave(q->queue_lock, flags); + blk_run_queue_async(q); + spin_unlock_irqrestore(q->queue_lock, flags); } /** * fc_bsg_rport_dispatch - process rport bsg requests and dispatch to LLDD - * @q: rport request queue * @shost: scsi host rport attached to - * @rport: rport request destined to * @job: bsg job to be processed */ -static enum fc_dispatch_result -fc_bsg_rport_dispatch(struct request_queue *q, struct Scsi_Host *shost, - struct fc_rport *rport, struct fc_bsg_job *job) +static int fc_bsg_rport_dispatch(struct Scsi_Host *shost, struct bsg_job *job) { struct fc_internal *i = to_fc_internal(shost->transportt); + struct fc_bsg_request *bsg_request = job->request; + struct fc_bsg_reply *bsg_reply = job->reply; int cmdlen = sizeof(uint32_t); /* start with length of msgcode */ int ret; + /* check if we really have all the request data needed */ + if (job->request_len < cmdlen) { + ret = -ENOMSG; + goto fail_rport_msg; + } + /* Validate the rport command */ - switch (job->request->msgcode) { + switch (bsg_request->msgcode) { case FC_BSG_RPT_ELS: cmdlen += sizeof(struct fc_bsg_rport_els); goto check_bidi; @@ -3902,133 +3729,31 @@ check_bidi: goto fail_rport_msg; } - /* check if we really have all the request data needed */ - if (job->request_len < cmdlen) { - ret = -ENOMSG; - goto fail_rport_msg; - } - ret = i->f->bsg_request(job); if (!ret) - return FC_DISPATCH_UNLOCKED; + return 0; fail_rport_msg: /* return the errno failure code as the only status */ BUG_ON(job->reply_len < sizeof(uint32_t)); - job->reply->reply_payload_rcv_len = 0; - job->reply->result = ret; + bsg_reply->reply_payload_rcv_len = 0; + bsg_reply->result = ret; job->reply_len = sizeof(uint32_t); - fc_bsg_jobdone(job); - return FC_DISPATCH_UNLOCKED; -} - - -/** - * fc_bsg_request_handler - generic handler for bsg requests - * @q: request queue to manage - * @shost: Scsi_Host related to the bsg object - * @rport: FC remote port related to the bsg object (optional) - * @dev: device structure for bsg object - */ -static void -fc_bsg_request_handler(struct request_queue *q, struct Scsi_Host *shost, - struct fc_rport *rport, struct device *dev) -{ - struct request *req; - struct fc_bsg_job *job; - enum fc_dispatch_result ret; - - if (!get_device(dev)) - return; - - while (1) { - if (rport && (rport->port_state == FC_PORTSTATE_BLOCKED) && - !(rport->flags & FC_RPORT_FAST_FAIL_TIMEDOUT)) - break; - - req = blk_fetch_request(q); - if (!req) - break; - - if (rport && (rport->port_state != FC_PORTSTATE_ONLINE)) { - req->errors = -ENXIO; - spin_unlock_irq(q->queue_lock); - blk_end_request_all(req, -ENXIO); - spin_lock_irq(q->queue_lock); - continue; - } - - spin_unlock_irq(q->queue_lock); - - ret = fc_req_to_bsgjob(shost, rport, req); - if (ret) { - req->errors = ret; - blk_end_request_all(req, ret); - spin_lock_irq(q->queue_lock); - continue; - } - - job = req->special; - - /* check if we have the msgcode value at least */ - if (job->request_len < sizeof(uint32_t)) { - BUG_ON(job->reply_len < sizeof(uint32_t)); - job->reply->reply_payload_rcv_len = 0; - job->reply->result = -ENOMSG; - job->reply_len = sizeof(uint32_t); - fc_bsg_jobdone(job); - spin_lock_irq(q->queue_lock); - continue; - } - - /* the dispatch routines will unlock the queue_lock */ - if (rport) - ret = fc_bsg_rport_dispatch(q, shost, rport, job); - else - ret = fc_bsg_host_dispatch(q, shost, job); - - /* did dispatcher hit state that can't process any more */ - if (ret == FC_DISPATCH_BREAK) - break; - - /* did dispatcher had released the lock */ - if (ret == FC_DISPATCH_UNLOCKED) - spin_lock_irq(q->queue_lock); - } - - spin_unlock_irq(q->queue_lock); - put_device(dev); - spin_lock_irq(q->queue_lock); -} - - -/** - * fc_bsg_host_handler - handler for bsg requests for a fc host - * @q: fc host request queue - */ -static void -fc_bsg_host_handler(struct request_queue *q) -{ - struct Scsi_Host *shost = q->queuedata; - - fc_bsg_request_handler(q, shost, NULL, &shost->shost_gendev); + bsg_job_done(job, bsg_reply->result, + bsg_reply->reply_payload_rcv_len); + return 0; } - -/** - * fc_bsg_rport_handler - handler for bsg requests for a fc rport - * @q: rport request queue - */ -static void -fc_bsg_rport_handler(struct request_queue *q) +static int fc_bsg_dispatch(struct bsg_job *job) { - struct fc_rport *rport = q->queuedata; - struct Scsi_Host *shost = rport_to_shost(rport); + struct Scsi_Host *shost = fc_bsg_to_shost(job); - fc_bsg_request_handler(q, shost, rport, &rport->dev); + if (scsi_is_fc_rport(job->dev)) + return fc_bsg_rport_dispatch(shost, job); + else + return fc_bsg_host_dispatch(shost, job); } - /** * fc_bsg_hostadd - Create and add the bsg hooks so we can receive requests * @shost: shost for fc_host @@ -4051,33 +3776,42 @@ fc_bsg_hostadd(struct Scsi_Host *shost, struct fc_host_attrs *fc_host) snprintf(bsg_name, sizeof(bsg_name), "fc_host%d", shost->host_no); - q = __scsi_alloc_queue(shost, fc_bsg_host_handler); + q = __scsi_alloc_queue(shost, bsg_request_fn); if (!q) { - printk(KERN_ERR "fc_host%d: bsg interface failed to " - "initialize - no request queue\n", - shost->host_no); + dev_err(dev, + "fc_host%d: bsg interface failed to initialize - no request queue\n", + shost->host_no); return -ENOMEM; } - q->queuedata = shost; - queue_flag_set_unlocked(QUEUE_FLAG_BIDI, q); - blk_queue_softirq_done(q, fc_bsg_softirq_done); - blk_queue_rq_timed_out(q, fc_bsg_job_timeout); - blk_queue_rq_timeout(q, FC_DEFAULT_BSG_TIMEOUT); - - err = bsg_register_queue(q, dev, bsg_name, NULL); + err = bsg_setup_queue(dev, q, bsg_name, fc_bsg_dispatch, + i->f->dd_bsg_size); if (err) { - printk(KERN_ERR "fc_host%d: bsg interface failed to " - "initialize - register queue\n", - shost->host_no); + dev_err(dev, + "fc_host%d: bsg interface failed to initialize - setup queue\n", + shost->host_no); blk_cleanup_queue(q); return err; } - + blk_queue_rq_timed_out(q, fc_bsg_job_timeout); + blk_queue_rq_timeout(q, FC_DEFAULT_BSG_TIMEOUT); fc_host->rqst_q = q; return 0; } +static int fc_bsg_rport_prep(struct request_queue *q, struct request *req) +{ + struct fc_rport *rport = dev_to_rport(q->queuedata); + + if (rport->port_state == FC_PORTSTATE_BLOCKED && + !(rport->flags & FC_RPORT_FAST_FAIL_TIMEDOUT)) + return BLKPREP_DEFER; + + if (rport->port_state != FC_PORTSTATE_ONLINE) + return BLKPREP_KILL; + + return BLKPREP_OK; +} /** * fc_bsg_rportadd - Create and add the bsg hooks so we can receive requests @@ -4097,29 +3831,22 @@ fc_bsg_rportadd(struct Scsi_Host *shost, struct fc_rport *rport) if (!i->f->bsg_request) return -ENOTSUPP; - q = __scsi_alloc_queue(shost, fc_bsg_rport_handler); + q = __scsi_alloc_queue(shost, bsg_request_fn); if (!q) { - printk(KERN_ERR "%s: bsg interface failed to " - "initialize - no request queue\n", - dev->kobj.name); + dev_err(dev, "bsg interface failed to initialize - no request queue\n"); return -ENOMEM; } - q->queuedata = rport; - queue_flag_set_unlocked(QUEUE_FLAG_BIDI, q); - blk_queue_softirq_done(q, fc_bsg_softirq_done); - blk_queue_rq_timed_out(q, fc_bsg_job_timeout); - blk_queue_rq_timeout(q, BLK_DEFAULT_SG_TIMEOUT); - - err = bsg_register_queue(q, dev, NULL, NULL); + err = bsg_setup_queue(dev, q, NULL, fc_bsg_dispatch, i->f->dd_bsg_size); if (err) { - printk(KERN_ERR "%s: bsg interface failed to " - "initialize - register queue\n", - dev->kobj.name); + dev_err(dev, "failed to setup bsg queue\n"); blk_cleanup_queue(q); return err; } + blk_queue_prep_rq(q, fc_bsg_rport_prep); + blk_queue_rq_timed_out(q, fc_bsg_job_timeout); + blk_queue_rq_timeout(q, BLK_DEFAULT_SG_TIMEOUT); rport->rqst_q = q; return 0; } diff --git a/drivers/scsi/scsi_transport_srp.c b/drivers/scsi/scsi_transport_srp.c index e3cd3ec..b87a786 100644 --- a/drivers/scsi/scsi_transport_srp.c +++ b/drivers/scsi/scsi_transport_srp.c @@ -24,7 +24,6 @@ #include <linux/err.h> #include <linux/slab.h> #include <linux/string.h> -#include <linux/delay.h> #include <scsi/scsi.h> #include <scsi/scsi_cmnd.h> @@ -115,21 +114,12 @@ static DECLARE_TRANSPORT_CLASS(srp_host_class, "srp_host", srp_host_setup, static DECLARE_TRANSPORT_CLASS(srp_rport_class, "srp_remote_ports", NULL, NULL, NULL); -#define SRP_PID(p) \ - (p)->port_id[0], (p)->port_id[1], (p)->port_id[2], (p)->port_id[3], \ - (p)->port_id[4], (p)->port_id[5], (p)->port_id[6], (p)->port_id[7], \ - (p)->port_id[8], (p)->port_id[9], (p)->port_id[10], (p)->port_id[11], \ - (p)->port_id[12], (p)->port_id[13], (p)->port_id[14], (p)->port_id[15] - -#define SRP_PID_FMT "%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:" \ - "%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x" - static ssize_t show_srp_rport_id(struct device *dev, struct device_attribute *attr, char *buf) { struct srp_rport *rport = transport_class_to_srp_rport(dev); - return sprintf(buf, SRP_PID_FMT "\n", SRP_PID(rport)); + return sprintf(buf, "%16phC\n", rport->port_id); } static DEVICE_ATTR(port_id, S_IRUGO, show_srp_rport_id, NULL); @@ -402,36 +392,6 @@ static void srp_reconnect_work(struct work_struct *work) } } -/** - * scsi_request_fn_active() - number of kernel threads inside scsi_request_fn() - * @shost: SCSI host for which to count the number of scsi_request_fn() callers. - * - * To do: add support for scsi-mq in this function. - */ -static int scsi_request_fn_active(struct Scsi_Host *shost) -{ - struct scsi_device *sdev; - struct request_queue *q; - int request_fn_active = 0; - - shost_for_each_device(sdev, shost) { - q = sdev->request_queue; - - spin_lock_irq(q->queue_lock); - request_fn_active += q->request_fn_active; - spin_unlock_irq(q->queue_lock); - } - - return request_fn_active; -} - -/* Wait until ongoing shost->hostt->queuecommand() calls have finished. */ -static void srp_wait_for_queuecommand(struct Scsi_Host *shost) -{ - while (scsi_request_fn_active(shost)) - msleep(20); -} - static void __rport_fail_io_fast(struct srp_rport *rport) { struct Scsi_Host *shost = rport_to_shost(rport); @@ -441,14 +401,17 @@ static void __rport_fail_io_fast(struct srp_rport *rport) if (srp_rport_set_state(rport, SRP_RPORT_FAIL_FAST)) return; + /* + * Call scsi_target_block() to wait for ongoing shost->queuecommand() + * calls before invoking i->f->terminate_rport_io(). + */ + scsi_target_block(rport->dev.parent); scsi_target_unblock(rport->dev.parent, SDEV_TRANSPORT_OFFLINE); /* Involve the LLD if possible to terminate all I/O on the rport. */ i = to_srp_internal(shost->transportt); - if (i->f->terminate_rport_io) { - srp_wait_for_queuecommand(shost); + if (i->f->terminate_rport_io) i->f->terminate_rport_io(rport); - } } /** @@ -576,7 +539,6 @@ int srp_reconnect_rport(struct srp_rport *rport) if (res) goto out; scsi_target_block(&shost->shost_gendev); - srp_wait_for_queuecommand(shost); res = rport->state != SRP_RPORT_LOST ? i->f->reconnect(rport) : -ENODEV; pr_debug("%s (state %d): transport.reconnect() returned %d\n", dev_name(&shost->shost_gendev), rport->state, res); diff --git a/drivers/scsi/sd.c b/drivers/scsi/sd.c index 079c2d9..1622e23 100644 --- a/drivers/scsi/sd.c +++ b/drivers/scsi/sd.c @@ -2465,9 +2465,7 @@ sd_read_write_protect_flag(struct scsi_disk *sdkp, unsigned char *buffer) if (sdkp->first_scan || old_wp != sdkp->write_prot) { sd_printk(KERN_NOTICE, sdkp, "Write Protect is %s\n", sdkp->write_prot ? "on" : "off"); - sd_printk(KERN_DEBUG, sdkp, - "Mode Sense: %02x %02x %02x %02x\n", - buffer[0], buffer[1], buffer[2], buffer[3]); + sd_printk(KERN_DEBUG, sdkp, "Mode Sense: %4ph\n", buffer); } } } diff --git a/drivers/scsi/smartpqi/smartpqi.h b/drivers/scsi/smartpqi/smartpqi.h index 07b6444..b673825 100644 --- a/drivers/scsi/smartpqi/smartpqi.h +++ b/drivers/scsi/smartpqi/smartpqi.h @@ -929,8 +929,6 @@ struct pqi_ctrl_info { int max_msix_vectors; int num_msix_vectors_enabled; int num_msix_vectors_initialized; - u32 msix_vectors[PQI_MAX_MSIX_VECTORS]; - void *intr_data[PQI_MAX_MSIX_VECTORS]; int event_irq; struct Scsi_Host *scsi_host; diff --git a/drivers/scsi/smartpqi/smartpqi_init.c b/drivers/scsi/smartpqi/smartpqi_init.c index a535b26..8702d9c 100644 --- a/drivers/scsi/smartpqi/smartpqi_init.c +++ b/drivers/scsi/smartpqi/smartpqi_init.c @@ -25,6 +25,7 @@ #include <linux/rtc.h> #include <linux/bcd.h> #include <linux/cciss_ioctl.h> +#include <linux/blk-mq-pci.h> #include <scsi/scsi_host.h> #include <scsi/scsi_cmnd.h> #include <scsi/scsi_device.h> @@ -2887,19 +2888,19 @@ static irqreturn_t pqi_irq_handler(int irq, void *data) static int pqi_request_irqs(struct pqi_ctrl_info *ctrl_info) { + struct pci_dev *pdev = ctrl_info->pci_dev; int i; int rc; - ctrl_info->event_irq = ctrl_info->msix_vectors[0]; + ctrl_info->event_irq = pci_irq_vector(pdev, 0); for (i = 0; i < ctrl_info->num_msix_vectors_enabled; i++) { - rc = request_irq(ctrl_info->msix_vectors[i], - pqi_irq_handler, 0, - DRIVER_NAME_SHORT, ctrl_info->intr_data[i]); + rc = request_irq(pci_irq_vector(pdev, i), pqi_irq_handler, 0, + DRIVER_NAME_SHORT, &ctrl_info->queue_groups[i]); if (rc) { - dev_err(&ctrl_info->pci_dev->dev, + dev_err(&pdev->dev, "irq %u init failed with error %d\n", - ctrl_info->msix_vectors[i], rc); + pci_irq_vector(pdev, i), rc); return rc; } ctrl_info->num_msix_vectors_initialized++; @@ -2908,72 +2909,23 @@ static int pqi_request_irqs(struct pqi_ctrl_info *ctrl_info) return 0; } -static void pqi_free_irqs(struct pqi_ctrl_info *ctrl_info) -{ - int i; - - for (i = 0; i < ctrl_info->num_msix_vectors_initialized; i++) - free_irq(ctrl_info->msix_vectors[i], - ctrl_info->intr_data[i]); -} - static int pqi_enable_msix_interrupts(struct pqi_ctrl_info *ctrl_info) { - unsigned int i; - int max_vectors; - int num_vectors_enabled; - struct msix_entry msix_entries[PQI_MAX_MSIX_VECTORS]; - - max_vectors = ctrl_info->num_queue_groups; - - for (i = 0; i < max_vectors; i++) - msix_entries[i].entry = i; - - num_vectors_enabled = pci_enable_msix_range(ctrl_info->pci_dev, - msix_entries, PQI_MIN_MSIX_VECTORS, max_vectors); + int ret; - if (num_vectors_enabled < 0) { + ret = pci_alloc_irq_vectors(ctrl_info->pci_dev, + PQI_MIN_MSIX_VECTORS, ctrl_info->num_queue_groups, + PCI_IRQ_MSIX | PCI_IRQ_AFFINITY); + if (ret < 0) { dev_err(&ctrl_info->pci_dev->dev, - "MSI-X init failed with error %d\n", - num_vectors_enabled); - return num_vectors_enabled; - } - - ctrl_info->num_msix_vectors_enabled = num_vectors_enabled; - for (i = 0; i < num_vectors_enabled; i++) { - ctrl_info->msix_vectors[i] = msix_entries[i].vector; - ctrl_info->intr_data[i] = &ctrl_info->queue_groups[i]; + "MSI-X init failed with error %d\n", ret); + return ret; } + ctrl_info->num_msix_vectors_enabled = ret; return 0; } -static void pqi_irq_set_affinity_hint(struct pqi_ctrl_info *ctrl_info) -{ - int i; - int rc; - int cpu; - - cpu = cpumask_first(cpu_online_mask); - for (i = 0; i < ctrl_info->num_msix_vectors_initialized; i++) { - rc = irq_set_affinity_hint(ctrl_info->msix_vectors[i], - get_cpu_mask(cpu)); - if (rc) - dev_err(&ctrl_info->pci_dev->dev, - "error %d setting affinity hint for irq vector %u\n", - rc, ctrl_info->msix_vectors[i]); - cpu = cpumask_next(cpu, cpu_online_mask); - } -} - -static void pqi_irq_unset_affinity_hint(struct pqi_ctrl_info *ctrl_info) -{ - int i; - - for (i = 0; i < ctrl_info->num_msix_vectors_initialized; i++) - irq_set_affinity_hint(ctrl_info->msix_vectors[i], NULL); -} - static int pqi_alloc_operational_queues(struct pqi_ctrl_info *ctrl_info) { unsigned int i; @@ -4743,6 +4695,13 @@ static int pqi_slave_configure(struct scsi_device *sdev) return 0; } +static int pqi_map_queues(struct Scsi_Host *shost) +{ + struct pqi_ctrl_info *ctrl_info = shost_to_hba(shost); + + return blk_mq_pci_map_queues(&shost->tag_set, ctrl_info->pci_dev); +} + static int pqi_getpciinfo_ioctl(struct pqi_ctrl_info *ctrl_info, void __user *arg) { @@ -5130,6 +5089,7 @@ static struct scsi_host_template pqi_driver_template = { .ioctl = pqi_ioctl, .slave_alloc = pqi_slave_alloc, .slave_configure = pqi_slave_configure, + .map_queues = pqi_map_queues, .sdev_attrs = pqi_sdev_attrs, .shost_attrs = pqi_shost_attrs, }; @@ -5159,7 +5119,7 @@ static int pqi_register_scsi(struct pqi_ctrl_info *ctrl_info) shost->cmd_per_lun = shost->can_queue; shost->sg_tablesize = ctrl_info->sg_tablesize; shost->transportt = pqi_sas_transport_template; - shost->irq = ctrl_info->msix_vectors[0]; + shost->irq = pci_irq_vector(ctrl_info->pci_dev, 0); shost->unique_id = shost->irq; shost->nr_hw_queues = ctrl_info->num_queue_groups; shost->hostdata[0] = (unsigned long)ctrl_info; @@ -5409,8 +5369,6 @@ static int pqi_ctrl_init(struct pqi_ctrl_info *ctrl_info) if (rc) return rc; - pqi_irq_set_affinity_hint(ctrl_info); - rc = pqi_create_queues(ctrl_info); if (rc) return rc; @@ -5557,10 +5515,14 @@ static inline void pqi_free_ctrl_info(struct pqi_ctrl_info *ctrl_info) static void pqi_free_interrupts(struct pqi_ctrl_info *ctrl_info) { - pqi_irq_unset_affinity_hint(ctrl_info); - pqi_free_irqs(ctrl_info); - if (ctrl_info->num_msix_vectors_enabled) - pci_disable_msix(ctrl_info->pci_dev); + int i; + + for (i = 0; i < ctrl_info->num_msix_vectors_initialized; i++) { + free_irq(pci_irq_vector(ctrl_info->pci_dev, i), + &ctrl_info->queue_groups[i]); + } + + pci_free_irq_vectors(ctrl_info->pci_dev); } static void pqi_free_ctrl_resources(struct pqi_ctrl_info *ctrl_info) diff --git a/drivers/scsi/storvsc_drv.c b/drivers/scsi/storvsc_drv.c index 8ccfc9e..05526b7 100644 --- a/drivers/scsi/storvsc_drv.c +++ b/drivers/scsi/storvsc_drv.c @@ -1495,9 +1495,9 @@ static int storvsc_queuecommand(struct Scsi_Host *host, struct scsi_cmnd *scmnd) if (sg_count) { if (sg_count > MAX_PAGE_BUFFER_COUNT) { - payload_sz = (sg_count * sizeof(void *) + + payload_sz = (sg_count * sizeof(u64) + sizeof(struct vmbus_packet_mpb_array)); - payload = kmalloc(payload_sz, GFP_ATOMIC); + payload = kzalloc(payload_sz, GFP_ATOMIC); if (!payload) return SCSI_MLQUEUE_DEVICE_BUSY; } diff --git a/drivers/scsi/sun3_scsi.c b/drivers/scsi/sun3_scsi.c index 3c4c070..88db699 100644 --- a/drivers/scsi/sun3_scsi.c +++ b/drivers/scsi/sun3_scsi.c @@ -43,20 +43,18 @@ #define NCR5380_implementation_fields /* none */ -#define NCR5380_read(reg) sun3scsi_read(reg) -#define NCR5380_write(reg, value) sun3scsi_write(reg, value) +#define NCR5380_read(reg) in_8(hostdata->io + (reg)) +#define NCR5380_write(reg, value) out_8(hostdata->io + (reg), value) #define NCR5380_queue_command sun3scsi_queue_command #define NCR5380_bus_reset sun3scsi_bus_reset #define NCR5380_abort sun3scsi_abort #define NCR5380_info sun3scsi_info -#define NCR5380_dma_recv_setup(instance, data, count) (count) -#define NCR5380_dma_send_setup(instance, data, count) (count) -#define NCR5380_dma_residual(instance) \ - sun3scsi_dma_residual(instance) -#define NCR5380_dma_xfer_len(instance, cmd, phase) \ - sun3scsi_dma_xfer_len(cmd->SCp.this_residual, cmd) +#define NCR5380_dma_xfer_len sun3scsi_dma_xfer_len +#define NCR5380_dma_recv_setup sun3scsi_dma_count +#define NCR5380_dma_send_setup sun3scsi_dma_count +#define NCR5380_dma_residual sun3scsi_dma_residual #define NCR5380_acquire_dma_irq(instance) (1) #define NCR5380_release_dma_irq(instance) @@ -82,7 +80,6 @@ module_param(setup_hostid, int, 0); #define SUN3_DVMA_BUFSIZE 0xe000 static struct scsi_cmnd *sun3_dma_setup_done; -static unsigned char *sun3_scsi_regp; static volatile struct sun3_dma_regs *dregs; static struct sun3_udc_regs *udc_regs; static unsigned char *sun3_dma_orig_addr; @@ -90,20 +87,6 @@ static unsigned long sun3_dma_orig_count; static int sun3_dma_active; static unsigned long last_residual; -/* - * NCR 5380 register access functions - */ - -static inline unsigned char sun3scsi_read(int reg) -{ - return in_8(sun3_scsi_regp + reg); -} - -static inline void sun3scsi_write(int reg, int value) -{ - out_8(sun3_scsi_regp + reg, value); -} - #ifndef SUN3_SCSI_VME /* dma controller register access functions */ @@ -158,8 +141,8 @@ static irqreturn_t scsi_sun3_intr(int irq, void *dev) } /* sun3scsi_dma_setup() -- initialize the dma controller for a read/write */ -static unsigned long sun3scsi_dma_setup(struct Scsi_Host *instance, - void *data, unsigned long count, int write_flag) +static int sun3scsi_dma_setup(struct NCR5380_hostdata *hostdata, + unsigned char *data, int count, int write_flag) { void *addr; @@ -211,9 +194,10 @@ static unsigned long sun3scsi_dma_setup(struct Scsi_Host *instance, dregs->csr |= CSR_FIFO; if(dregs->fifo_count != count) { - shost_printk(KERN_ERR, instance, "FIFO mismatch %04x not %04x\n", + shost_printk(KERN_ERR, hostdata->host, + "FIFO mismatch %04x not %04x\n", dregs->fifo_count, (unsigned int) count); - NCR5380_dprint(NDEBUG_DMA, instance); + NCR5380_dprint(NDEBUG_DMA, hostdata->host); } /* setup udc */ @@ -248,14 +232,34 @@ static unsigned long sun3scsi_dma_setup(struct Scsi_Host *instance, } -static inline unsigned long sun3scsi_dma_residual(struct Scsi_Host *instance) +static int sun3scsi_dma_count(struct NCR5380_hostdata *hostdata, + unsigned char *data, int count) +{ + return count; +} + +static inline int sun3scsi_dma_recv_setup(struct NCR5380_hostdata *hostdata, + unsigned char *data, int count) +{ + return sun3scsi_dma_setup(hostdata, data, count, 0); +} + +static inline int sun3scsi_dma_send_setup(struct NCR5380_hostdata *hostdata, + unsigned char *data, int count) +{ + return sun3scsi_dma_setup(hostdata, data, count, 1); +} + +static int sun3scsi_dma_residual(struct NCR5380_hostdata *hostdata) { return last_residual; } -static inline unsigned long sun3scsi_dma_xfer_len(unsigned long wanted_len, - struct scsi_cmnd *cmd) +static int sun3scsi_dma_xfer_len(struct NCR5380_hostdata *hostdata, + struct scsi_cmnd *cmd) { + int wanted_len = cmd->SCp.this_residual; + if (wanted_len < DMA_MIN_SIZE || cmd->request->cmd_type != REQ_TYPE_FS) return 0; @@ -428,9 +432,10 @@ static struct scsi_host_template sun3_scsi_template = { static int __init sun3_scsi_probe(struct platform_device *pdev) { struct Scsi_Host *instance; + struct NCR5380_hostdata *hostdata; int error; struct resource *irq, *mem; - unsigned char *ioaddr; + void __iomem *ioaddr; int host_flags = 0; #ifdef SUN3_SCSI_VME int i; @@ -493,8 +498,6 @@ static int __init sun3_scsi_probe(struct platform_device *pdev) } #endif - sun3_scsi_regp = ioaddr; - instance = scsi_host_alloc(&sun3_scsi_template, sizeof(struct NCR5380_hostdata)); if (!instance) { @@ -502,9 +505,12 @@ static int __init sun3_scsi_probe(struct platform_device *pdev) goto fail_alloc; } - instance->io_port = (unsigned long)ioaddr; instance->irq = irq->start; + hostdata = shost_priv(instance); + hostdata->base = mem->start; + hostdata->io = ioaddr; + error = NCR5380_init(instance, host_flags); if (error) goto fail_init; @@ -552,13 +558,15 @@ fail_init: fail_alloc: if (udc_regs) dvma_free(udc_regs); - iounmap(sun3_scsi_regp); + iounmap(ioaddr); return error; } static int __exit sun3_scsi_remove(struct platform_device *pdev) { struct Scsi_Host *instance = platform_get_drvdata(pdev); + struct NCR5380_hostdata *hostdata = shost_priv(instance); + void __iomem *ioaddr = hostdata->io; scsi_remove_host(instance); free_irq(instance->irq, instance); @@ -566,7 +574,7 @@ static int __exit sun3_scsi_remove(struct platform_device *pdev) scsi_host_put(instance); if (udc_regs) dvma_free(udc_regs); - iounmap(sun3_scsi_regp); + iounmap(ioaddr); return 0; } diff --git a/drivers/scsi/ufs/ufs-qcom.c b/drivers/scsi/ufs/ufs-qcom.c index 3aedf73..aa43bfe 100644 --- a/drivers/scsi/ufs/ufs-qcom.c +++ b/drivers/scsi/ufs/ufs-qcom.c @@ -1094,10 +1094,12 @@ static void ufs_qcom_set_caps(struct ufs_hba *hba) * ufs_qcom_setup_clocks - enables/disable clocks * @hba: host controller instance * @on: If true, enable clocks else disable them. + * @status: PRE_CHANGE or POST_CHANGE notify * * Returns 0 on success, non-zero on failure. */ -static int ufs_qcom_setup_clocks(struct ufs_hba *hba, bool on) +static int ufs_qcom_setup_clocks(struct ufs_hba *hba, bool on, + enum ufs_notify_change_status status) { struct ufs_qcom_host *host = ufshcd_get_variant(hba); int err; @@ -1111,18 +1113,9 @@ static int ufs_qcom_setup_clocks(struct ufs_hba *hba, bool on) if (!host) return 0; - if (on) { - err = ufs_qcom_phy_enable_iface_clk(host->generic_phy); - if (err) - goto out; + if (on && (status == POST_CHANGE)) { + phy_power_on(host->generic_phy); - err = ufs_qcom_phy_enable_ref_clk(host->generic_phy); - if (err) { - dev_err(hba->dev, "%s enable phy ref clock failed, err=%d\n", - __func__, err); - ufs_qcom_phy_disable_iface_clk(host->generic_phy); - goto out; - } /* enable the device ref clock for HS mode*/ if (ufshcd_is_hs_mode(&hba->pwr_info)) ufs_qcom_dev_ref_clk_ctrl(host, true); @@ -1130,14 +1123,15 @@ static int ufs_qcom_setup_clocks(struct ufs_hba *hba, bool on) if (vote == host->bus_vote.min_bw_vote) ufs_qcom_update_bus_bw_vote(host); - } else { - - /* M-PHY RMMI interface clocks can be turned off */ - ufs_qcom_phy_disable_iface_clk(host->generic_phy); - if (!ufs_qcom_is_link_active(hba)) + } else if (!on && (status == PRE_CHANGE)) { + if (!ufs_qcom_is_link_active(hba)) { /* disable device ref_clk */ ufs_qcom_dev_ref_clk_ctrl(host, false); + /* powering off PHY during aggressive clk gating */ + phy_power_off(host->generic_phy); + } + vote = host->bus_vote.min_bw_vote; } @@ -1146,7 +1140,6 @@ static int ufs_qcom_setup_clocks(struct ufs_hba *hba, bool on) dev_err(hba->dev, "%s: set bus vote failed %d\n", __func__, err); -out: return err; } @@ -1204,12 +1197,12 @@ static int ufs_qcom_init(struct ufs_hba *hba) if (IS_ERR(host->generic_phy)) { err = PTR_ERR(host->generic_phy); dev_err(dev, "%s: PHY get failed %d\n", __func__, err); - goto out; + goto out_variant_clear; } err = ufs_qcom_bus_register(host); if (err) - goto out_host_free; + goto out_variant_clear; ufs_qcom_get_controller_revision(hba, &host->hw_ver.major, &host->hw_ver.minor, &host->hw_ver.step); @@ -1254,7 +1247,7 @@ static int ufs_qcom_init(struct ufs_hba *hba) ufs_qcom_set_caps(hba); ufs_qcom_advertise_quirks(hba); - ufs_qcom_setup_clocks(hba, true); + ufs_qcom_setup_clocks(hba, true, POST_CHANGE); if (hba->dev->id < MAX_UFS_QCOM_HOSTS) ufs_qcom_hosts[hba->dev->id] = host; @@ -1274,8 +1267,7 @@ out_disable_phy: phy_power_off(host->generic_phy); out_unregister_bus: phy_exit(host->generic_phy); -out_host_free: - devm_kfree(dev, host); +out_variant_clear: ufshcd_set_variant(hba, NULL); out: return err; @@ -1287,6 +1279,7 @@ static void ufs_qcom_exit(struct ufs_hba *hba) ufs_qcom_disable_lane_clks(host); phy_power_off(host->generic_phy); + phy_exit(host->generic_phy); } static int ufs_qcom_set_dme_vs_core_clk_ctrl_clear_div(struct ufs_hba *hba, diff --git a/drivers/scsi/ufs/ufs.h b/drivers/scsi/ufs/ufs.h index 845b874..8e6709a 100644 --- a/drivers/scsi/ufs/ufs.h +++ b/drivers/scsi/ufs/ufs.h @@ -46,6 +46,7 @@ #define QUERY_DESC_HDR_SIZE 2 #define QUERY_OSF_SIZE (GENERAL_UPIU_REQUEST_SIZE - \ (sizeof(struct utp_upiu_header))) +#define RESPONSE_UPIU_SENSE_DATA_LENGTH 18 #define UPIU_HEADER_DWORD(byte3, byte2, byte1, byte0)\ cpu_to_be32((byte3 << 24) | (byte2 << 16) |\ @@ -162,7 +163,7 @@ enum desc_header_offset { }; enum ufs_desc_max_size { - QUERY_DESC_DEVICE_MAX_SIZE = 0x1F, + QUERY_DESC_DEVICE_MAX_SIZE = 0x40, QUERY_DESC_CONFIGURAION_MAX_SIZE = 0x90, QUERY_DESC_UNIT_MAX_SIZE = 0x23, QUERY_DESC_INTERCONNECT_MAX_SIZE = 0x06, @@ -416,7 +417,7 @@ struct utp_cmd_rsp { __be32 residual_transfer_count; __be32 reserved[4]; __be16 sense_data_len; - u8 sense_data[18]; + u8 sense_data[RESPONSE_UPIU_SENSE_DATA_LENGTH]; }; /** diff --git a/drivers/scsi/ufs/ufs_quirks.h b/drivers/scsi/ufs/ufs_quirks.h index 22f881e..f798305 100644 --- a/drivers/scsi/ufs/ufs_quirks.h +++ b/drivers/scsi/ufs/ufs_quirks.h @@ -128,6 +128,13 @@ struct ufs_dev_fix { */ #define UFS_DEVICE_QUIRK_DELAY_BEFORE_LPM (1 << 6) +/* + * Some UFS devices require host PA_TACTIVATE to be lower than device + * PA_TACTIVATE, enabling this quirk ensure this. + */ +#define UFS_DEVICE_QUIRK_HOST_PA_TACTIVATE (1 << 7) + + struct ufs_hba; void ufs_advertise_fixup_device(struct ufs_hba *hba); @@ -140,6 +147,8 @@ static struct ufs_dev_fix ufs_fixups[] = { UFS_DEVICE_QUIRK_RECOVERY_FROM_DL_NAC_ERRORS), UFS_FIX(UFS_VENDOR_SAMSUNG, UFS_ANY_MODEL, UFS_DEVICE_NO_FASTAUTO), + UFS_FIX(UFS_VENDOR_SAMSUNG, UFS_ANY_MODEL, + UFS_DEVICE_QUIRK_HOST_PA_TACTIVATE), UFS_FIX(UFS_VENDOR_TOSHIBA, UFS_ANY_MODEL, UFS_DEVICE_QUIRK_DELAY_BEFORE_LPM), UFS_FIX(UFS_VENDOR_TOSHIBA, "THGLF2G9C8KBADG", diff --git a/drivers/scsi/ufs/ufshcd-pci.c b/drivers/scsi/ufs/ufshcd-pci.c index d15eaa4..52b546f 100644 --- a/drivers/scsi/ufs/ufshcd-pci.c +++ b/drivers/scsi/ufs/ufshcd-pci.c @@ -104,6 +104,7 @@ static void ufshcd_pci_remove(struct pci_dev *pdev) pm_runtime_forbid(&pdev->dev); pm_runtime_get_noresume(&pdev->dev); ufshcd_remove(hba); + ufshcd_dealloc_host(hba); } /** @@ -147,6 +148,7 @@ ufshcd_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) err = ufshcd_init(hba, mmio_base, pdev->irq); if (err) { dev_err(&pdev->dev, "Initialization failed\n"); + ufshcd_dealloc_host(hba); return err; } diff --git a/drivers/scsi/ufs/ufshcd-pltfrm.c b/drivers/scsi/ufs/ufshcd-pltfrm.c index db53f38d..a72a4ba 100644 --- a/drivers/scsi/ufs/ufshcd-pltfrm.c +++ b/drivers/scsi/ufs/ufshcd-pltfrm.c @@ -163,7 +163,7 @@ static int ufshcd_populate_vreg(struct device *dev, const char *name, if (ret) { dev_err(dev, "%s: unable to find %s err %d\n", __func__, prop_name, ret); - goto out_free; + goto out; } vreg->min_uA = 0; @@ -185,9 +185,6 @@ static int ufshcd_populate_vreg(struct device *dev, const char *name, goto out; -out_free: - devm_kfree(dev, vreg); - vreg = NULL; out: if (!ret) *out_vreg = vreg; diff --git a/drivers/scsi/ufs/ufshcd.c b/drivers/scsi/ufs/ufshcd.c index cf54987..ef8548c 100644 --- a/drivers/scsi/ufs/ufshcd.c +++ b/drivers/scsi/ufs/ufshcd.c @@ -45,6 +45,8 @@ #include "ufs_quirks.h" #include "unipro.h" +#define UFSHCD_REQ_SENSE_SIZE 18 + #define UFSHCD_ENABLE_INTRS (UTP_TRANSFER_REQ_COMPL |\ UTP_TASK_REQ_COMPL |\ UFSHCD_ERROR_MASK) @@ -57,15 +59,9 @@ #define NOP_OUT_TIMEOUT 30 /* msecs */ /* Query request retries */ -#define QUERY_REQ_RETRIES 10 +#define QUERY_REQ_RETRIES 3 /* Query request timeout */ -#define QUERY_REQ_TIMEOUT 30 /* msec */ -/* - * Query request timeout for fDeviceInit flag - * fDeviceInit query response time for some devices is too large that default - * QUERY_REQ_TIMEOUT may not be enough for such devices. - */ -#define QUERY_FDEVICEINIT_REQ_TIMEOUT 600 /* msec */ +#define QUERY_REQ_TIMEOUT 1500 /* 1.5 seconds */ /* Task management command timeout */ #define TM_CMD_TIMEOUT 100 /* msecs */ @@ -123,6 +119,7 @@ enum { UFSHCD_STATE_RESET, UFSHCD_STATE_ERROR, UFSHCD_STATE_OPERATIONAL, + UFSHCD_STATE_EH_SCHEDULED, }; /* UFSHCD error handling flags */ @@ -598,6 +595,20 @@ static bool ufshcd_is_unipro_pa_params_tuning_req(struct ufs_hba *hba) return false; } +static void ufshcd_suspend_clkscaling(struct ufs_hba *hba) +{ + if (ufshcd_is_clkscaling_enabled(hba)) { + devfreq_suspend_device(hba->devfreq); + hba->clk_scaling.window_start_t = 0; + } +} + +static void ufshcd_resume_clkscaling(struct ufs_hba *hba) +{ + if (ufshcd_is_clkscaling_enabled(hba)) + devfreq_resume_device(hba->devfreq); +} + static void ufshcd_ungate_work(struct work_struct *work) { int ret; @@ -631,8 +642,7 @@ static void ufshcd_ungate_work(struct work_struct *work) hba->clk_gating.is_suspended = false; } unblock_reqs: - if (ufshcd_is_clkscaling_enabled(hba)) - devfreq_resume_device(hba->devfreq); + ufshcd_resume_clkscaling(hba); scsi_unblock_requests(hba->host); } @@ -660,6 +670,21 @@ int ufshcd_hold(struct ufs_hba *hba, bool async) start: switch (hba->clk_gating.state) { case CLKS_ON: + /* + * Wait for the ungate work to complete if in progress. + * Though the clocks may be in ON state, the link could + * still be in hibner8 state if hibern8 is allowed + * during clock gating. + * Make sure we exit hibern8 state also in addition to + * clocks being ON. + */ + if (ufshcd_can_hibern8_during_gating(hba) && + ufshcd_is_link_hibern8(hba)) { + spin_unlock_irqrestore(hba->host->host_lock, flags); + flush_work(&hba->clk_gating.ungate_work); + spin_lock_irqsave(hba->host->host_lock, flags); + goto start; + } break; case REQ_CLKS_OFF: if (cancel_delayed_work(&hba->clk_gating.gate_work)) { @@ -709,7 +734,14 @@ static void ufshcd_gate_work(struct work_struct *work) unsigned long flags; spin_lock_irqsave(hba->host->host_lock, flags); - if (hba->clk_gating.is_suspended) { + /* + * In case you are here to cancel this work the gating state + * would be marked as REQ_CLKS_ON. In this case save time by + * skipping the gating work and exit after changing the clock + * state to CLKS_ON. + */ + if (hba->clk_gating.is_suspended || + (hba->clk_gating.state == REQ_CLKS_ON)) { hba->clk_gating.state = CLKS_ON; goto rel_lock; } @@ -731,10 +763,7 @@ static void ufshcd_gate_work(struct work_struct *work) ufshcd_set_link_hibern8(hba); } - if (ufshcd_is_clkscaling_enabled(hba)) { - devfreq_suspend_device(hba->devfreq); - hba->clk_scaling.window_start_t = 0; - } + ufshcd_suspend_clkscaling(hba); if (!ufshcd_is_link_active(hba)) ufshcd_setup_clocks(hba, false); @@ -878,6 +907,8 @@ void ufshcd_send_command(struct ufs_hba *hba, unsigned int task_tag) ufshcd_clk_scaling_start_busy(hba); __set_bit(task_tag, &hba->outstanding_reqs); ufshcd_writel(hba, 1 << task_tag, REG_UTP_TRANSFER_REQ_DOOR_BELL); + /* Make sure that doorbell is committed immediately */ + wmb(); } /** @@ -889,10 +920,14 @@ static inline void ufshcd_copy_sense_data(struct ufshcd_lrb *lrbp) int len; if (lrbp->sense_buffer && ufshcd_get_rsp_upiu_data_seg_len(lrbp->ucd_rsp_ptr)) { + int len_to_copy; + len = be16_to_cpu(lrbp->ucd_rsp_ptr->sr.sense_data_len); + len_to_copy = min_t(int, RESPONSE_UPIU_SENSE_DATA_LENGTH, len); + memcpy(lrbp->sense_buffer, lrbp->ucd_rsp_ptr->sr.sense_data, - min_t(int, len, SCSI_SENSE_BUFFERSIZE)); + min_t(int, len_to_copy, UFSHCD_REQ_SENSE_SIZE)); } } @@ -1088,7 +1123,7 @@ ufshcd_send_uic_cmd(struct ufs_hba *hba, struct uic_command *uic_cmd) * * Returns 0 in case of success, non-zero value in case of failure */ -static int ufshcd_map_sg(struct ufshcd_lrb *lrbp) +static int ufshcd_map_sg(struct ufs_hba *hba, struct ufshcd_lrb *lrbp) { struct ufshcd_sg_entry *prd_table; struct scatterlist *sg; @@ -1102,8 +1137,13 @@ static int ufshcd_map_sg(struct ufshcd_lrb *lrbp) return sg_segments; if (sg_segments) { - lrbp->utr_descriptor_ptr->prd_table_length = - cpu_to_le16((u16) (sg_segments)); + if (hba->quirks & UFSHCD_QUIRK_PRDT_BYTE_GRAN) + lrbp->utr_descriptor_ptr->prd_table_length = + cpu_to_le16((u16)(sg_segments * + sizeof(struct ufshcd_sg_entry))); + else + lrbp->utr_descriptor_ptr->prd_table_length = + cpu_to_le16((u16) (sg_segments)); prd_table = (struct ufshcd_sg_entry *)lrbp->ucd_prdt_ptr; @@ -1410,6 +1450,7 @@ static int ufshcd_queuecommand(struct Scsi_Host *host, struct scsi_cmnd *cmd) switch (hba->ufshcd_state) { case UFSHCD_STATE_OPERATIONAL: break; + case UFSHCD_STATE_EH_SCHEDULED: case UFSHCD_STATE_RESET: err = SCSI_MLQUEUE_HOST_BUSY; goto out_unlock; @@ -1457,7 +1498,7 @@ static int ufshcd_queuecommand(struct Scsi_Host *host, struct scsi_cmnd *cmd) WARN_ON(lrbp->cmd); lrbp->cmd = cmd; - lrbp->sense_bufflen = SCSI_SENSE_BUFFERSIZE; + lrbp->sense_bufflen = UFSHCD_REQ_SENSE_SIZE; lrbp->sense_buffer = cmd->sense_buffer; lrbp->task_tag = tag; lrbp->lun = ufshcd_scsi_to_upiu_lun(cmd->device->lun); @@ -1465,15 +1506,18 @@ static int ufshcd_queuecommand(struct Scsi_Host *host, struct scsi_cmnd *cmd) ufshcd_comp_scsi_upiu(hba, lrbp); - err = ufshcd_map_sg(lrbp); + err = ufshcd_map_sg(hba, lrbp); if (err) { lrbp->cmd = NULL; clear_bit_unlock(tag, &hba->lrb_in_use); goto out; } + /* Make sure descriptors are ready before ringing the doorbell */ + wmb(); /* issue command to the controller */ spin_lock_irqsave(hba->host->host_lock, flags); + ufshcd_vops_setup_xfer_req(hba, tag, (lrbp->cmd ? true : false)); ufshcd_send_command(hba, tag); out_unlock: spin_unlock_irqrestore(hba->host->host_lock, flags); @@ -1581,6 +1625,8 @@ static int ufshcd_wait_for_dev_cmd(struct ufs_hba *hba, time_left = wait_for_completion_timeout(hba->dev_cmd.complete, msecs_to_jiffies(max_timeout)); + /* Make sure descriptors are ready before ringing the doorbell */ + wmb(); spin_lock_irqsave(hba->host->host_lock, flags); hba->dev_cmd.complete = NULL; if (likely(time_left)) { @@ -1683,6 +1729,7 @@ static int ufshcd_exec_dev_cmd(struct ufs_hba *hba, /* Make sure descriptors are ready before ringing the doorbell */ wmb(); spin_lock_irqsave(hba->host->host_lock, flags); + ufshcd_vops_setup_xfer_req(hba, tag, (lrbp->cmd ? true : false)); ufshcd_send_command(hba, tag); spin_unlock_irqrestore(hba->host->host_lock, flags); @@ -1789,9 +1836,6 @@ int ufshcd_query_flag(struct ufs_hba *hba, enum query_opcode opcode, goto out_unlock; } - if (idn == QUERY_FLAG_IDN_FDEVICEINIT) - timeout = QUERY_FDEVICEINIT_REQ_TIMEOUT; - err = ufshcd_exec_dev_cmd(hba, DEV_CMD_TYPE_QUERY, timeout); if (err) { @@ -1861,8 +1905,8 @@ static int ufshcd_query_attr(struct ufs_hba *hba, enum query_opcode opcode, err = ufshcd_exec_dev_cmd(hba, DEV_CMD_TYPE_QUERY, QUERY_REQ_TIMEOUT); if (err) { - dev_err(hba->dev, "%s: opcode 0x%.2x for idn %d failed, err = %d\n", - __func__, opcode, idn, err); + dev_err(hba->dev, "%s: opcode 0x%.2x for idn %d failed, index %d, err = %d\n", + __func__, opcode, idn, index, err); goto out_unlock; } @@ -1961,8 +2005,8 @@ static int __ufshcd_query_descriptor(struct ufs_hba *hba, err = ufshcd_exec_dev_cmd(hba, DEV_CMD_TYPE_QUERY, QUERY_REQ_TIMEOUT); if (err) { - dev_err(hba->dev, "%s: opcode 0x%.2x for idn %d failed, err = %d\n", - __func__, opcode, idn, err); + dev_err(hba->dev, "%s: opcode 0x%.2x for idn %d failed, index %d, err = %d\n", + __func__, opcode, idn, index, err); goto out_unlock; } @@ -2055,18 +2099,41 @@ static int ufshcd_read_desc_param(struct ufs_hba *hba, desc_id, desc_index, 0, desc_buf, &buff_len); - if (ret || (buff_len < ufs_query_desc_max_size[desc_id]) || - (desc_buf[QUERY_DESC_LENGTH_OFFSET] != - ufs_query_desc_max_size[desc_id]) - || (desc_buf[QUERY_DESC_DESC_TYPE_OFFSET] != desc_id)) { - dev_err(hba->dev, "%s: Failed reading descriptor. desc_id %d param_offset %d buff_len %d ret %d", - __func__, desc_id, param_offset, buff_len, ret); - if (!ret) - ret = -EINVAL; + if (ret) { + dev_err(hba->dev, "%s: Failed reading descriptor. desc_id %d, desc_index %d, param_offset %d, ret %d", + __func__, desc_id, desc_index, param_offset, ret); goto out; } + /* Sanity check */ + if (desc_buf[QUERY_DESC_DESC_TYPE_OFFSET] != desc_id) { + dev_err(hba->dev, "%s: invalid desc_id %d in descriptor header", + __func__, desc_buf[QUERY_DESC_DESC_TYPE_OFFSET]); + ret = -EINVAL; + goto out; + } + + /* + * While reading variable size descriptors (like string descriptor), + * some UFS devices may report the "LENGTH" (field in "Transaction + * Specific fields" of Query Response UPIU) same as what was requested + * in Query Request UPIU instead of reporting the actual size of the + * variable size descriptor. + * Although it's safe to ignore the "LENGTH" field for variable size + * descriptors as we can always derive the length of the descriptor from + * the descriptor header fields. Hence this change impose the length + * match check only for fixed size descriptors (for which we always + * request the correct size as part of Query Request UPIU). + */ + if ((desc_id != QUERY_DESC_IDN_STRING) && + (buff_len != desc_buf[QUERY_DESC_LENGTH_OFFSET])) { + dev_err(hba->dev, "%s: desc_buf length mismatch: buff_len %d, buff_len(desc_header) %d", + __func__, buff_len, desc_buf[QUERY_DESC_LENGTH_OFFSET]); + ret = -EINVAL; + goto out; + } + if (is_kmalloc) memcpy(param_read_buf, &desc_buf[param_offset], param_size); out: @@ -2088,7 +2155,18 @@ static inline int ufshcd_read_power_desc(struct ufs_hba *hba, u8 *buf, u32 size) { - return ufshcd_read_desc(hba, QUERY_DESC_IDN_POWER, 0, buf, size); + int err = 0; + int retries; + + for (retries = QUERY_REQ_RETRIES; retries > 0; retries--) { + /* Read descriptor*/ + err = ufshcd_read_desc(hba, QUERY_DESC_IDN_POWER, 0, buf, size); + if (!err) + break; + dev_dbg(hba->dev, "%s: error %d retrying\n", __func__, err); + } + + return err; } int ufshcd_read_device_desc(struct ufs_hba *hba, u8 *buf, u32 size) @@ -2320,12 +2398,21 @@ static void ufshcd_host_memory_configure(struct ufs_hba *hba) cpu_to_le32(upper_32_bits(cmd_desc_element_addr)); /* Response upiu and prdt offset should be in double words */ - utrdlp[i].response_upiu_offset = + if (hba->quirks & UFSHCD_QUIRK_PRDT_BYTE_GRAN) { + utrdlp[i].response_upiu_offset = + cpu_to_le16(response_offset); + utrdlp[i].prd_table_offset = + cpu_to_le16(prdt_offset); + utrdlp[i].response_upiu_length = + cpu_to_le16(ALIGNED_UPIU_SIZE); + } else { + utrdlp[i].response_upiu_offset = cpu_to_le16((response_offset >> 2)); - utrdlp[i].prd_table_offset = + utrdlp[i].prd_table_offset = cpu_to_le16((prdt_offset >> 2)); - utrdlp[i].response_upiu_length = + utrdlp[i].response_upiu_length = cpu_to_le16(ALIGNED_UPIU_SIZE >> 2); + } hba->lrb[i].utr_descriptor_ptr = (utrdlp + i); hba->lrb[i].ucd_req_ptr = @@ -2429,10 +2516,10 @@ int ufshcd_dme_set_attr(struct ufs_hba *hba, u32 attr_sel, set, UIC_GET_ATTR_ID(attr_sel), mib_val, ret); } while (ret && peer && --retries); - if (!retries) + if (ret) dev_err(hba->dev, "%s: attr-id 0x%x val 0x%x failed %d retries\n", - set, UIC_GET_ATTR_ID(attr_sel), mib_val, - retries); + set, UIC_GET_ATTR_ID(attr_sel), mib_val, + UFS_UIC_COMMAND_RETRIES - retries); return ret; } @@ -2496,9 +2583,10 @@ int ufshcd_dme_get_attr(struct ufs_hba *hba, u32 attr_sel, get, UIC_GET_ATTR_ID(attr_sel), ret); } while (ret && peer && --retries); - if (!retries) + if (ret) dev_err(hba->dev, "%s: attr-id 0x%x failed %d retries\n", - get, UIC_GET_ATTR_ID(attr_sel), retries); + get, UIC_GET_ATTR_ID(attr_sel), + UFS_UIC_COMMAND_RETRIES - retries); if (mib_val && !ret) *mib_val = uic_cmd.argument3; @@ -2651,6 +2739,8 @@ static int __ufshcd_uic_hibern8_enter(struct ufs_hba *hba) int ret; struct uic_command uic_cmd = {0}; + ufshcd_vops_hibern8_notify(hba, UIC_CMD_DME_HIBER_ENTER, PRE_CHANGE); + uic_cmd.command = UIC_CMD_DME_HIBER_ENTER; ret = ufshcd_uic_pwr_ctrl(hba, &uic_cmd); @@ -2664,7 +2754,9 @@ static int __ufshcd_uic_hibern8_enter(struct ufs_hba *hba) */ if (ufshcd_link_recovery(hba)) ret = -ENOLINK; - } + } else + ufshcd_vops_hibern8_notify(hba, UIC_CMD_DME_HIBER_ENTER, + POST_CHANGE); return ret; } @@ -2687,13 +2779,17 @@ static int ufshcd_uic_hibern8_exit(struct ufs_hba *hba) struct uic_command uic_cmd = {0}; int ret; + ufshcd_vops_hibern8_notify(hba, UIC_CMD_DME_HIBER_EXIT, PRE_CHANGE); + uic_cmd.command = UIC_CMD_DME_HIBER_EXIT; ret = ufshcd_uic_pwr_ctrl(hba, &uic_cmd); if (ret) { dev_err(hba->dev, "%s: hibern8 exit failed. ret = %d\n", __func__, ret); ret = ufshcd_link_recovery(hba); - } + } else + ufshcd_vops_hibern8_notify(hba, UIC_CMD_DME_HIBER_EXIT, + POST_CHANGE); return ret; } @@ -2725,8 +2821,8 @@ static int ufshcd_get_max_pwr_mode(struct ufs_hba *hba) if (hba->max_pwr_info.is_valid) return 0; - pwr_info->pwr_tx = FASTAUTO_MODE; - pwr_info->pwr_rx = FASTAUTO_MODE; + pwr_info->pwr_tx = FAST_MODE; + pwr_info->pwr_rx = FAST_MODE; pwr_info->hs_rate = PA_HS_MODE_B; /* Get the connected lane count */ @@ -2757,7 +2853,7 @@ static int ufshcd_get_max_pwr_mode(struct ufs_hba *hba) __func__, pwr_info->gear_rx); return -EINVAL; } - pwr_info->pwr_rx = SLOWAUTO_MODE; + pwr_info->pwr_rx = SLOW_MODE; } ufshcd_dme_peer_get(hba, UIC_ARG_MIB(PA_MAXRXHSGEAR), @@ -2770,7 +2866,7 @@ static int ufshcd_get_max_pwr_mode(struct ufs_hba *hba) __func__, pwr_info->gear_tx); return -EINVAL; } - pwr_info->pwr_tx = SLOWAUTO_MODE; + pwr_info->pwr_tx = SLOW_MODE; } hba->max_pwr_info.is_valid = true; @@ -3090,7 +3186,16 @@ static int ufshcd_link_startup(struct ufs_hba *hba) { int ret; int retries = DME_LINKSTARTUP_RETRIES; + bool link_startup_again = false; + + /* + * If UFS device isn't active then we will have to issue link startup + * 2 times to make sure the device state move to active. + */ + if (!ufshcd_is_ufs_dev_active(hba)) + link_startup_again = true; +link_startup: do { ufshcd_vops_link_startup_notify(hba, PRE_CHANGE); @@ -3116,6 +3221,12 @@ static int ufshcd_link_startup(struct ufs_hba *hba) /* failed to get the link up... retire */ goto out; + if (link_startup_again) { + link_startup_again = false; + retries = DME_LINKSTARTUP_RETRIES; + goto link_startup; + } + if (hba->quirks & UFSHCD_QUIRK_BROKEN_LCC) { ret = ufshcd_disable_device_tx_lcc(hba); if (ret) @@ -3181,16 +3292,24 @@ static void ufshcd_set_queue_depth(struct scsi_device *sdev) { int ret = 0; u8 lun_qdepth; + int retries; struct ufs_hba *hba; hba = shost_priv(sdev->host); lun_qdepth = hba->nutrs; - ret = ufshcd_read_unit_desc_param(hba, - ufshcd_scsi_to_upiu_lun(sdev->lun), - UNIT_DESC_PARAM_LU_Q_DEPTH, - &lun_qdepth, - sizeof(lun_qdepth)); + for (retries = QUERY_REQ_RETRIES; retries > 0; retries--) { + /* Read descriptor*/ + ret = ufshcd_read_unit_desc_param(hba, + ufshcd_scsi_to_upiu_lun(sdev->lun), + UNIT_DESC_PARAM_LU_Q_DEPTH, + &lun_qdepth, + sizeof(lun_qdepth)); + if (!ret || ret == -ENOTSUPP) + break; + + dev_dbg(hba->dev, "%s: error %d retrying\n", __func__, ret); + } /* Some WLUN doesn't support unit descriptor */ if (ret == -EOPNOTSUPP) @@ -4097,6 +4216,17 @@ static void ufshcd_update_uic_error(struct ufs_hba *hba) { u32 reg; + /* PHY layer lane error */ + reg = ufshcd_readl(hba, REG_UIC_ERROR_CODE_PHY_ADAPTER_LAYER); + /* Ignore LINERESET indication, as this is not an error */ + if ((reg & UIC_PHY_ADAPTER_LAYER_ERROR) && + (reg & UIC_PHY_ADAPTER_LAYER_LANE_ERR_MASK)) + /* + * To know whether this error is fatal or not, DB timeout + * must be checked but this error is handled separately. + */ + dev_dbg(hba->dev, "%s: UIC Lane error reported\n", __func__); + /* PA_INIT_ERROR is fatal and needs UIC reset */ reg = ufshcd_readl(hba, REG_UIC_ERROR_CODE_DATA_LINK_LAYER); if (reg & UIC_DATA_LINK_LAYER_ERROR_PA_INIT) @@ -4158,7 +4288,7 @@ static void ufshcd_check_errors(struct ufs_hba *hba) /* block commands from scsi mid-layer */ scsi_block_requests(hba->host); - hba->ufshcd_state = UFSHCD_STATE_ERROR; + hba->ufshcd_state = UFSHCD_STATE_EH_SCHEDULED; schedule_work(&hba->eh_work); } } @@ -4311,6 +4441,8 @@ static int ufshcd_issue_tm_cmd(struct ufs_hba *hba, int lun_id, int task_id, task_req_upiup->input_param1 = cpu_to_be32(lun_id); task_req_upiup->input_param2 = cpu_to_be32(task_id); + ufshcd_vops_setup_task_mgmt(hba, free_slot, tm_function); + /* send command to the controller */ __set_bit(free_slot, &hba->outstanding_tasks); @@ -4318,6 +4450,8 @@ static int ufshcd_issue_tm_cmd(struct ufs_hba *hba, int lun_id, int task_id, wmb(); ufshcd_writel(hba, 1 << free_slot, REG_UTP_TASK_REQ_DOOR_BELL); + /* Make sure that doorbell is committed immediately */ + wmb(); spin_unlock_irqrestore(host->host_lock, flags); @@ -4722,6 +4856,24 @@ out: return icc_level; } +static int ufshcd_set_icc_levels_attr(struct ufs_hba *hba, u32 icc_level) +{ + int ret = 0; + int retries; + + for (retries = QUERY_REQ_RETRIES; retries > 0; retries--) { + /* write attribute */ + ret = ufshcd_query_attr(hba, UPIU_QUERY_OPCODE_WRITE_ATTR, + QUERY_ATTR_IDN_ACTIVE_ICC_LVL, 0, 0, &icc_level); + if (!ret) + break; + + dev_dbg(hba->dev, "%s: failed with error %d\n", __func__, ret); + } + + return ret; +} + static void ufshcd_init_icc_levels(struct ufs_hba *hba) { int ret; @@ -4742,9 +4894,8 @@ static void ufshcd_init_icc_levels(struct ufs_hba *hba) dev_dbg(hba->dev, "%s: setting icc_level 0x%x", __func__, hba->init_prefetch_data.icc_level); - ret = ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_WRITE_ATTR, - QUERY_ATTR_IDN_ACTIVE_ICC_LVL, 0, 0, - &hba->init_prefetch_data.icc_level); + ret = ufshcd_set_icc_levels_attr(hba, + hba->init_prefetch_data.icc_level); if (ret) dev_err(hba->dev, @@ -4965,6 +5116,76 @@ out: return ret; } +/** + * ufshcd_quirk_tune_host_pa_tactivate - Ensures that host PA_TACTIVATE is + * less than device PA_TACTIVATE time. + * @hba: per-adapter instance + * + * Some UFS devices require host PA_TACTIVATE to be lower than device + * PA_TACTIVATE, we need to enable UFS_DEVICE_QUIRK_HOST_PA_TACTIVATE quirk + * for such devices. + * + * Returns zero on success, non-zero error value on failure. + */ +static int ufshcd_quirk_tune_host_pa_tactivate(struct ufs_hba *hba) +{ + int ret = 0; + u32 granularity, peer_granularity; + u32 pa_tactivate, peer_pa_tactivate; + u32 pa_tactivate_us, peer_pa_tactivate_us; + u8 gran_to_us_table[] = {1, 4, 8, 16, 32, 100}; + + ret = ufshcd_dme_get(hba, UIC_ARG_MIB(PA_GRANULARITY), + &granularity); + if (ret) + goto out; + + ret = ufshcd_dme_peer_get(hba, UIC_ARG_MIB(PA_GRANULARITY), + &peer_granularity); + if (ret) + goto out; + + if ((granularity < PA_GRANULARITY_MIN_VAL) || + (granularity > PA_GRANULARITY_MAX_VAL)) { + dev_err(hba->dev, "%s: invalid host PA_GRANULARITY %d", + __func__, granularity); + return -EINVAL; + } + + if ((peer_granularity < PA_GRANULARITY_MIN_VAL) || + (peer_granularity > PA_GRANULARITY_MAX_VAL)) { + dev_err(hba->dev, "%s: invalid device PA_GRANULARITY %d", + __func__, peer_granularity); + return -EINVAL; + } + + ret = ufshcd_dme_get(hba, UIC_ARG_MIB(PA_TACTIVATE), &pa_tactivate); + if (ret) + goto out; + + ret = ufshcd_dme_peer_get(hba, UIC_ARG_MIB(PA_TACTIVATE), + &peer_pa_tactivate); + if (ret) + goto out; + + pa_tactivate_us = pa_tactivate * gran_to_us_table[granularity - 1]; + peer_pa_tactivate_us = peer_pa_tactivate * + gran_to_us_table[peer_granularity - 1]; + + if (pa_tactivate_us > peer_pa_tactivate_us) { + u32 new_peer_pa_tactivate; + + new_peer_pa_tactivate = pa_tactivate_us / + gran_to_us_table[peer_granularity - 1]; + new_peer_pa_tactivate++; + ret = ufshcd_dme_peer_set(hba, UIC_ARG_MIB(PA_TACTIVATE), + new_peer_pa_tactivate); + } + +out: + return ret; +} + static void ufshcd_tune_unipro_params(struct ufs_hba *hba) { if (ufshcd_is_unipro_pa_params_tuning_req(hba)) { @@ -4975,6 +5196,9 @@ static void ufshcd_tune_unipro_params(struct ufs_hba *hba) if (hba->dev_quirks & UFS_DEVICE_QUIRK_PA_TACTIVATE) /* set 1ms timeout for PA_TACTIVATE */ ufshcd_dme_set(hba, UIC_ARG_MIB(PA_TACTIVATE), 10); + + if (hba->dev_quirks & UFS_DEVICE_QUIRK_HOST_PA_TACTIVATE) + ufshcd_quirk_tune_host_pa_tactivate(hba); } /** @@ -5027,9 +5251,11 @@ static int ufshcd_probe_hba(struct ufs_hba *hba) __func__); } else { ret = ufshcd_config_pwr_mode(hba, &hba->max_pwr_info.info); - if (ret) + if (ret) { dev_err(hba->dev, "%s: Failed setting power mode, err = %d\n", __func__, ret); + goto out; + } } /* set the state as operational after switching to desired gear */ @@ -5062,8 +5288,7 @@ static int ufshcd_probe_hba(struct ufs_hba *hba) hba->is_init_prefetch = true; /* Resume devfreq after UFS device is detected */ - if (ufshcd_is_clkscaling_enabled(hba)) - devfreq_resume_device(hba->devfreq); + ufshcd_resume_clkscaling(hba); out: /* @@ -5389,6 +5614,10 @@ static int __ufshcd_setup_clocks(struct ufs_hba *hba, bool on, if (!head || list_empty(head)) goto out; + ret = ufshcd_vops_setup_clocks(hba, on, PRE_CHANGE); + if (ret) + return ret; + list_for_each_entry(clki, head, list) { if (!IS_ERR_OR_NULL(clki->clk)) { if (skip_ref_clk && !strcmp(clki->name, "ref_clk")) @@ -5410,7 +5639,10 @@ static int __ufshcd_setup_clocks(struct ufs_hba *hba, bool on, } } - ret = ufshcd_vops_setup_clocks(hba, on); + ret = ufshcd_vops_setup_clocks(hba, on, POST_CHANGE); + if (ret) + return ret; + out: if (ret) { list_for_each_entry(clki, head, list) { @@ -5500,8 +5732,6 @@ static void ufshcd_variant_hba_exit(struct ufs_hba *hba) if (!hba->vops) return; - ufshcd_vops_setup_clocks(hba, false); - ufshcd_vops_setup_regulators(hba, false); ufshcd_vops_exit(hba); @@ -5564,6 +5794,7 @@ static void ufshcd_hba_exit(struct ufs_hba *hba) if (hba->is_powered) { ufshcd_variant_hba_exit(hba); ufshcd_setup_vreg(hba, false); + ufshcd_suspend_clkscaling(hba); ufshcd_setup_clocks(hba, false); ufshcd_setup_hba_vreg(hba, false); hba->is_powered = false; @@ -5577,19 +5808,19 @@ ufshcd_send_request_sense(struct ufs_hba *hba, struct scsi_device *sdp) 0, 0, 0, - SCSI_SENSE_BUFFERSIZE, + UFSHCD_REQ_SENSE_SIZE, 0}; char *buffer; int ret; - buffer = kzalloc(SCSI_SENSE_BUFFERSIZE, GFP_KERNEL); + buffer = kzalloc(UFSHCD_REQ_SENSE_SIZE, GFP_KERNEL); if (!buffer) { ret = -ENOMEM; goto out; } ret = scsi_execute_req_flags(sdp, cmd, DMA_FROM_DEVICE, buffer, - SCSI_SENSE_BUFFERSIZE, NULL, + UFSHCD_REQ_SENSE_SIZE, NULL, msecs_to_jiffies(1000), 3, NULL, 0, RQF_PM); if (ret) pr_err("%s: failed with err %d\n", __func__, ret); @@ -5766,7 +5997,6 @@ static int ufshcd_vreg_set_hpm(struct ufs_hba *hba) !hba->dev_info.is_lu_power_on_wp) { ret = ufshcd_setup_vreg(hba, true); } else if (!ufshcd_is_ufs_dev_active(hba)) { - ret = ufshcd_toggle_vreg(hba->dev, hba->vreg_info.vcc, true); if (!ret && !ufshcd_is_link_active(hba)) { ret = ufshcd_config_vreg_hpm(hba, hba->vreg_info.vccq); if (ret) @@ -5775,6 +6005,7 @@ static int ufshcd_vreg_set_hpm(struct ufs_hba *hba) if (ret) goto vccq_lpm; } + ret = ufshcd_toggle_vreg(hba->dev, hba->vreg_info.vcc, true); } goto out; @@ -5839,6 +6070,8 @@ static int ufshcd_suspend(struct ufs_hba *hba, enum ufs_pm_op pm_op) ufshcd_hold(hba, false); hba->clk_gating.is_suspended = true; + ufshcd_suspend_clkscaling(hba); + if (req_dev_pwr_mode == UFS_ACTIVE_PWR_MODE && req_link_state == UIC_LINK_ACTIVE_STATE) { goto disable_clks; @@ -5846,12 +6079,12 @@ static int ufshcd_suspend(struct ufs_hba *hba, enum ufs_pm_op pm_op) if ((req_dev_pwr_mode == hba->curr_dev_pwr_mode) && (req_link_state == hba->uic_link_state)) - goto out; + goto enable_gating; /* UFS device & link must be active before we enter in this function */ if (!ufshcd_is_ufs_dev_active(hba) || !ufshcd_is_link_active(hba)) { ret = -EINVAL; - goto out; + goto enable_gating; } if (ufshcd_is_runtime_pm(pm_op)) { @@ -5888,15 +6121,6 @@ static int ufshcd_suspend(struct ufs_hba *hba, enum ufs_pm_op pm_op) disable_clks: /* - * The clock scaling needs access to controller registers. Hence, Wait - * for pending clock scaling work to be done before clocks are - * turned off. - */ - if (ufshcd_is_clkscaling_enabled(hba)) { - devfreq_suspend_device(hba->devfreq); - hba->clk_scaling.window_start_t = 0; - } - /* * Call vendor specific suspend callback. As these callbacks may access * vendor specific host controller register space call them before the * host clocks are ON. @@ -5905,10 +6129,6 @@ disable_clks: if (ret) goto set_link_active; - ret = ufshcd_vops_setup_clocks(hba, false); - if (ret) - goto vops_resume; - if (!ufshcd_is_link_active(hba)) ufshcd_setup_clocks(hba, false); else @@ -5925,9 +6145,8 @@ disable_clks: ufshcd_hba_vreg_set_lpm(hba); goto out; -vops_resume: - ufshcd_vops_resume(hba, pm_op); set_link_active: + ufshcd_resume_clkscaling(hba); ufshcd_vreg_set_hpm(hba); if (ufshcd_is_link_hibern8(hba) && !ufshcd_uic_hibern8_exit(hba)) ufshcd_set_link_active(hba); @@ -5937,6 +6156,7 @@ set_dev_active: if (!ufshcd_set_dev_pwr_mode(hba, UFS_ACTIVE_PWR_MODE)) ufshcd_disable_auto_bkops(hba); enable_gating: + ufshcd_resume_clkscaling(hba); hba->clk_gating.is_suspended = false; ufshcd_release(hba); out: @@ -6015,8 +6235,7 @@ static int ufshcd_resume(struct ufs_hba *hba, enum ufs_pm_op pm_op) ufshcd_urgent_bkops(hba); hba->clk_gating.is_suspended = false; - if (ufshcd_is_clkscaling_enabled(hba)) - devfreq_resume_device(hba->devfreq); + ufshcd_resume_clkscaling(hba); /* Schedule clock gating in case of no access to UFS device yet */ ufshcd_release(hba); @@ -6030,6 +6249,7 @@ disable_vreg: ufshcd_vreg_set_lpm(hba); disable_irq_and_vops_clks: ufshcd_disable_irq(hba); + ufshcd_suspend_clkscaling(hba); ufshcd_setup_clocks(hba, false); out: hba->pm_op_in_progress = 0; @@ -6052,16 +6272,13 @@ int ufshcd_system_suspend(struct ufs_hba *hba) if (!hba || !hba->is_powered) return 0; - if (pm_runtime_suspended(hba->dev)) { - if (hba->rpm_lvl == hba->spm_lvl) - /* - * There is possibility that device may still be in - * active state during the runtime suspend. - */ - if ((ufs_get_pm_lvl_to_dev_pwr_mode(hba->spm_lvl) == - hba->curr_dev_pwr_mode) && !hba->auto_bkops_enabled) - goto out; + if ((ufs_get_pm_lvl_to_dev_pwr_mode(hba->spm_lvl) == + hba->curr_dev_pwr_mode) && + (ufs_get_pm_lvl_to_link_pwr_state(hba->spm_lvl) == + hba->uic_link_state)) + goto out; + if (pm_runtime_suspended(hba->dev)) { /* * UFS device and/or UFS link low power states during runtime * suspend seems to be different than what is expected during @@ -6092,7 +6309,10 @@ EXPORT_SYMBOL(ufshcd_system_suspend); int ufshcd_system_resume(struct ufs_hba *hba) { - if (!hba || !hba->is_powered || pm_runtime_suspended(hba->dev)) + if (!hba) + return -EINVAL; + + if (!hba->is_powered || pm_runtime_suspended(hba->dev)) /* * Let the runtime resume take care of resuming * if runtime suspended. @@ -6113,7 +6333,10 @@ EXPORT_SYMBOL(ufshcd_system_resume); */ int ufshcd_runtime_suspend(struct ufs_hba *hba) { - if (!hba || !hba->is_powered) + if (!hba) + return -EINVAL; + + if (!hba->is_powered) return 0; return ufshcd_suspend(hba, UFS_RUNTIME_PM); @@ -6143,10 +6366,13 @@ EXPORT_SYMBOL(ufshcd_runtime_suspend); */ int ufshcd_runtime_resume(struct ufs_hba *hba) { - if (!hba || !hba->is_powered) + if (!hba) + return -EINVAL; + + if (!hba->is_powered) return 0; - else - return ufshcd_resume(hba, UFS_RUNTIME_PM); + + return ufshcd_resume(hba, UFS_RUNTIME_PM); } EXPORT_SYMBOL(ufshcd_runtime_resume); @@ -6198,11 +6424,7 @@ void ufshcd_remove(struct ufs_hba *hba) ufshcd_disable_intr(hba, hba->intr_mask); ufshcd_hba_stop(hba, true); - scsi_host_put(hba->host); - ufshcd_exit_clk_gating(hba); - if (ufshcd_is_clkscaling_enabled(hba)) - devfreq_remove_device(hba->devfreq); ufshcd_hba_exit(hba); } EXPORT_SYMBOL_GPL(ufshcd_remove); @@ -6324,15 +6546,47 @@ static int ufshcd_devfreq_target(struct device *dev, { int err = 0; struct ufs_hba *hba = dev_get_drvdata(dev); + bool release_clk_hold = false; + unsigned long irq_flags; if (!ufshcd_is_clkscaling_enabled(hba)) return -EINVAL; + spin_lock_irqsave(hba->host->host_lock, irq_flags); + if (ufshcd_eh_in_progress(hba)) { + spin_unlock_irqrestore(hba->host->host_lock, irq_flags); + return 0; + } + + if (ufshcd_is_clkgating_allowed(hba) && + (hba->clk_gating.state != CLKS_ON)) { + if (cancel_delayed_work(&hba->clk_gating.gate_work)) { + /* hold the vote until the scaling work is completed */ + hba->clk_gating.active_reqs++; + release_clk_hold = true; + hba->clk_gating.state = CLKS_ON; + } else { + /* + * Clock gating work seems to be running in parallel + * hence skip scaling work to avoid deadlock between + * current scaling work and gating work. + */ + spin_unlock_irqrestore(hba->host->host_lock, irq_flags); + return 0; + } + } + spin_unlock_irqrestore(hba->host->host_lock, irq_flags); + if (*freq == UINT_MAX) err = ufshcd_scale_clks(hba, true); else if (*freq == 0) err = ufshcd_scale_clks(hba, false); + spin_lock_irqsave(hba->host->host_lock, irq_flags); + if (release_clk_hold) + __ufshcd_release(hba); + spin_unlock_irqrestore(hba->host->host_lock, irq_flags); + return err; } @@ -6498,7 +6752,7 @@ int ufshcd_init(struct ufs_hba *hba, void __iomem *mmio_base, unsigned int irq) } if (ufshcd_is_clkscaling_enabled(hba)) { - hba->devfreq = devfreq_add_device(dev, &ufs_devfreq_profile, + hba->devfreq = devm_devfreq_add_device(dev, &ufs_devfreq_profile, "simple_ondemand", NULL); if (IS_ERR(hba->devfreq)) { dev_err(hba->dev, "Unable to register with devfreq %ld\n", @@ -6507,18 +6761,19 @@ int ufshcd_init(struct ufs_hba *hba, void __iomem *mmio_base, unsigned int irq) goto out_remove_scsi_host; } /* Suspend devfreq until the UFS device is detected */ - devfreq_suspend_device(hba->devfreq); - hba->clk_scaling.window_start_t = 0; + ufshcd_suspend_clkscaling(hba); } /* Hold auto suspend until async scan completes */ pm_runtime_get_sync(dev); /* - * The device-initialize-sequence hasn't been invoked yet. - * Set the device to power-off state + * We are assuming that device wasn't put in sleep/power-down + * state exclusively during the boot stage before kernel. + * This assumption helps avoid doing link startup twice during + * ufshcd_probe_hba(). */ - ufshcd_set_ufs_dev_poweroff(hba); + ufshcd_set_ufs_dev_active(hba); async_schedule(ufshcd_async_scan, hba); @@ -6530,7 +6785,6 @@ exit_gating: ufshcd_exit_clk_gating(hba); out_disable: hba->is_irq_enabled = false; - scsi_host_put(host); ufshcd_hba_exit(hba); out_error: return err; diff --git a/drivers/scsi/ufs/ufshcd.h b/drivers/scsi/ufs/ufshcd.h index 430bef1..7d9ff22 100644 --- a/drivers/scsi/ufs/ufshcd.h +++ b/drivers/scsi/ufs/ufshcd.h @@ -261,6 +261,12 @@ struct ufs_pwr_mode_info { * @pwr_change_notify: called before and after a power mode change * is carried out to allow vendor spesific capabilities * to be set. + * @setup_xfer_req: called before any transfer request is issued + * to set some things + * @setup_task_mgmt: called before any task management request is issued + * to set some things + * @hibern8_notify: called around hibern8 enter/exit + * to configure some things * @suspend: called during host controller PM callback * @resume: called during host controller PM callback * @dbg_register_dump: used to dump controller debug information @@ -273,7 +279,8 @@ struct ufs_hba_variant_ops { u32 (*get_ufs_hci_version)(struct ufs_hba *); int (*clk_scale_notify)(struct ufs_hba *, bool, enum ufs_notify_change_status); - int (*setup_clocks)(struct ufs_hba *, bool); + int (*setup_clocks)(struct ufs_hba *, bool, + enum ufs_notify_change_status); int (*setup_regulators)(struct ufs_hba *, bool); int (*hce_enable_notify)(struct ufs_hba *, enum ufs_notify_change_status); @@ -283,6 +290,10 @@ struct ufs_hba_variant_ops { enum ufs_notify_change_status status, struct ufs_pa_layer_attr *, struct ufs_pa_layer_attr *); + void (*setup_xfer_req)(struct ufs_hba *, int, bool); + void (*setup_task_mgmt)(struct ufs_hba *, int, u8); + void (*hibern8_notify)(struct ufs_hba *, enum uic_cmd_dme, + enum ufs_notify_change_status); int (*suspend)(struct ufs_hba *, enum ufs_pm_op); int (*resume)(struct ufs_hba *, enum ufs_pm_op); void (*dbg_register_dump)(struct ufs_hba *hba); @@ -474,6 +485,12 @@ struct ufs_hba { */ #define UFSHCD_QUIRK_BROKEN_UFS_HCI_VERSION UFS_BIT(5) + /* + * This quirk needs to be enabled if the host contoller regards + * resolution of the values of PRDTO and PRDTL in UTRD as byte. + */ + #define UFSHCD_QUIRK_PRDT_BYTE_GRAN UFS_BIT(7) + unsigned int quirks; /* Deviations from standard UFSHCI spec. */ /* Device deviations from standard UFS device spec. */ @@ -755,10 +772,11 @@ static inline int ufshcd_vops_clk_scale_notify(struct ufs_hba *hba, return 0; } -static inline int ufshcd_vops_setup_clocks(struct ufs_hba *hba, bool on) +static inline int ufshcd_vops_setup_clocks(struct ufs_hba *hba, bool on, + enum ufs_notify_change_status status) { if (hba->vops && hba->vops->setup_clocks) - return hba->vops->setup_clocks(hba, on); + return hba->vops->setup_clocks(hba, on, status); return 0; } @@ -799,6 +817,28 @@ static inline int ufshcd_vops_pwr_change_notify(struct ufs_hba *hba, return -ENOTSUPP; } +static inline void ufshcd_vops_setup_xfer_req(struct ufs_hba *hba, int tag, + bool is_scsi_cmd) +{ + if (hba->vops && hba->vops->setup_xfer_req) + return hba->vops->setup_xfer_req(hba, tag, is_scsi_cmd); +} + +static inline void ufshcd_vops_setup_task_mgmt(struct ufs_hba *hba, + int tag, u8 tm_function) +{ + if (hba->vops && hba->vops->setup_task_mgmt) + return hba->vops->setup_task_mgmt(hba, tag, tm_function); +} + +static inline void ufshcd_vops_hibern8_notify(struct ufs_hba *hba, + enum uic_cmd_dme cmd, + enum ufs_notify_change_status status) +{ + if (hba->vops && hba->vops->hibern8_notify) + return hba->vops->hibern8_notify(hba, cmd, status); +} + static inline int ufshcd_vops_suspend(struct ufs_hba *hba, enum ufs_pm_op op) { if (hba->vops && hba->vops->suspend) diff --git a/drivers/scsi/ufs/ufshci.h b/drivers/scsi/ufs/ufshci.h index 9599741..5d97886 100644 --- a/drivers/scsi/ufs/ufshci.h +++ b/drivers/scsi/ufs/ufshci.h @@ -83,6 +83,8 @@ enum { MASK_UIC_DME_TEST_MODE_SUPPORT = 0x04000000, }; +#define UFS_MASK(mask, offset) ((mask) << (offset)) + /* UFS Version 08h */ #define MINOR_VERSION_NUM_MASK UFS_MASK(0xFFFF, 0) #define MAJOR_VERSION_NUM_MASK UFS_MASK(0xFFFF, 16) @@ -166,6 +168,7 @@ enum { /* UECPA - Host UIC Error Code PHY Adapter Layer 38h */ #define UIC_PHY_ADAPTER_LAYER_ERROR UFS_BIT(31) #define UIC_PHY_ADAPTER_LAYER_ERROR_CODE_MASK 0x1F +#define UIC_PHY_ADAPTER_LAYER_LANE_ERR_MASK 0xF /* UECDL - Host UIC Error Code Data Link Layer 3Ch */ #define UIC_DATA_LINK_LAYER_ERROR UFS_BIT(31) diff --git a/drivers/scsi/ufs/unipro.h b/drivers/scsi/ufs/unipro.h index eff8b56..23129d7 100644 --- a/drivers/scsi/ufs/unipro.h +++ b/drivers/scsi/ufs/unipro.h @@ -123,6 +123,7 @@ #define PA_MAXRXHSGEAR 0x1587 #define PA_RXHSUNTERMCAP 0x15A5 #define PA_RXLSTERMCAP 0x15A6 +#define PA_GRANULARITY 0x15AA #define PA_PACPREQTIMEOUT 0x1590 #define PA_PACPREQEOBTIMEOUT 0x1591 #define PA_HIBERN8TIME 0x15A7 @@ -158,6 +159,9 @@ #define VS_DEBUGOMC 0xD09E #define VS_POWERSTATE 0xD083 +#define PA_GRANULARITY_MIN_VAL 1 +#define PA_GRANULARITY_MAX_VAL 6 + /* PHY Adapter Protocol Constants */ #define PA_MAXDATALANES 4 diff --git a/drivers/scsi/xen-scsifront.c b/drivers/scsi/xen-scsifront.c index 9dc8687..9aa1fe1 100644 --- a/drivers/scsi/xen-scsifront.c +++ b/drivers/scsi/xen-scsifront.c @@ -79,10 +79,13 @@ struct vscsifrnt_shadow { /* command between backend and frontend */ unsigned char act; + uint8_t nr_segments; uint16_t rqid; + uint16_t ref_rqid; unsigned int nr_grants; /* number of grants in gref[] */ struct scsiif_request_segment *sg; /* scatter/gather elements */ + struct scsiif_request_segment seg[VSCSIIF_SG_TABLESIZE]; /* Do reset or abort function. */ wait_queue_head_t wq_reset; /* reset work queue */ @@ -172,68 +175,90 @@ static void scsifront_put_rqid(struct vscsifrnt_info *info, uint32_t id) scsifront_wake_up(info); } -static struct vscsiif_request *scsifront_pre_req(struct vscsifrnt_info *info) +static int scsifront_do_request(struct vscsifrnt_info *info, + struct vscsifrnt_shadow *shadow) { struct vscsiif_front_ring *ring = &(info->ring); struct vscsiif_request *ring_req; + struct scsi_cmnd *sc = shadow->sc; uint32_t id; + int i, notify; + + if (RING_FULL(&info->ring)) + return -EBUSY; id = scsifront_get_rqid(info); /* use id in response */ if (id >= VSCSIIF_MAX_REQS) - return NULL; + return -EBUSY; - ring_req = RING_GET_REQUEST(&(info->ring), ring->req_prod_pvt); + info->shadow[id] = shadow; + shadow->rqid = id; + ring_req = RING_GET_REQUEST(&(info->ring), ring->req_prod_pvt); ring->req_prod_pvt++; - ring_req->rqid = (uint16_t)id; + ring_req->rqid = id; + ring_req->act = shadow->act; + ring_req->ref_rqid = shadow->ref_rqid; + ring_req->nr_segments = shadow->nr_segments; - return ring_req; -} + ring_req->id = sc->device->id; + ring_req->lun = sc->device->lun; + ring_req->channel = sc->device->channel; + ring_req->cmd_len = sc->cmd_len; -static void scsifront_do_request(struct vscsifrnt_info *info) -{ - struct vscsiif_front_ring *ring = &(info->ring); - int notify; + BUG_ON(sc->cmd_len > VSCSIIF_MAX_COMMAND_SIZE); + + memcpy(ring_req->cmnd, sc->cmnd, sc->cmd_len); + + ring_req->sc_data_direction = (uint8_t)sc->sc_data_direction; + ring_req->timeout_per_command = sc->request->timeout / HZ; + + for (i = 0; i < (shadow->nr_segments & ~VSCSIIF_SG_GRANT); i++) + ring_req->seg[i] = shadow->seg[i]; RING_PUSH_REQUESTS_AND_CHECK_NOTIFY(ring, notify); if (notify) notify_remote_via_irq(info->irq); + + return 0; } -static void scsifront_gnttab_done(struct vscsifrnt_info *info, uint32_t id) +static void scsifront_gnttab_done(struct vscsifrnt_info *info, + struct vscsifrnt_shadow *shadow) { - struct vscsifrnt_shadow *s = info->shadow[id]; int i; - if (s->sc->sc_data_direction == DMA_NONE) + if (shadow->sc->sc_data_direction == DMA_NONE) return; - for (i = 0; i < s->nr_grants; i++) { - if (unlikely(gnttab_query_foreign_access(s->gref[i]) != 0)) { + for (i = 0; i < shadow->nr_grants; i++) { + if (unlikely(gnttab_query_foreign_access(shadow->gref[i]))) { shost_printk(KERN_ALERT, info->host, KBUILD_MODNAME "grant still in use by backend\n"); BUG(); } - gnttab_end_foreign_access(s->gref[i], 0, 0UL); + gnttab_end_foreign_access(shadow->gref[i], 0, 0UL); } - kfree(s->sg); + kfree(shadow->sg); } static void scsifront_cdb_cmd_done(struct vscsifrnt_info *info, struct vscsiif_response *ring_rsp) { + struct vscsifrnt_shadow *shadow; struct scsi_cmnd *sc; uint32_t id; uint8_t sense_len; id = ring_rsp->rqid; - sc = info->shadow[id]->sc; + shadow = info->shadow[id]; + sc = shadow->sc; BUG_ON(sc == NULL); - scsifront_gnttab_done(info, id); + scsifront_gnttab_done(info, shadow); scsifront_put_rqid(info, id); sc->result = ring_rsp->rslt; @@ -366,7 +391,6 @@ static void scsifront_finish_all(struct vscsifrnt_info *info) static int map_data_for_request(struct vscsifrnt_info *info, struct scsi_cmnd *sc, - struct vscsiif_request *ring_req, struct vscsifrnt_shadow *shadow) { grant_ref_t gref_head; @@ -379,7 +403,6 @@ static int map_data_for_request(struct vscsifrnt_info *info, struct scatterlist *sg; struct scsiif_request_segment *seg; - ring_req->nr_segments = 0; if (sc->sc_data_direction == DMA_NONE || !data_len) return 0; @@ -398,7 +421,7 @@ static int map_data_for_request(struct vscsifrnt_info *info, if (!shadow->sg) return -ENOMEM; } - seg = shadow->sg ? : ring_req->seg; + seg = shadow->sg ? : shadow->seg; err = gnttab_alloc_grant_references(seg_grants + data_grants, &gref_head); @@ -423,9 +446,9 @@ static int map_data_for_request(struct vscsifrnt_info *info, info->dev->otherend_id, xen_page_to_gfn(page), 1); shadow->gref[ref_cnt] = ref; - ring_req->seg[ref_cnt].gref = ref; - ring_req->seg[ref_cnt].offset = (uint16_t)off; - ring_req->seg[ref_cnt].length = (uint16_t)bytes; + shadow->seg[ref_cnt].gref = ref; + shadow->seg[ref_cnt].offset = (uint16_t)off; + shadow->seg[ref_cnt].length = (uint16_t)bytes; page++; len -= bytes; @@ -473,44 +496,14 @@ static int map_data_for_request(struct vscsifrnt_info *info, } if (seg_grants) - ring_req->nr_segments = VSCSIIF_SG_GRANT | seg_grants; + shadow->nr_segments = VSCSIIF_SG_GRANT | seg_grants; else - ring_req->nr_segments = (uint8_t)ref_cnt; + shadow->nr_segments = (uint8_t)ref_cnt; shadow->nr_grants = ref_cnt; return 0; } -static struct vscsiif_request *scsifront_command2ring( - struct vscsifrnt_info *info, struct scsi_cmnd *sc, - struct vscsifrnt_shadow *shadow) -{ - struct vscsiif_request *ring_req; - - memset(shadow, 0, sizeof(*shadow)); - - ring_req = scsifront_pre_req(info); - if (!ring_req) - return NULL; - - info->shadow[ring_req->rqid] = shadow; - shadow->rqid = ring_req->rqid; - - ring_req->id = sc->device->id; - ring_req->lun = sc->device->lun; - ring_req->channel = sc->device->channel; - ring_req->cmd_len = sc->cmd_len; - - BUG_ON(sc->cmd_len > VSCSIIF_MAX_COMMAND_SIZE); - - memcpy(ring_req->cmnd, sc->cmnd, sc->cmd_len); - - ring_req->sc_data_direction = (uint8_t)sc->sc_data_direction; - ring_req->timeout_per_command = sc->request->timeout / HZ; - - return ring_req; -} - static int scsifront_enter(struct vscsifrnt_info *info) { if (info->pause) @@ -536,36 +529,25 @@ static int scsifront_queuecommand(struct Scsi_Host *shost, struct scsi_cmnd *sc) { struct vscsifrnt_info *info = shost_priv(shost); - struct vscsiif_request *ring_req; struct vscsifrnt_shadow *shadow = scsi_cmd_priv(sc); unsigned long flags; int err; - uint16_t rqid; + + sc->result = 0; + memset(shadow, 0, sizeof(*shadow)); + + shadow->sc = sc; + shadow->act = VSCSIIF_ACT_SCSI_CDB; spin_lock_irqsave(shost->host_lock, flags); if (scsifront_enter(info)) { spin_unlock_irqrestore(shost->host_lock, flags); return SCSI_MLQUEUE_HOST_BUSY; } - if (RING_FULL(&info->ring)) - goto busy; - - ring_req = scsifront_command2ring(info, sc, shadow); - if (!ring_req) - goto busy; - - sc->result = 0; - - rqid = ring_req->rqid; - ring_req->act = VSCSIIF_ACT_SCSI_CDB; - shadow->sc = sc; - shadow->act = VSCSIIF_ACT_SCSI_CDB; - - err = map_data_for_request(info, sc, ring_req, shadow); + err = map_data_for_request(info, sc, shadow); if (err < 0) { pr_debug("%s: err %d\n", __func__, err); - scsifront_put_rqid(info, rqid); scsifront_return(info); spin_unlock_irqrestore(shost->host_lock, flags); if (err == -ENOMEM) @@ -575,7 +557,11 @@ static int scsifront_queuecommand(struct Scsi_Host *shost, return 0; } - scsifront_do_request(info); + if (scsifront_do_request(info, shadow)) { + scsifront_gnttab_done(info, shadow); + goto busy; + } + scsifront_return(info); spin_unlock_irqrestore(shost->host_lock, flags); @@ -598,26 +584,30 @@ static int scsifront_action_handler(struct scsi_cmnd *sc, uint8_t act) struct Scsi_Host *host = sc->device->host; struct vscsifrnt_info *info = shost_priv(host); struct vscsifrnt_shadow *shadow, *s = scsi_cmd_priv(sc); - struct vscsiif_request *ring_req; int err = 0; - shadow = kmalloc(sizeof(*shadow), GFP_NOIO); + shadow = kzalloc(sizeof(*shadow), GFP_NOIO); if (!shadow) return FAILED; + shadow->act = act; + shadow->rslt_reset = RSLT_RESET_WAITING; + shadow->sc = sc; + shadow->ref_rqid = s->rqid; + init_waitqueue_head(&shadow->wq_reset); + spin_lock_irq(host->host_lock); for (;;) { - if (!RING_FULL(&info->ring)) { - ring_req = scsifront_command2ring(info, sc, shadow); - if (ring_req) - break; - } - if (err || info->pause) { - spin_unlock_irq(host->host_lock); - kfree(shadow); - return FAILED; - } + if (scsifront_enter(info)) + goto fail; + + if (!scsifront_do_request(info, shadow)) + break; + + scsifront_return(info); + if (err) + goto fail; info->wait_ring_available = 1; spin_unlock_irq(host->host_lock); err = wait_event_interruptible(info->wq_sync, @@ -625,22 +615,6 @@ static int scsifront_action_handler(struct scsi_cmnd *sc, uint8_t act) spin_lock_irq(host->host_lock); } - if (scsifront_enter(info)) { - spin_unlock_irq(host->host_lock); - return FAILED; - } - - ring_req->act = act; - ring_req->ref_rqid = s->rqid; - - shadow->act = act; - shadow->rslt_reset = RSLT_RESET_WAITING; - init_waitqueue_head(&shadow->wq_reset); - - ring_req->nr_segments = 0; - - scsifront_do_request(info); - spin_unlock_irq(host->host_lock); err = wait_event_interruptible(shadow->wq_reset, shadow->wait_reset); spin_lock_irq(host->host_lock); @@ -659,6 +633,11 @@ static int scsifront_action_handler(struct scsi_cmnd *sc, uint8_t act) scsifront_return(info); spin_unlock_irq(host->host_lock); return err; + +fail: + spin_unlock_irq(host->host_lock); + kfree(shadow); + return FAILED; } static int scsifront_eh_abort_handler(struct scsi_cmnd *sc) @@ -1060,13 +1039,9 @@ static void scsifront_read_backend_params(struct xenbus_device *dev, struct vscsifrnt_info *info) { unsigned int sg_grant, nr_segs; - int ret; struct Scsi_Host *host = info->host; - ret = xenbus_scanf(XBT_NIL, dev->otherend, "feature-sg-grant", "%u", - &sg_grant); - if (ret != 1) - sg_grant = 0; + sg_grant = xenbus_read_unsigned(dev->otherend, "feature-sg-grant", 0); nr_segs = min_t(unsigned int, sg_grant, SG_ALL); nr_segs = max_t(unsigned int, nr_segs, VSCSIIF_SG_TABLESIZE); nr_segs = min_t(unsigned int, nr_segs, diff --git a/drivers/spi/Kconfig b/drivers/spi/Kconfig index b799547..ec4aa25 100644 --- a/drivers/spi/Kconfig +++ b/drivers/spi/Kconfig @@ -67,6 +67,13 @@ config SPI_ATH79 This enables support for the SPI controller present on the Atheros AR71XX/AR724X/AR913X SoCs. +config SPI_ARMADA_3700 + tristate "Marvell Armada 3700 SPI Controller" + depends on (ARCH_MVEBU && OF) || COMPILE_TEST + help + This enables support for the SPI controller present on the + Marvell Armada 3700 SoCs. + config SPI_ATMEL tristate "Atmel SPI Controller" depends on HAS_DMA @@ -264,6 +271,12 @@ config SPI_FALCON has only been tested with m25p80 type chips. The hardware has no support for other types of SPI peripherals. +config SPI_FSL_LPSPI + tristate "Freescale i.MX LPSPI controller" + depends on ARCH_MXC || COMPILE_TEST + help + This enables Freescale i.MX LPSPI controllers in master mode. + config SPI_GPIO tristate "GPIO-based bitbanging SPI Master" depends on GPIOLIB || COMPILE_TEST @@ -373,7 +386,6 @@ config SPI_FSL_DSPI config SPI_FSL_ESPI tristate "Freescale eSPI controller" depends on FSL_SOC - select SPI_FSL_LIB help This enables using the Freescale eSPI controllers in master mode. From MPC8536, 85xx platform uses the controller, and all P10xx, @@ -451,7 +463,8 @@ config SPI_ORION tristate "Orion SPI master" depends on PLAT_ORION || ARCH_MVEBU || COMPILE_TEST help - This enables using the SPI master controller on the Orion chips. + This enables using the SPI master controller on the Orion + and MVEBU chips. config SPI_PIC32 tristate "Microchip PIC32 series SPI" @@ -553,7 +566,7 @@ config SPI_S3C24XX_FIQ config SPI_S3C64XX tristate "Samsung S3C64XX series type SPI" - depends on (PLAT_SAMSUNG || ARCH_EXYNOS) + depends on (PLAT_SAMSUNG || ARCH_EXYNOS || COMPILE_TEST) help SPI driver for Samsung S3C64XX and newer SoCs. diff --git a/drivers/spi/Makefile b/drivers/spi/Makefile index aa939d9..7a6b646 100644 --- a/drivers/spi/Makefile +++ b/drivers/spi/Makefile @@ -12,6 +12,7 @@ obj-$(CONFIG_SPI_LOOPBACK_TEST) += spi-loopback-test.o # SPI master controller drivers (bus) obj-$(CONFIG_SPI_ALTERA) += spi-altera.o +obj-$(CONFIG_SPI_ARMADA_3700) += spi-armada-3700.o obj-$(CONFIG_SPI_ATMEL) += spi-atmel.o obj-$(CONFIG_SPI_ATH79) += spi-ath79.o obj-$(CONFIG_SPI_AU1550) += spi-au1550.o @@ -43,6 +44,7 @@ obj-$(CONFIG_SPI_FSL_CPM) += spi-fsl-cpm.o obj-$(CONFIG_SPI_FSL_DSPI) += spi-fsl-dspi.o obj-$(CONFIG_SPI_FSL_LIB) += spi-fsl-lib.o obj-$(CONFIG_SPI_FSL_ESPI) += spi-fsl-espi.o +obj-$(CONFIG_SPI_FSL_LPSPI) += spi-fsl-lpspi.o obj-$(CONFIG_SPI_FSL_SPI) += spi-fsl-spi.o obj-$(CONFIG_SPI_GPIO) += spi-gpio.o obj-$(CONFIG_SPI_IMG_SPFI) += spi-img-spfi.o diff --git a/drivers/spi/spi-armada-3700.c b/drivers/spi/spi-armada-3700.c new file mode 100644 index 0000000..e89da0a --- /dev/null +++ b/drivers/spi/spi-armada-3700.c @@ -0,0 +1,923 @@ +/* + * Marvell Armada-3700 SPI controller driver + * + * Copyright (C) 2016 Marvell Ltd. + * + * Author: Wilson Ding <dingwei@marvell.com> + * Author: Romain Perier <romain.perier@free-electrons.com> + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +#include <linux/clk.h> +#include <linux/completion.h> +#include <linux/delay.h> +#include <linux/err.h> +#include <linux/interrupt.h> +#include <linux/io.h> +#include <linux/kernel.h> +#include <linux/module.h> +#include <linux/of.h> +#include <linux/of_irq.h> +#include <linux/of_device.h> +#include <linux/pinctrl/consumer.h> +#include <linux/spi/spi.h> + +#define DRIVER_NAME "armada_3700_spi" + +#define A3700_SPI_TIMEOUT 10 + +/* SPI Register Offest */ +#define A3700_SPI_IF_CTRL_REG 0x00 +#define A3700_SPI_IF_CFG_REG 0x04 +#define A3700_SPI_DATA_OUT_REG 0x08 +#define A3700_SPI_DATA_IN_REG 0x0C +#define A3700_SPI_IF_INST_REG 0x10 +#define A3700_SPI_IF_ADDR_REG 0x14 +#define A3700_SPI_IF_RMODE_REG 0x18 +#define A3700_SPI_IF_HDR_CNT_REG 0x1C +#define A3700_SPI_IF_DIN_CNT_REG 0x20 +#define A3700_SPI_IF_TIME_REG 0x24 +#define A3700_SPI_INT_STAT_REG 0x28 +#define A3700_SPI_INT_MASK_REG 0x2C + +/* A3700_SPI_IF_CTRL_REG */ +#define A3700_SPI_EN BIT(16) +#define A3700_SPI_ADDR_NOT_CONFIG BIT(12) +#define A3700_SPI_WFIFO_OVERFLOW BIT(11) +#define A3700_SPI_WFIFO_UNDERFLOW BIT(10) +#define A3700_SPI_RFIFO_OVERFLOW BIT(9) +#define A3700_SPI_RFIFO_UNDERFLOW BIT(8) +#define A3700_SPI_WFIFO_FULL BIT(7) +#define A3700_SPI_WFIFO_EMPTY BIT(6) +#define A3700_SPI_RFIFO_FULL BIT(5) +#define A3700_SPI_RFIFO_EMPTY BIT(4) +#define A3700_SPI_WFIFO_RDY BIT(3) +#define A3700_SPI_RFIFO_RDY BIT(2) +#define A3700_SPI_XFER_RDY BIT(1) +#define A3700_SPI_XFER_DONE BIT(0) + +/* A3700_SPI_IF_CFG_REG */ +#define A3700_SPI_WFIFO_THRS BIT(28) +#define A3700_SPI_RFIFO_THRS BIT(24) +#define A3700_SPI_AUTO_CS BIT(20) +#define A3700_SPI_DMA_RD_EN BIT(18) +#define A3700_SPI_FIFO_MODE BIT(17) +#define A3700_SPI_SRST BIT(16) +#define A3700_SPI_XFER_START BIT(15) +#define A3700_SPI_XFER_STOP BIT(14) +#define A3700_SPI_INST_PIN BIT(13) +#define A3700_SPI_ADDR_PIN BIT(12) +#define A3700_SPI_DATA_PIN1 BIT(11) +#define A3700_SPI_DATA_PIN0 BIT(10) +#define A3700_SPI_FIFO_FLUSH BIT(9) +#define A3700_SPI_RW_EN BIT(8) +#define A3700_SPI_CLK_POL BIT(7) +#define A3700_SPI_CLK_PHA BIT(6) +#define A3700_SPI_BYTE_LEN BIT(5) +#define A3700_SPI_CLK_PRESCALE BIT(0) +#define A3700_SPI_CLK_PRESCALE_MASK (0x1f) + +#define A3700_SPI_WFIFO_THRS_BIT 28 +#define A3700_SPI_RFIFO_THRS_BIT 24 +#define A3700_SPI_FIFO_THRS_MASK 0x7 + +#define A3700_SPI_DATA_PIN_MASK 0x3 + +/* A3700_SPI_IF_HDR_CNT_REG */ +#define A3700_SPI_DUMMY_CNT_BIT 12 +#define A3700_SPI_DUMMY_CNT_MASK 0x7 +#define A3700_SPI_RMODE_CNT_BIT 8 +#define A3700_SPI_RMODE_CNT_MASK 0x3 +#define A3700_SPI_ADDR_CNT_BIT 4 +#define A3700_SPI_ADDR_CNT_MASK 0x7 +#define A3700_SPI_INSTR_CNT_BIT 0 +#define A3700_SPI_INSTR_CNT_MASK 0x3 + +/* A3700_SPI_IF_TIME_REG */ +#define A3700_SPI_CLK_CAPT_EDGE BIT(7) + +/* Flags and macros for struct a3700_spi */ +#define A3700_INSTR_CNT 1 +#define A3700_ADDR_CNT 3 +#define A3700_DUMMY_CNT 1 + +struct a3700_spi { + struct spi_master *master; + void __iomem *base; + struct clk *clk; + unsigned int irq; + unsigned int flags; + bool xmit_data; + const u8 *tx_buf; + u8 *rx_buf; + size_t buf_len; + u8 byte_len; + u32 wait_mask; + struct completion done; + u32 addr_cnt; + u32 instr_cnt; + size_t hdr_cnt; +}; + +static u32 spireg_read(struct a3700_spi *a3700_spi, u32 offset) +{ + return readl(a3700_spi->base + offset); +} + +static void spireg_write(struct a3700_spi *a3700_spi, u32 offset, u32 data) +{ + writel(data, a3700_spi->base + offset); +} + +static void a3700_spi_auto_cs_unset(struct a3700_spi *a3700_spi) +{ + u32 val; + + val = spireg_read(a3700_spi, A3700_SPI_IF_CFG_REG); + val &= ~A3700_SPI_AUTO_CS; + spireg_write(a3700_spi, A3700_SPI_IF_CFG_REG, val); +} + +static void a3700_spi_activate_cs(struct a3700_spi *a3700_spi, unsigned int cs) +{ + u32 val; + + val = spireg_read(a3700_spi, A3700_SPI_IF_CTRL_REG); + val |= (A3700_SPI_EN << cs); + spireg_write(a3700_spi, A3700_SPI_IF_CTRL_REG, val); +} + +static void a3700_spi_deactivate_cs(struct a3700_spi *a3700_spi, + unsigned int cs) +{ + u32 val; + + val = spireg_read(a3700_spi, A3700_SPI_IF_CTRL_REG); + val &= ~(A3700_SPI_EN << cs); + spireg_write(a3700_spi, A3700_SPI_IF_CTRL_REG, val); +} + +static int a3700_spi_pin_mode_set(struct a3700_spi *a3700_spi, + unsigned int pin_mode) +{ + u32 val; + + val = spireg_read(a3700_spi, A3700_SPI_IF_CFG_REG); + val &= ~(A3700_SPI_INST_PIN | A3700_SPI_ADDR_PIN); + val &= ~(A3700_SPI_DATA_PIN0 | A3700_SPI_DATA_PIN1); + + switch (pin_mode) { + case 1: + break; + case 2: + val |= A3700_SPI_DATA_PIN0; + break; + case 4: + val |= A3700_SPI_DATA_PIN1; + break; + default: + dev_err(&a3700_spi->master->dev, "wrong pin mode %u", pin_mode); + return -EINVAL; + } + + spireg_write(a3700_spi, A3700_SPI_IF_CFG_REG, val); + + return 0; +} + +static void a3700_spi_fifo_mode_set(struct a3700_spi *a3700_spi) +{ + u32 val; + + val = spireg_read(a3700_spi, A3700_SPI_IF_CFG_REG); + val |= A3700_SPI_FIFO_MODE; + spireg_write(a3700_spi, A3700_SPI_IF_CFG_REG, val); +} + +static void a3700_spi_mode_set(struct a3700_spi *a3700_spi, + unsigned int mode_bits) +{ + u32 val; + + val = spireg_read(a3700_spi, A3700_SPI_IF_CFG_REG); + + if (mode_bits & SPI_CPOL) + val |= A3700_SPI_CLK_POL; + else + val &= ~A3700_SPI_CLK_POL; + + if (mode_bits & SPI_CPHA) + val |= A3700_SPI_CLK_PHA; + else + val &= ~A3700_SPI_CLK_PHA; + + spireg_write(a3700_spi, A3700_SPI_IF_CFG_REG, val); +} + +static void a3700_spi_clock_set(struct a3700_spi *a3700_spi, + unsigned int speed_hz, u16 mode) +{ + u32 val; + u32 prescale; + + prescale = DIV_ROUND_UP(clk_get_rate(a3700_spi->clk), speed_hz); + + val = spireg_read(a3700_spi, A3700_SPI_IF_CFG_REG); + val = val & ~A3700_SPI_CLK_PRESCALE_MASK; + + val = val | (prescale & A3700_SPI_CLK_PRESCALE_MASK); + spireg_write(a3700_spi, A3700_SPI_IF_CFG_REG, val); + + if (prescale <= 2) { + val = spireg_read(a3700_spi, A3700_SPI_IF_TIME_REG); + val |= A3700_SPI_CLK_CAPT_EDGE; + spireg_write(a3700_spi, A3700_SPI_IF_TIME_REG, val); + } + + val = spireg_read(a3700_spi, A3700_SPI_IF_CFG_REG); + val &= ~(A3700_SPI_CLK_POL | A3700_SPI_CLK_PHA); + + if (mode & SPI_CPOL) + val |= A3700_SPI_CLK_POL; + + if (mode & SPI_CPHA) + val |= A3700_SPI_CLK_PHA; + + spireg_write(a3700_spi, A3700_SPI_IF_CFG_REG, val); +} + +static void a3700_spi_bytelen_set(struct a3700_spi *a3700_spi, unsigned int len) +{ + u32 val; + + val = spireg_read(a3700_spi, A3700_SPI_IF_CFG_REG); + if (len == 4) + val |= A3700_SPI_BYTE_LEN; + else + val &= ~A3700_SPI_BYTE_LEN; + spireg_write(a3700_spi, A3700_SPI_IF_CFG_REG, val); + + a3700_spi->byte_len = len; +} + +static int a3700_spi_fifo_flush(struct a3700_spi *a3700_spi) +{ + int timeout = A3700_SPI_TIMEOUT; + u32 val; + + val = spireg_read(a3700_spi, A3700_SPI_IF_CFG_REG); + val |= A3700_SPI_FIFO_FLUSH; + spireg_write(a3700_spi, A3700_SPI_IF_CFG_REG, val); + + while (--timeout) { + val = spireg_read(a3700_spi, A3700_SPI_IF_CFG_REG); + if (!(val & A3700_SPI_FIFO_FLUSH)) + return 0; + udelay(1); + } + + return -ETIMEDOUT; +} + +static int a3700_spi_init(struct a3700_spi *a3700_spi) +{ + struct spi_master *master = a3700_spi->master; + u32 val; + int i, ret = 0; + + /* Reset SPI unit */ + val = spireg_read(a3700_spi, A3700_SPI_IF_CFG_REG); + val |= A3700_SPI_SRST; + spireg_write(a3700_spi, A3700_SPI_IF_CFG_REG, val); + + udelay(A3700_SPI_TIMEOUT); + + val = spireg_read(a3700_spi, A3700_SPI_IF_CFG_REG); + val &= ~A3700_SPI_SRST; + spireg_write(a3700_spi, A3700_SPI_IF_CFG_REG, val); + + /* Disable AUTO_CS and deactivate all chip-selects */ + a3700_spi_auto_cs_unset(a3700_spi); + for (i = 0; i < master->num_chipselect; i++) + a3700_spi_deactivate_cs(a3700_spi, i); + + /* Enable FIFO mode */ + a3700_spi_fifo_mode_set(a3700_spi); + + /* Set SPI mode */ + a3700_spi_mode_set(a3700_spi, master->mode_bits); + + /* Reset counters */ + spireg_write(a3700_spi, A3700_SPI_IF_HDR_CNT_REG, 0); + spireg_write(a3700_spi, A3700_SPI_IF_DIN_CNT_REG, 0); + + /* Mask the interrupts and clear cause bits */ + spireg_write(a3700_spi, A3700_SPI_INT_MASK_REG, 0); + spireg_write(a3700_spi, A3700_SPI_INT_STAT_REG, ~0U); + + return ret; +} + +static irqreturn_t a3700_spi_interrupt(int irq, void *dev_id) +{ + struct spi_master *master = dev_id; + struct a3700_spi *a3700_spi; + u32 cause; + + a3700_spi = spi_master_get_devdata(master); + + /* Get interrupt causes */ + cause = spireg_read(a3700_spi, A3700_SPI_INT_STAT_REG); + + if (!cause || !(a3700_spi->wait_mask & cause)) + return IRQ_NONE; + + /* mask and acknowledge the SPI interrupts */ + spireg_write(a3700_spi, A3700_SPI_INT_MASK_REG, 0); + spireg_write(a3700_spi, A3700_SPI_INT_STAT_REG, cause); + + /* Wake up the transfer */ + if (a3700_spi->wait_mask & cause) + complete(&a3700_spi->done); + + return IRQ_HANDLED; +} + +static bool a3700_spi_wait_completion(struct spi_device *spi) +{ + struct a3700_spi *a3700_spi; + unsigned int timeout; + unsigned int ctrl_reg; + unsigned long timeout_jiffies; + + a3700_spi = spi_master_get_devdata(spi->master); + + /* SPI interrupt is edge-triggered, which means an interrupt will + * be generated only when detecting a specific status bit changed + * from '0' to '1'. So when we start waiting for a interrupt, we + * need to check status bit in control reg first, if it is already 1, + * then we do not need to wait for interrupt + */ + ctrl_reg = spireg_read(a3700_spi, A3700_SPI_IF_CTRL_REG); + if (a3700_spi->wait_mask & ctrl_reg) + return true; + + reinit_completion(&a3700_spi->done); + + spireg_write(a3700_spi, A3700_SPI_INT_MASK_REG, + a3700_spi->wait_mask); + + timeout_jiffies = msecs_to_jiffies(A3700_SPI_TIMEOUT); + timeout = wait_for_completion_timeout(&a3700_spi->done, + timeout_jiffies); + + a3700_spi->wait_mask = 0; + + if (timeout) + return true; + + /* there might be the case that right after we checked the + * status bits in this routine and before start to wait for + * interrupt by wait_for_completion_timeout, the interrupt + * happens, to avoid missing it we need to double check + * status bits in control reg, if it is already 1, then + * consider that we have the interrupt successfully and + * return true. + */ + ctrl_reg = spireg_read(a3700_spi, A3700_SPI_IF_CTRL_REG); + if (a3700_spi->wait_mask & ctrl_reg) + return true; + + spireg_write(a3700_spi, A3700_SPI_INT_MASK_REG, 0); + + return true; +} + +static bool a3700_spi_transfer_wait(struct spi_device *spi, + unsigned int bit_mask) +{ + struct a3700_spi *a3700_spi; + + a3700_spi = spi_master_get_devdata(spi->master); + a3700_spi->wait_mask = bit_mask; + + return a3700_spi_wait_completion(spi); +} + +static void a3700_spi_fifo_thres_set(struct a3700_spi *a3700_spi, + unsigned int bytes) +{ + u32 val; + + val = spireg_read(a3700_spi, A3700_SPI_IF_CFG_REG); + val &= ~(A3700_SPI_FIFO_THRS_MASK << A3700_SPI_RFIFO_THRS_BIT); + val |= (bytes - 1) << A3700_SPI_RFIFO_THRS_BIT; + val &= ~(A3700_SPI_FIFO_THRS_MASK << A3700_SPI_WFIFO_THRS_BIT); + val |= (7 - bytes) << A3700_SPI_WFIFO_THRS_BIT; + spireg_write(a3700_spi, A3700_SPI_IF_CFG_REG, val); +} + +static void a3700_spi_transfer_setup(struct spi_device *spi, + struct spi_transfer *xfer) +{ + struct a3700_spi *a3700_spi; + unsigned int byte_len; + + a3700_spi = spi_master_get_devdata(spi->master); + + a3700_spi_clock_set(a3700_spi, xfer->speed_hz, spi->mode); + + byte_len = xfer->bits_per_word >> 3; + + a3700_spi_fifo_thres_set(a3700_spi, byte_len); +} + +static void a3700_spi_set_cs(struct spi_device *spi, bool enable) +{ + struct a3700_spi *a3700_spi = spi_master_get_devdata(spi->master); + + if (!enable) + a3700_spi_activate_cs(a3700_spi, spi->chip_select); + else + a3700_spi_deactivate_cs(a3700_spi, spi->chip_select); +} + +static void a3700_spi_header_set(struct a3700_spi *a3700_spi) +{ + u32 instr_cnt = 0, addr_cnt = 0, dummy_cnt = 0; + u32 val = 0; + + /* Clear the header registers */ + spireg_write(a3700_spi, A3700_SPI_IF_INST_REG, 0); + spireg_write(a3700_spi, A3700_SPI_IF_ADDR_REG, 0); + spireg_write(a3700_spi, A3700_SPI_IF_RMODE_REG, 0); + + /* Set header counters */ + if (a3700_spi->tx_buf) { + if (a3700_spi->buf_len <= a3700_spi->instr_cnt) { + instr_cnt = a3700_spi->buf_len; + } else if (a3700_spi->buf_len <= (a3700_spi->instr_cnt + + a3700_spi->addr_cnt)) { + instr_cnt = a3700_spi->instr_cnt; + addr_cnt = a3700_spi->buf_len - instr_cnt; + } else if (a3700_spi->buf_len <= a3700_spi->hdr_cnt) { + instr_cnt = a3700_spi->instr_cnt; + addr_cnt = a3700_spi->addr_cnt; + /* Need to handle the normal write case with 1 byte + * data + */ + if (!a3700_spi->tx_buf[instr_cnt + addr_cnt]) + dummy_cnt = a3700_spi->buf_len - instr_cnt - + addr_cnt; + } + val |= ((instr_cnt & A3700_SPI_INSTR_CNT_MASK) + << A3700_SPI_INSTR_CNT_BIT); + val |= ((addr_cnt & A3700_SPI_ADDR_CNT_MASK) + << A3700_SPI_ADDR_CNT_BIT); + val |= ((dummy_cnt & A3700_SPI_DUMMY_CNT_MASK) + << A3700_SPI_DUMMY_CNT_BIT); + } + spireg_write(a3700_spi, A3700_SPI_IF_HDR_CNT_REG, val); + + /* Update the buffer length to be transferred */ + a3700_spi->buf_len -= (instr_cnt + addr_cnt + dummy_cnt); + + /* Set Instruction */ + val = 0; + while (instr_cnt--) { + val = (val << 8) | a3700_spi->tx_buf[0]; + a3700_spi->tx_buf++; + } + spireg_write(a3700_spi, A3700_SPI_IF_INST_REG, val); + + /* Set Address */ + val = 0; + while (addr_cnt--) { + val = (val << 8) | a3700_spi->tx_buf[0]; + a3700_spi->tx_buf++; + } + spireg_write(a3700_spi, A3700_SPI_IF_ADDR_REG, val); +} + +static int a3700_is_wfifo_full(struct a3700_spi *a3700_spi) +{ + u32 val; + + val = spireg_read(a3700_spi, A3700_SPI_IF_CTRL_REG); + return (val & A3700_SPI_WFIFO_FULL); +} + +static int a3700_spi_fifo_write(struct a3700_spi *a3700_spi) +{ + u32 val; + int i = 0; + + while (!a3700_is_wfifo_full(a3700_spi) && a3700_spi->buf_len) { + val = 0; + if (a3700_spi->buf_len >= 4) { + val = cpu_to_le32(*(u32 *)a3700_spi->tx_buf); + spireg_write(a3700_spi, A3700_SPI_DATA_OUT_REG, val); + + a3700_spi->buf_len -= 4; + a3700_spi->tx_buf += 4; + } else { + /* + * If the remained buffer length is less than 4-bytes, + * we should pad the write buffer with all ones. So that + * it avoids overwrite the unexpected bytes following + * the last one. + */ + val = GENMASK(31, 0); + while (a3700_spi->buf_len) { + val &= ~(0xff << (8 * i)); + val |= *a3700_spi->tx_buf++ << (8 * i); + i++; + a3700_spi->buf_len--; + + spireg_write(a3700_spi, A3700_SPI_DATA_OUT_REG, + val); + } + break; + } + } + + return 0; +} + +static int a3700_is_rfifo_empty(struct a3700_spi *a3700_spi) +{ + u32 val = spireg_read(a3700_spi, A3700_SPI_IF_CTRL_REG); + + return (val & A3700_SPI_RFIFO_EMPTY); +} + +static int a3700_spi_fifo_read(struct a3700_spi *a3700_spi) +{ + u32 val; + + while (!a3700_is_rfifo_empty(a3700_spi) && a3700_spi->buf_len) { + val = spireg_read(a3700_spi, A3700_SPI_DATA_IN_REG); + if (a3700_spi->buf_len >= 4) { + u32 data = le32_to_cpu(val); + memcpy(a3700_spi->rx_buf, &data, 4); + + a3700_spi->buf_len -= 4; + a3700_spi->rx_buf += 4; + } else { + /* + * When remain bytes is not larger than 4, we should + * avoid memory overwriting and just write the left rx + * buffer bytes. + */ + while (a3700_spi->buf_len) { + *a3700_spi->rx_buf = val & 0xff; + val >>= 8; + + a3700_spi->buf_len--; + a3700_spi->rx_buf++; + } + } + } + + return 0; +} + +static void a3700_spi_transfer_abort_fifo(struct a3700_spi *a3700_spi) +{ + int timeout = A3700_SPI_TIMEOUT; + u32 val; + + val = spireg_read(a3700_spi, A3700_SPI_IF_CFG_REG); + val |= A3700_SPI_XFER_STOP; + spireg_write(a3700_spi, A3700_SPI_IF_CFG_REG, val); + + while (--timeout) { + val = spireg_read(a3700_spi, A3700_SPI_IF_CFG_REG); + if (!(val & A3700_SPI_XFER_START)) + break; + udelay(1); + } + + a3700_spi_fifo_flush(a3700_spi); + + val &= ~A3700_SPI_XFER_STOP; + spireg_write(a3700_spi, A3700_SPI_IF_CFG_REG, val); +} + +static int a3700_spi_prepare_message(struct spi_master *master, + struct spi_message *message) +{ + struct a3700_spi *a3700_spi = spi_master_get_devdata(master); + struct spi_device *spi = message->spi; + int ret; + + ret = clk_enable(a3700_spi->clk); + if (ret) { + dev_err(&spi->dev, "failed to enable clk with error %d\n", ret); + return ret; + } + + /* Flush the FIFOs */ + ret = a3700_spi_fifo_flush(a3700_spi); + if (ret) + return ret; + + a3700_spi_bytelen_set(a3700_spi, 4); + + return 0; +} + +static int a3700_spi_transfer_one(struct spi_master *master, + struct spi_device *spi, + struct spi_transfer *xfer) +{ + struct a3700_spi *a3700_spi = spi_master_get_devdata(master); + int ret = 0, timeout = A3700_SPI_TIMEOUT; + unsigned int nbits = 0; + u32 val; + + a3700_spi_transfer_setup(spi, xfer); + + a3700_spi->tx_buf = xfer->tx_buf; + a3700_spi->rx_buf = xfer->rx_buf; + a3700_spi->buf_len = xfer->len; + + /* SPI transfer headers */ + a3700_spi_header_set(a3700_spi); + + if (xfer->tx_buf) + nbits = xfer->tx_nbits; + else if (xfer->rx_buf) + nbits = xfer->rx_nbits; + + a3700_spi_pin_mode_set(a3700_spi, nbits); + + if (xfer->rx_buf) { + /* Set read data length */ + spireg_write(a3700_spi, A3700_SPI_IF_DIN_CNT_REG, + a3700_spi->buf_len); + /* Start READ transfer */ + val = spireg_read(a3700_spi, A3700_SPI_IF_CFG_REG); + val &= ~A3700_SPI_RW_EN; + val |= A3700_SPI_XFER_START; + spireg_write(a3700_spi, A3700_SPI_IF_CFG_REG, val); + } else if (xfer->tx_buf) { + /* Start Write transfer */ + val = spireg_read(a3700_spi, A3700_SPI_IF_CFG_REG); + val |= (A3700_SPI_XFER_START | A3700_SPI_RW_EN); + spireg_write(a3700_spi, A3700_SPI_IF_CFG_REG, val); + + /* + * If there are data to be written to the SPI device, xmit_data + * flag is set true; otherwise the instruction in SPI_INSTR does + * not require data to be written to the SPI device, then + * xmit_data flag is set false. + */ + a3700_spi->xmit_data = (a3700_spi->buf_len != 0); + } + + while (a3700_spi->buf_len) { + if (a3700_spi->tx_buf) { + /* Wait wfifo ready */ + if (!a3700_spi_transfer_wait(spi, + A3700_SPI_WFIFO_RDY)) { + dev_err(&spi->dev, + "wait wfifo ready timed out\n"); + ret = -ETIMEDOUT; + goto error; + } + /* Fill up the wfifo */ + ret = a3700_spi_fifo_write(a3700_spi); + if (ret) + goto error; + } else if (a3700_spi->rx_buf) { + /* Wait rfifo ready */ + if (!a3700_spi_transfer_wait(spi, + A3700_SPI_RFIFO_RDY)) { + dev_err(&spi->dev, + "wait rfifo ready timed out\n"); + ret = -ETIMEDOUT; + goto error; + } + /* Drain out the rfifo */ + ret = a3700_spi_fifo_read(a3700_spi); + if (ret) + goto error; + } + } + + /* + * Stop a write transfer in fifo mode: + * - wait all the bytes in wfifo to be shifted out + * - set XFER_STOP bit + * - wait XFER_START bit clear + * - clear XFER_STOP bit + * Stop a read transfer in fifo mode: + * - the hardware is to reset the XFER_START bit + * after the number of bytes indicated in DIN_CNT + * register + * - just wait XFER_START bit clear + */ + if (a3700_spi->tx_buf) { + if (a3700_spi->xmit_data) { + /* + * If there are data written to the SPI device, wait + * until SPI_WFIFO_EMPTY is 1 to wait for all data to + * transfer out of write FIFO. + */ + if (!a3700_spi_transfer_wait(spi, + A3700_SPI_WFIFO_EMPTY)) { + dev_err(&spi->dev, "wait wfifo empty timed out\n"); + return -ETIMEDOUT; + } + } else { + /* + * If the instruction in SPI_INSTR does not require data + * to be written to the SPI device, wait until SPI_RDY + * is 1 for the SPI interface to be in idle. + */ + if (!a3700_spi_transfer_wait(spi, A3700_SPI_XFER_RDY)) { + dev_err(&spi->dev, "wait xfer ready timed out\n"); + return -ETIMEDOUT; + } + } + + val = spireg_read(a3700_spi, A3700_SPI_IF_CFG_REG); + val |= A3700_SPI_XFER_STOP; + spireg_write(a3700_spi, A3700_SPI_IF_CFG_REG, val); + } + + while (--timeout) { + val = spireg_read(a3700_spi, A3700_SPI_IF_CFG_REG); + if (!(val & A3700_SPI_XFER_START)) + break; + udelay(1); + } + + if (timeout == 0) { + dev_err(&spi->dev, "wait transfer start clear timed out\n"); + ret = -ETIMEDOUT; + goto error; + } + + val &= ~A3700_SPI_XFER_STOP; + spireg_write(a3700_spi, A3700_SPI_IF_CFG_REG, val); + goto out; + +error: + a3700_spi_transfer_abort_fifo(a3700_spi); +out: + spi_finalize_current_transfer(master); + + return ret; +} + +static int a3700_spi_unprepare_message(struct spi_master *master, + struct spi_message *message) +{ + struct a3700_spi *a3700_spi = spi_master_get_devdata(master); + + clk_disable(a3700_spi->clk); + + return 0; +} + +static const struct of_device_id a3700_spi_dt_ids[] = { + { .compatible = "marvell,armada-3700-spi", .data = NULL }, + {}, +}; + +MODULE_DEVICE_TABLE(of, a3700_spi_dt_ids); + +static int a3700_spi_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct device_node *of_node = dev->of_node; + struct resource *res; + struct spi_master *master; + struct a3700_spi *spi; + u32 num_cs = 0; + int ret = 0; + + master = spi_alloc_master(dev, sizeof(*spi)); + if (!master) { + dev_err(dev, "master allocation failed\n"); + ret = -ENOMEM; + goto out; + } + + if (of_property_read_u32(of_node, "num-cs", &num_cs)) { + dev_err(dev, "could not find num-cs\n"); + ret = -ENXIO; + goto error; + } + + master->bus_num = pdev->id; + master->dev.of_node = of_node; + master->mode_bits = SPI_MODE_3; + master->num_chipselect = num_cs; + master->bits_per_word_mask = SPI_BPW_MASK(8) | SPI_BPW_MASK(32); + master->prepare_message = a3700_spi_prepare_message; + master->transfer_one = a3700_spi_transfer_one; + master->unprepare_message = a3700_spi_unprepare_message; + master->set_cs = a3700_spi_set_cs; + master->flags = SPI_MASTER_HALF_DUPLEX; + master->mode_bits |= (SPI_RX_DUAL | SPI_RX_DUAL | + SPI_RX_QUAD | SPI_TX_QUAD); + + platform_set_drvdata(pdev, master); + + spi = spi_master_get_devdata(master); + memset(spi, 0, sizeof(struct a3700_spi)); + + spi->master = master; + spi->instr_cnt = A3700_INSTR_CNT; + spi->addr_cnt = A3700_ADDR_CNT; + spi->hdr_cnt = A3700_INSTR_CNT + A3700_ADDR_CNT + + A3700_DUMMY_CNT; + + res = platform_get_resource(pdev, IORESOURCE_MEM, 0); + spi->base = devm_ioremap_resource(dev, res); + if (IS_ERR(spi->base)) { + ret = PTR_ERR(spi->base); + goto error; + } + + spi->irq = platform_get_irq(pdev, 0); + if (spi->irq < 0) { + dev_err(dev, "could not get irq: %d\n", spi->irq); + ret = -ENXIO; + goto error; + } + + init_completion(&spi->done); + + spi->clk = devm_clk_get(dev, NULL); + if (IS_ERR(spi->clk)) { + dev_err(dev, "could not find clk: %ld\n", PTR_ERR(spi->clk)); + goto error; + } + + ret = clk_prepare(spi->clk); + if (ret) { + dev_err(dev, "could not prepare clk: %d\n", ret); + goto error; + } + + ret = a3700_spi_init(spi); + if (ret) + goto error_clk; + + ret = devm_request_irq(dev, spi->irq, a3700_spi_interrupt, 0, + dev_name(dev), master); + if (ret) { + dev_err(dev, "could not request IRQ: %d\n", ret); + goto error_clk; + } + + ret = devm_spi_register_master(dev, master); + if (ret) { + dev_err(dev, "Failed to register master\n"); + goto error_clk; + } + + return 0; + +error_clk: + clk_disable_unprepare(spi->clk); +error: + spi_master_put(master); +out: + return ret; +} + +static int a3700_spi_remove(struct platform_device *pdev) +{ + struct spi_master *master = platform_get_drvdata(pdev); + struct a3700_spi *spi = spi_master_get_devdata(master); + + clk_unprepare(spi->clk); + spi_master_put(master); + + return 0; +} + +static struct platform_driver a3700_spi_driver = { + .driver = { + .name = DRIVER_NAME, + .owner = THIS_MODULE, + .of_match_table = of_match_ptr(a3700_spi_dt_ids), + }, + .probe = a3700_spi_probe, + .remove = a3700_spi_remove, +}; + +module_platform_driver(a3700_spi_driver); + +MODULE_DESCRIPTION("Armada-3700 SPI driver"); +MODULE_AUTHOR("Wilson Ding <dingwei@marvell.com>"); +MODULE_LICENSE("GPL"); +MODULE_ALIAS("platform:" DRIVER_NAME); diff --git a/drivers/spi/spi-ath79.c b/drivers/spi/spi-ath79.c index 6165bf2..f369174 100644 --- a/drivers/spi/spi-ath79.c +++ b/drivers/spi/spi-ath79.c @@ -304,6 +304,7 @@ static const struct of_device_id ath79_spi_of_match[] = { { .compatible = "qca,ar7100-spi", }, { }, }; +MODULE_DEVICE_TABLE(of, ath79_spi_of_match); static struct platform_driver ath79_spi_driver = { .probe = ath79_spi_probe, diff --git a/drivers/spi/spi-atmel.c b/drivers/spi/spi-atmel.c index 8feac59..0e7712b 100644 --- a/drivers/spi/spi-atmel.c +++ b/drivers/spi/spi-atmel.c @@ -24,6 +24,7 @@ #include <linux/io.h> #include <linux/gpio.h> +#include <linux/of_gpio.h> #include <linux/pinctrl/consumer.h> #include <linux/pm_runtime.h> @@ -264,17 +265,6 @@ #define AUTOSUSPEND_TIMEOUT 2000 -struct atmel_spi_dma { - struct dma_chan *chan_rx; - struct dma_chan *chan_tx; - struct scatterlist sgrx; - struct scatterlist sgtx; - struct dma_async_tx_descriptor *data_desc_rx; - struct dma_async_tx_descriptor *data_desc_tx; - - struct at_dma_slave dma_slave; -}; - struct atmel_spi_caps { bool is_spi2; bool has_wdrbt; @@ -295,6 +285,7 @@ struct atmel_spi { int irq; struct clk *clk; struct platform_device *pdev; + unsigned long spi_clk; struct spi_transfer *current_transfer; int current_remaining_bytes; @@ -302,17 +293,11 @@ struct atmel_spi { struct completion xfer_completion; - /* scratch buffer */ - void *buffer; - dma_addr_t buffer_dma; - struct atmel_spi_caps caps; bool use_dma; bool use_pdc; bool use_cs_gpios; - /* dmaengine data */ - struct atmel_spi_dma dma; bool keep_cs; bool cs_active; @@ -326,7 +311,7 @@ struct atmel_spi_device { u32 csr; }; -#define BUFFER_SIZE PAGE_SIZE +#define SPI_MAX_DMA_XFER 65535 /* true for both PDC and DMA */ #define INVALID_DMA_ADDRESS 0xffffffff /* @@ -456,10 +441,20 @@ static inline bool atmel_spi_use_dma(struct atmel_spi *as, return as->use_dma && xfer->len >= DMA_MIN_BYTES; } +static bool atmel_spi_can_dma(struct spi_master *master, + struct spi_device *spi, + struct spi_transfer *xfer) +{ + struct atmel_spi *as = spi_master_get_devdata(master); + + return atmel_spi_use_dma(as, xfer); +} + static int atmel_spi_dma_slave_config(struct atmel_spi *as, struct dma_slave_config *slave_config, u8 bits_per_word) { + struct spi_master *master = platform_get_drvdata(as->pdev); int err = 0; if (bits_per_word > 8) { @@ -491,7 +486,7 @@ static int atmel_spi_dma_slave_config(struct atmel_spi *as, * path works the same whether FIFOs are available (and enabled) or not. */ slave_config->direction = DMA_MEM_TO_DEV; - if (dmaengine_slave_config(as->dma.chan_tx, slave_config)) { + if (dmaengine_slave_config(master->dma_tx, slave_config)) { dev_err(&as->pdev->dev, "failed to configure tx dma channel\n"); err = -EINVAL; @@ -506,7 +501,7 @@ static int atmel_spi_dma_slave_config(struct atmel_spi *as, * enabled) or not. */ slave_config->direction = DMA_DEV_TO_MEM; - if (dmaengine_slave_config(as->dma.chan_rx, slave_config)) { + if (dmaengine_slave_config(master->dma_rx, slave_config)) { dev_err(&as->pdev->dev, "failed to configure rx dma channel\n"); err = -EINVAL; @@ -515,7 +510,8 @@ static int atmel_spi_dma_slave_config(struct atmel_spi *as, return err; } -static int atmel_spi_configure_dma(struct atmel_spi *as) +static int atmel_spi_configure_dma(struct spi_master *master, + struct atmel_spi *as) { struct dma_slave_config slave_config; struct device *dev = &as->pdev->dev; @@ -525,26 +521,26 @@ static int atmel_spi_configure_dma(struct atmel_spi *as) dma_cap_zero(mask); dma_cap_set(DMA_SLAVE, mask); - as->dma.chan_tx = dma_request_slave_channel_reason(dev, "tx"); - if (IS_ERR(as->dma.chan_tx)) { - err = PTR_ERR(as->dma.chan_tx); + master->dma_tx = dma_request_slave_channel_reason(dev, "tx"); + if (IS_ERR(master->dma_tx)) { + err = PTR_ERR(master->dma_tx); if (err == -EPROBE_DEFER) { dev_warn(dev, "no DMA channel available at the moment\n"); - return err; + goto error_clear; } dev_err(dev, "DMA TX channel not available, SPI unable to use DMA\n"); err = -EBUSY; - goto error; + goto error_clear; } /* * No reason to check EPROBE_DEFER here since we have already requested * tx channel. If it fails here, it's for another reason. */ - as->dma.chan_rx = dma_request_slave_channel(dev, "rx"); + master->dma_rx = dma_request_slave_channel(dev, "rx"); - if (!as->dma.chan_rx) { + if (!master->dma_rx) { dev_err(dev, "DMA RX channel not available, SPI unable to use DMA\n"); err = -EBUSY; @@ -557,31 +553,38 @@ static int atmel_spi_configure_dma(struct atmel_spi *as) dev_info(&as->pdev->dev, "Using %s (tx) and %s (rx) for DMA transfers\n", - dma_chan_name(as->dma.chan_tx), - dma_chan_name(as->dma.chan_rx)); + dma_chan_name(master->dma_tx), + dma_chan_name(master->dma_rx)); + return 0; error: - if (as->dma.chan_rx) - dma_release_channel(as->dma.chan_rx); - if (!IS_ERR(as->dma.chan_tx)) - dma_release_channel(as->dma.chan_tx); + if (master->dma_rx) + dma_release_channel(master->dma_rx); + if (!IS_ERR(master->dma_tx)) + dma_release_channel(master->dma_tx); +error_clear: + master->dma_tx = master->dma_rx = NULL; return err; } -static void atmel_spi_stop_dma(struct atmel_spi *as) +static void atmel_spi_stop_dma(struct spi_master *master) { - if (as->dma.chan_rx) - dmaengine_terminate_all(as->dma.chan_rx); - if (as->dma.chan_tx) - dmaengine_terminate_all(as->dma.chan_tx); + if (master->dma_rx) + dmaengine_terminate_all(master->dma_rx); + if (master->dma_tx) + dmaengine_terminate_all(master->dma_tx); } -static void atmel_spi_release_dma(struct atmel_spi *as) +static void atmel_spi_release_dma(struct spi_master *master) { - if (as->dma.chan_rx) - dma_release_channel(as->dma.chan_rx); - if (as->dma.chan_tx) - dma_release_channel(as->dma.chan_tx); + if (master->dma_rx) { + dma_release_channel(master->dma_rx); + master->dma_rx = NULL; + } + if (master->dma_tx) { + dma_release_channel(master->dma_tx); + master->dma_tx = NULL; + } } /* This function is called by the DMA driver from tasklet context */ @@ -611,14 +614,10 @@ static void atmel_spi_next_xfer_single(struct spi_master *master, cpu_relax(); } - if (xfer->tx_buf) { - if (xfer->bits_per_word > 8) - spi_writel(as, TDR, *(u16 *)(xfer->tx_buf + xfer_pos)); - else - spi_writel(as, TDR, *(u8 *)(xfer->tx_buf + xfer_pos)); - } else { - spi_writel(as, TDR, 0); - } + if (xfer->bits_per_word > 8) + spi_writel(as, TDR, *(u16 *)(xfer->tx_buf + xfer_pos)); + else + spi_writel(as, TDR, *(u8 *)(xfer->tx_buf + xfer_pos)); dev_dbg(master->dev.parent, " start pio xfer %p: len %u tx %p rx %p bitpw %d\n", @@ -665,17 +664,12 @@ static void atmel_spi_next_xfer_fifo(struct spi_master *master, /* Fill TX FIFO */ while (num_data >= 2) { - if (xfer->tx_buf) { - if (xfer->bits_per_word > 8) { - td0 = *words++; - td1 = *words++; - } else { - td0 = *bytes++; - td1 = *bytes++; - } + if (xfer->bits_per_word > 8) { + td0 = *words++; + td1 = *words++; } else { - td0 = 0; - td1 = 0; + td0 = *bytes++; + td1 = *bytes++; } spi_writel(as, TDR, (td1 << 16) | td0); @@ -683,14 +677,10 @@ static void atmel_spi_next_xfer_fifo(struct spi_master *master, } if (num_data) { - if (xfer->tx_buf) { - if (xfer->bits_per_word > 8) - td0 = *words++; - else - td0 = *bytes++; - } else { - td0 = 0; - } + if (xfer->bits_per_word > 8) + td0 = *words++; + else + td0 = *bytes++; spi_writew(as, TDR, td0); num_data--; @@ -730,13 +720,12 @@ static int atmel_spi_next_xfer_dma_submit(struct spi_master *master, u32 *plen) { struct atmel_spi *as = spi_master_get_devdata(master); - struct dma_chan *rxchan = as->dma.chan_rx; - struct dma_chan *txchan = as->dma.chan_tx; + struct dma_chan *rxchan = master->dma_rx; + struct dma_chan *txchan = master->dma_tx; struct dma_async_tx_descriptor *rxdesc; struct dma_async_tx_descriptor *txdesc; struct dma_slave_config slave_config; dma_cookie_t cookie; - u32 len = *plen; dev_vdbg(master->dev.parent, "atmel_spi_next_xfer_dma_submit\n"); @@ -747,44 +736,22 @@ static int atmel_spi_next_xfer_dma_submit(struct spi_master *master, /* release lock for DMA operations */ atmel_spi_unlock(as); - /* prepare the RX dma transfer */ - sg_init_table(&as->dma.sgrx, 1); - if (xfer->rx_buf) { - as->dma.sgrx.dma_address = xfer->rx_dma + xfer->len - *plen; - } else { - as->dma.sgrx.dma_address = as->buffer_dma; - if (len > BUFFER_SIZE) - len = BUFFER_SIZE; - } - - /* prepare the TX dma transfer */ - sg_init_table(&as->dma.sgtx, 1); - if (xfer->tx_buf) { - as->dma.sgtx.dma_address = xfer->tx_dma + xfer->len - *plen; - } else { - as->dma.sgtx.dma_address = as->buffer_dma; - if (len > BUFFER_SIZE) - len = BUFFER_SIZE; - memset(as->buffer, 0, len); - } - - sg_dma_len(&as->dma.sgtx) = len; - sg_dma_len(&as->dma.sgrx) = len; - - *plen = len; + *plen = xfer->len; if (atmel_spi_dma_slave_config(as, &slave_config, xfer->bits_per_word)) goto err_exit; /* Send both scatterlists */ - rxdesc = dmaengine_prep_slave_sg(rxchan, &as->dma.sgrx, 1, + rxdesc = dmaengine_prep_slave_sg(rxchan, + xfer->rx_sg.sgl, xfer->rx_sg.nents, DMA_FROM_DEVICE, DMA_PREP_INTERRUPT | DMA_CTRL_ACK); if (!rxdesc) goto err_dma; - txdesc = dmaengine_prep_slave_sg(txchan, &as->dma.sgtx, 1, + txdesc = dmaengine_prep_slave_sg(txchan, + xfer->tx_sg.sgl, xfer->tx_sg.nents, DMA_TO_DEVICE, DMA_PREP_INTERRUPT | DMA_CTRL_ACK); if (!txdesc) @@ -818,7 +785,7 @@ static int atmel_spi_next_xfer_dma_submit(struct spi_master *master, err_dma: spi_writel(as, IDR, SPI_BIT(OVRES)); - atmel_spi_stop_dma(as); + atmel_spi_stop_dma(master); err_exit: atmel_spi_lock(as); return -ENOMEM; @@ -830,30 +797,10 @@ static void atmel_spi_next_xfer_data(struct spi_master *master, dma_addr_t *rx_dma, u32 *plen) { - struct atmel_spi *as = spi_master_get_devdata(master); - u32 len = *plen; - - /* use scratch buffer only when rx or tx data is unspecified */ - if (xfer->rx_buf) - *rx_dma = xfer->rx_dma + xfer->len - *plen; - else { - *rx_dma = as->buffer_dma; - if (len > BUFFER_SIZE) - len = BUFFER_SIZE; - } - - if (xfer->tx_buf) - *tx_dma = xfer->tx_dma + xfer->len - *plen; - else { - *tx_dma = as->buffer_dma; - if (len > BUFFER_SIZE) - len = BUFFER_SIZE; - memset(as->buffer, 0, len); - dma_sync_single_for_device(&as->pdev->dev, - as->buffer_dma, len, DMA_TO_DEVICE); - } - - *plen = len; + *rx_dma = xfer->rx_dma + xfer->len - *plen; + *tx_dma = xfer->tx_dma + xfer->len - *plen; + if (*plen > master->max_dma_len) + *plen = master->max_dma_len; } static int atmel_spi_set_xfer_speed(struct atmel_spi *as, @@ -864,7 +811,7 @@ static int atmel_spi_set_xfer_speed(struct atmel_spi *as, unsigned long bus_hz; /* v1 chips start out at half the peripheral bus speed. */ - bus_hz = clk_get_rate(as->clk); + bus_hz = as->spi_clk; if (!atmel_spi_is_v2(as)) bus_hz /= 2; @@ -1025,16 +972,12 @@ atmel_spi_pump_single_data(struct atmel_spi *as, struct spi_transfer *xfer) u16 *rxp16; unsigned long xfer_pos = xfer->len - as->current_remaining_bytes; - if (xfer->rx_buf) { - if (xfer->bits_per_word > 8) { - rxp16 = (u16 *)(((u8 *)xfer->rx_buf) + xfer_pos); - *rxp16 = spi_readl(as, RDR); - } else { - rxp = ((u8 *)xfer->rx_buf) + xfer_pos; - *rxp = spi_readl(as, RDR); - } + if (xfer->bits_per_word > 8) { + rxp16 = (u16 *)(((u8 *)xfer->rx_buf) + xfer_pos); + *rxp16 = spi_readl(as, RDR); } else { - spi_readl(as, RDR); + rxp = ((u8 *)xfer->rx_buf) + xfer_pos; + *rxp = spi_readl(as, RDR); } if (xfer->bits_per_word > 8) { if (as->current_remaining_bytes > 2) @@ -1073,12 +1016,10 @@ atmel_spi_pump_fifo_data(struct atmel_spi *as, struct spi_transfer *xfer) /* Read data */ while (num_data) { rd = spi_readl(as, RDR); - if (xfer->rx_buf) { - if (xfer->bits_per_word > 8) - *words++ = rd; - else - *bytes++ = rd; - } + if (xfer->bits_per_word > 8) + *words++ = rd; + else + *bytes++ = rd; num_data--; } } @@ -1204,7 +1145,6 @@ static int atmel_spi_setup(struct spi_device *spi) u32 csr; unsigned int bits = spi->bits_per_word; unsigned int npcs_pin; - int ret; as = spi_master_get_devdata(spi->master); @@ -1247,16 +1187,9 @@ static int atmel_spi_setup(struct spi_device *spi) if (!asd) return -ENOMEM; - if (as->use_cs_gpios) { - ret = gpio_request(npcs_pin, dev_name(&spi->dev)); - if (ret) { - kfree(asd); - return ret; - } - + if (as->use_cs_gpios) gpio_direction_output(npcs_pin, !(spi->mode & SPI_CS_HIGH)); - } asd->npcs_pin = npcs_pin; spi->controller_state = asd; @@ -1307,7 +1240,7 @@ static int atmel_spi_one_transfer(struct spi_master *master, * better fault reporting. */ if ((!msg->is_dma_mapped) - && (atmel_spi_use_dma(as, xfer) || as->use_pdc)) { + && as->use_pdc) { if (atmel_spi_dma_map_xfer(as, xfer) < 0) return -ENOMEM; } @@ -1380,11 +1313,11 @@ static int atmel_spi_one_transfer(struct spi_master *master, spi_readl(as, SR); } else if (atmel_spi_use_dma(as, xfer)) { - atmel_spi_stop_dma(as); + atmel_spi_stop_dma(master); } if (!msg->is_dma_mapped - && (atmel_spi_use_dma(as, xfer) || as->use_pdc)) + && as->use_pdc) atmel_spi_dma_unmap_xfer(master, xfer); return 0; @@ -1395,7 +1328,7 @@ static int atmel_spi_one_transfer(struct spi_master *master, } if (!msg->is_dma_mapped - && (atmel_spi_use_dma(as, xfer) || as->use_pdc)) + && as->use_pdc) atmel_spi_dma_unmap_xfer(master, xfer); if (xfer->delay_usecs) @@ -1471,13 +1404,11 @@ msg_done: static void atmel_spi_cleanup(struct spi_device *spi) { struct atmel_spi_device *asd = spi->controller_state; - unsigned gpio = (unsigned long) spi->controller_data; if (!asd) return; spi->controller_state = NULL; - gpio_free(gpio); kfree(asd); } @@ -1499,6 +1430,39 @@ static void atmel_get_caps(struct atmel_spi *as) } /*-------------------------------------------------------------------------*/ +static int atmel_spi_gpio_cs(struct platform_device *pdev) +{ + struct spi_master *master = platform_get_drvdata(pdev); + struct atmel_spi *as = spi_master_get_devdata(master); + struct device_node *np = master->dev.of_node; + int i; + int ret = 0; + int nb = 0; + + if (!as->use_cs_gpios) + return 0; + + if (!np) + return 0; + + nb = of_gpio_named_count(np, "cs-gpios"); + for (i = 0; i < nb; i++) { + int cs_gpio = of_get_named_gpio(pdev->dev.of_node, + "cs-gpios", i); + + if (cs_gpio == -EPROBE_DEFER) + return cs_gpio; + + if (gpio_is_valid(cs_gpio)) { + ret = devm_gpio_request(&pdev->dev, cs_gpio, + dev_name(&pdev->dev)); + if (ret) + return ret; + } + } + + return 0; +} static int atmel_spi_probe(struct platform_device *pdev) { @@ -1537,29 +1501,23 @@ static int atmel_spi_probe(struct platform_device *pdev) master->bus_num = pdev->id; master->num_chipselect = master->dev.of_node ? 0 : 4; master->setup = atmel_spi_setup; + master->flags = (SPI_MASTER_MUST_RX | SPI_MASTER_MUST_TX); master->transfer_one_message = atmel_spi_transfer_one_message; master->cleanup = atmel_spi_cleanup; master->auto_runtime_pm = true; + master->max_dma_len = SPI_MAX_DMA_XFER; + master->can_dma = atmel_spi_can_dma; platform_set_drvdata(pdev, master); as = spi_master_get_devdata(master); - /* - * Scratch buffer is used for throwaway rx and tx data. - * It's coherent to minimize dcache pollution. - */ - as->buffer = dma_alloc_coherent(&pdev->dev, BUFFER_SIZE, - &as->buffer_dma, GFP_KERNEL); - if (!as->buffer) - goto out_free; - spin_lock_init(&as->lock); as->pdev = pdev; as->regs = devm_ioremap_resource(&pdev->dev, regs); if (IS_ERR(as->regs)) { ret = PTR_ERR(as->regs); - goto out_free_buffer; + goto out_unmap_regs; } as->phybase = regs->start; as->irq = irq; @@ -1577,14 +1535,19 @@ static int atmel_spi_probe(struct platform_device *pdev) master->num_chipselect = 4; } + ret = atmel_spi_gpio_cs(pdev); + if (ret) + goto out_unmap_regs; + as->use_dma = false; as->use_pdc = false; if (as->caps.has_dma_support) { - ret = atmel_spi_configure_dma(as); - if (ret == 0) + ret = atmel_spi_configure_dma(master, as); + if (ret == 0) { as->use_dma = true; - else if (ret == -EPROBE_DEFER) + } else if (ret == -EPROBE_DEFER) { return ret; + } } else { as->use_pdc = true; } @@ -1606,6 +1569,9 @@ static int atmel_spi_probe(struct platform_device *pdev) ret = clk_prepare_enable(clk); if (ret) goto out_free_irq; + + as->spi_clk = clk_get_rate(clk); + spi_writel(as, CR, SPI_BIT(SWRST)); spi_writel(as, CR, SPI_BIT(SWRST)); /* AT91SAM9263 Rev B workaround */ if (as->caps.has_wdrbt) { @@ -1626,10 +1592,6 @@ static int atmel_spi_probe(struct platform_device *pdev) spi_writel(as, CR, SPI_BIT(FIFOEN)); } - /* go! */ - dev_info(&pdev->dev, "Atmel SPI Controller at 0x%08lx (irq %d)\n", - (unsigned long)regs->start, irq); - pm_runtime_set_autosuspend_delay(&pdev->dev, AUTOSUSPEND_TIMEOUT); pm_runtime_use_autosuspend(&pdev->dev); pm_runtime_set_active(&pdev->dev); @@ -1639,6 +1601,10 @@ static int atmel_spi_probe(struct platform_device *pdev) if (ret) goto out_free_dma; + /* go! */ + dev_info(&pdev->dev, "Atmel SPI Controller at 0x%08lx (irq %d)\n", + (unsigned long)regs->start, irq); + return 0; out_free_dma: @@ -1646,16 +1612,13 @@ out_free_dma: pm_runtime_set_suspended(&pdev->dev); if (as->use_dma) - atmel_spi_release_dma(as); + atmel_spi_release_dma(master); spi_writel(as, CR, SPI_BIT(SWRST)); spi_writel(as, CR, SPI_BIT(SWRST)); /* AT91SAM9263 Rev B workaround */ clk_disable_unprepare(clk); out_free_irq: out_unmap_regs: -out_free_buffer: - dma_free_coherent(&pdev->dev, BUFFER_SIZE, as->buffer, - as->buffer_dma); out_free: spi_master_put(master); return ret; @@ -1671,8 +1634,8 @@ static int atmel_spi_remove(struct platform_device *pdev) /* reset the hardware and block queue progress */ spin_lock_irq(&as->lock); if (as->use_dma) { - atmel_spi_stop_dma(as); - atmel_spi_release_dma(as); + atmel_spi_stop_dma(master); + atmel_spi_release_dma(master); } spi_writel(as, CR, SPI_BIT(SWRST)); @@ -1680,9 +1643,6 @@ static int atmel_spi_remove(struct platform_device *pdev) spi_readl(as, SR); spin_unlock_irq(&as->lock); - dma_free_coherent(&pdev->dev, BUFFER_SIZE, as->buffer, - as->buffer_dma); - clk_disable_unprepare(as->clk); pm_runtime_put_noidle(&pdev->dev); diff --git a/drivers/spi/spi-axi-spi-engine.c b/drivers/spi/spi-axi-spi-engine.c index 2b1456e..319225d 100644 --- a/drivers/spi/spi-axi-spi-engine.c +++ b/drivers/spi/spi-axi-spi-engine.c @@ -574,6 +574,7 @@ static const struct of_device_id spi_engine_match_table[] = { { .compatible = "adi,axi-spi-engine-1.00.a" }, { }, }; +MODULE_DEVICE_TABLE(of, spi_engine_match_table); static struct platform_driver spi_engine_driver = { .probe = spi_engine_probe, diff --git a/drivers/spi/spi-dw.c b/drivers/spi/spi-dw.c index 27960e4..b715a26 100644 --- a/drivers/spi/spi-dw.c +++ b/drivers/spi/spi-dw.c @@ -502,6 +502,7 @@ int dw_spi_add_host(struct device *dev, struct dw_spi *dws) master->handle_err = dw_spi_handle_err; master->max_speed_hz = dws->max_freq; master->dev.of_node = dev->of_node; + master->flags = SPI_MASTER_GPIO_SS; /* Basic HW init */ spi_hw_init(dev, dws); diff --git a/drivers/spi/spi-fsl-dspi.c b/drivers/spi/spi-fsl-dspi.c index a67b0ff..14c8e7c 100644 --- a/drivers/spi/spi-fsl-dspi.c +++ b/drivers/spi/spi-fsl-dspi.c @@ -15,6 +15,8 @@ #include <linux/clk.h> #include <linux/delay.h> +#include <linux/dmaengine.h> +#include <linux/dma-mapping.h> #include <linux/err.h> #include <linux/errno.h> #include <linux/interrupt.h> @@ -40,6 +42,7 @@ #define TRAN_STATE_WORD_ODD_NUM 0x04 #define DSPI_FIFO_SIZE 4 +#define DSPI_DMA_BUFSIZE (DSPI_FIFO_SIZE * 1024) #define SPI_MCR 0x00 #define SPI_MCR_MASTER (1 << 31) @@ -72,6 +75,11 @@ #define SPI_SR_TCFQF 0x80000000 #define SPI_SR_CLEAR 0xdaad0000 +#define SPI_RSER_TFFFE BIT(25) +#define SPI_RSER_TFFFD BIT(24) +#define SPI_RSER_RFDFE BIT(17) +#define SPI_RSER_RFDFD BIT(16) + #define SPI_RSER 0x30 #define SPI_RSER_EOQFE 0x10000000 #define SPI_RSER_TCFQE 0x80000000 @@ -109,6 +117,8 @@ #define SPI_TCR_TCNT_MAX 0x10000 +#define DMA_COMPLETION_TIMEOUT msecs_to_jiffies(3000) + struct chip_data { u32 mcr_val; u32 ctar_val; @@ -118,6 +128,7 @@ struct chip_data { enum dspi_trans_mode { DSPI_EOQ_MODE = 0, DSPI_TCFQ_MODE, + DSPI_DMA_MODE, }; struct fsl_dspi_devtype_data { @@ -126,7 +137,7 @@ struct fsl_dspi_devtype_data { }; static const struct fsl_dspi_devtype_data vf610_data = { - .trans_mode = DSPI_EOQ_MODE, + .trans_mode = DSPI_DMA_MODE, .max_clock_factor = 2, }; @@ -140,6 +151,23 @@ static const struct fsl_dspi_devtype_data ls2085a_data = { .max_clock_factor = 8, }; +struct fsl_dspi_dma { + /* Length of transfer in words of DSPI_FIFO_SIZE */ + u32 curr_xfer_len; + + u32 *tx_dma_buf; + struct dma_chan *chan_tx; + dma_addr_t tx_dma_phys; + struct completion cmd_tx_complete; + struct dma_async_tx_descriptor *tx_desc; + + u32 *rx_dma_buf; + struct dma_chan *chan_rx; + dma_addr_t rx_dma_phys; + struct completion cmd_rx_complete; + struct dma_async_tx_descriptor *rx_desc; +}; + struct fsl_dspi { struct spi_master *master; struct platform_device *pdev; @@ -166,8 +194,11 @@ struct fsl_dspi { u32 waitflags; u32 spi_tcnt; + struct fsl_dspi_dma *dma; }; +static u32 dspi_data_to_pushr(struct fsl_dspi *dspi, int tx_word); + static inline int is_double_byte_mode(struct fsl_dspi *dspi) { unsigned int val; @@ -177,6 +208,255 @@ static inline int is_double_byte_mode(struct fsl_dspi *dspi) return ((val & SPI_FRAME_BITS_MASK) == SPI_FRAME_BITS(8)) ? 0 : 1; } +static void dspi_tx_dma_callback(void *arg) +{ + struct fsl_dspi *dspi = arg; + struct fsl_dspi_dma *dma = dspi->dma; + + complete(&dma->cmd_tx_complete); +} + +static void dspi_rx_dma_callback(void *arg) +{ + struct fsl_dspi *dspi = arg; + struct fsl_dspi_dma *dma = dspi->dma; + int rx_word; + int i; + u16 d; + + rx_word = is_double_byte_mode(dspi); + + if (!(dspi->dataflags & TRAN_STATE_RX_VOID)) { + for (i = 0; i < dma->curr_xfer_len; i++) { + d = dspi->dma->rx_dma_buf[i]; + rx_word ? (*(u16 *)dspi->rx = d) : + (*(u8 *)dspi->rx = d); + dspi->rx += rx_word + 1; + } + } + + complete(&dma->cmd_rx_complete); +} + +static int dspi_next_xfer_dma_submit(struct fsl_dspi *dspi) +{ + struct fsl_dspi_dma *dma = dspi->dma; + struct device *dev = &dspi->pdev->dev; + int time_left; + int tx_word; + int i; + + tx_word = is_double_byte_mode(dspi); + + for (i = 0; i < dma->curr_xfer_len; i++) { + dspi->dma->tx_dma_buf[i] = dspi_data_to_pushr(dspi, tx_word); + if ((dspi->cs_change) && (!dspi->len)) + dspi->dma->tx_dma_buf[i] &= ~SPI_PUSHR_CONT; + } + + dma->tx_desc = dmaengine_prep_slave_single(dma->chan_tx, + dma->tx_dma_phys, + dma->curr_xfer_len * + DMA_SLAVE_BUSWIDTH_4_BYTES, + DMA_MEM_TO_DEV, + DMA_PREP_INTERRUPT | DMA_CTRL_ACK); + if (!dma->tx_desc) { + dev_err(dev, "Not able to get desc for DMA xfer\n"); + return -EIO; + } + + dma->tx_desc->callback = dspi_tx_dma_callback; + dma->tx_desc->callback_param = dspi; + if (dma_submit_error(dmaengine_submit(dma->tx_desc))) { + dev_err(dev, "DMA submit failed\n"); + return -EINVAL; + } + + dma->rx_desc = dmaengine_prep_slave_single(dma->chan_rx, + dma->rx_dma_phys, + dma->curr_xfer_len * + DMA_SLAVE_BUSWIDTH_4_BYTES, + DMA_DEV_TO_MEM, + DMA_PREP_INTERRUPT | DMA_CTRL_ACK); + if (!dma->rx_desc) { + dev_err(dev, "Not able to get desc for DMA xfer\n"); + return -EIO; + } + + dma->rx_desc->callback = dspi_rx_dma_callback; + dma->rx_desc->callback_param = dspi; + if (dma_submit_error(dmaengine_submit(dma->rx_desc))) { + dev_err(dev, "DMA submit failed\n"); + return -EINVAL; + } + + reinit_completion(&dspi->dma->cmd_rx_complete); + reinit_completion(&dspi->dma->cmd_tx_complete); + + dma_async_issue_pending(dma->chan_rx); + dma_async_issue_pending(dma->chan_tx); + + time_left = wait_for_completion_timeout(&dspi->dma->cmd_tx_complete, + DMA_COMPLETION_TIMEOUT); + if (time_left == 0) { + dev_err(dev, "DMA tx timeout\n"); + dmaengine_terminate_all(dma->chan_tx); + dmaengine_terminate_all(dma->chan_rx); + return -ETIMEDOUT; + } + + time_left = wait_for_completion_timeout(&dspi->dma->cmd_rx_complete, + DMA_COMPLETION_TIMEOUT); + if (time_left == 0) { + dev_err(dev, "DMA rx timeout\n"); + dmaengine_terminate_all(dma->chan_tx); + dmaengine_terminate_all(dma->chan_rx); + return -ETIMEDOUT; + } + + return 0; +} + +static int dspi_dma_xfer(struct fsl_dspi *dspi) +{ + struct fsl_dspi_dma *dma = dspi->dma; + struct device *dev = &dspi->pdev->dev; + int curr_remaining_bytes; + int bytes_per_buffer; + int word = 1; + int ret = 0; + + if (is_double_byte_mode(dspi)) + word = 2; + curr_remaining_bytes = dspi->len; + bytes_per_buffer = DSPI_DMA_BUFSIZE / DSPI_FIFO_SIZE; + while (curr_remaining_bytes) { + /* Check if current transfer fits the DMA buffer */ + dma->curr_xfer_len = curr_remaining_bytes / word; + if (dma->curr_xfer_len > bytes_per_buffer) + dma->curr_xfer_len = bytes_per_buffer; + + ret = dspi_next_xfer_dma_submit(dspi); + if (ret) { + dev_err(dev, "DMA transfer failed\n"); + goto exit; + + } else { + curr_remaining_bytes -= dma->curr_xfer_len * word; + if (curr_remaining_bytes < 0) + curr_remaining_bytes = 0; + } + } + +exit: + return ret; +} + +static int dspi_request_dma(struct fsl_dspi *dspi, phys_addr_t phy_addr) +{ + struct fsl_dspi_dma *dma; + struct dma_slave_config cfg; + struct device *dev = &dspi->pdev->dev; + int ret; + + dma = devm_kzalloc(dev, sizeof(*dma), GFP_KERNEL); + if (!dma) + return -ENOMEM; + + dma->chan_rx = dma_request_slave_channel(dev, "rx"); + if (!dma->chan_rx) { + dev_err(dev, "rx dma channel not available\n"); + ret = -ENODEV; + return ret; + } + + dma->chan_tx = dma_request_slave_channel(dev, "tx"); + if (!dma->chan_tx) { + dev_err(dev, "tx dma channel not available\n"); + ret = -ENODEV; + goto err_tx_channel; + } + + dma->tx_dma_buf = dma_alloc_coherent(dev, DSPI_DMA_BUFSIZE, + &dma->tx_dma_phys, GFP_KERNEL); + if (!dma->tx_dma_buf) { + ret = -ENOMEM; + goto err_tx_dma_buf; + } + + dma->rx_dma_buf = dma_alloc_coherent(dev, DSPI_DMA_BUFSIZE, + &dma->rx_dma_phys, GFP_KERNEL); + if (!dma->rx_dma_buf) { + ret = -ENOMEM; + goto err_rx_dma_buf; + } + + cfg.src_addr = phy_addr + SPI_POPR; + cfg.dst_addr = phy_addr + SPI_PUSHR; + cfg.src_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES; + cfg.dst_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES; + cfg.src_maxburst = 1; + cfg.dst_maxburst = 1; + + cfg.direction = DMA_DEV_TO_MEM; + ret = dmaengine_slave_config(dma->chan_rx, &cfg); + if (ret) { + dev_err(dev, "can't configure rx dma channel\n"); + ret = -EINVAL; + goto err_slave_config; + } + + cfg.direction = DMA_MEM_TO_DEV; + ret = dmaengine_slave_config(dma->chan_tx, &cfg); + if (ret) { + dev_err(dev, "can't configure tx dma channel\n"); + ret = -EINVAL; + goto err_slave_config; + } + + dspi->dma = dma; + init_completion(&dma->cmd_tx_complete); + init_completion(&dma->cmd_rx_complete); + + return 0; + +err_slave_config: + dma_free_coherent(dev, DSPI_DMA_BUFSIZE, + dma->rx_dma_buf, dma->rx_dma_phys); +err_rx_dma_buf: + dma_free_coherent(dev, DSPI_DMA_BUFSIZE, + dma->tx_dma_buf, dma->tx_dma_phys); +err_tx_dma_buf: + dma_release_channel(dma->chan_tx); +err_tx_channel: + dma_release_channel(dma->chan_rx); + + devm_kfree(dev, dma); + dspi->dma = NULL; + + return ret; +} + +static void dspi_release_dma(struct fsl_dspi *dspi) +{ + struct fsl_dspi_dma *dma = dspi->dma; + struct device *dev = &dspi->pdev->dev; + + if (dma) { + if (dma->chan_tx) { + dma_unmap_single(dev, dma->tx_dma_phys, + DSPI_DMA_BUFSIZE, DMA_TO_DEVICE); + dma_release_channel(dma->chan_tx); + } + + if (dma->chan_rx) { + dma_unmap_single(dev, dma->rx_dma_phys, + DSPI_DMA_BUFSIZE, DMA_FROM_DEVICE); + dma_release_channel(dma->chan_rx); + } + } +} + static void hz_to_spi_baud(char *pbr, char *br, int speed_hz, unsigned long clkrate) { @@ -425,6 +705,12 @@ static int dspi_transfer_one_message(struct spi_master *master, regmap_write(dspi->regmap, SPI_RSER, SPI_RSER_TCFQE); dspi_tcfq_write(dspi); break; + case DSPI_DMA_MODE: + regmap_write(dspi->regmap, SPI_RSER, + SPI_RSER_TFFFE | SPI_RSER_TFFFD | + SPI_RSER_RFDFE | SPI_RSER_RFDFD); + status = dspi_dma_xfer(dspi); + break; default: dev_err(&dspi->pdev->dev, "unsupported trans_mode %u\n", trans_mode); @@ -432,9 +718,13 @@ static int dspi_transfer_one_message(struct spi_master *master, goto out; } - if (wait_event_interruptible(dspi->waitq, dspi->waitflags)) - dev_err(&dspi->pdev->dev, "wait transfer complete fail!\n"); - dspi->waitflags = 0; + if (trans_mode != DSPI_DMA_MODE) { + if (wait_event_interruptible(dspi->waitq, + dspi->waitflags)) + dev_err(&dspi->pdev->dev, + "wait transfer complete fail!\n"); + dspi->waitflags = 0; + } if (transfer->delay_usecs) udelay(transfer->delay_usecs); @@ -740,6 +1030,13 @@ static int dspi_probe(struct platform_device *pdev) if (ret) goto out_master_put; + if (dspi->devtype_data->trans_mode == DSPI_DMA_MODE) { + if (dspi_request_dma(dspi, res->start)) { + dev_err(&pdev->dev, "can't get dma channels\n"); + goto out_clk_put; + } + } + master->max_speed_hz = clk_get_rate(dspi->clk) / dspi->devtype_data->max_clock_factor; @@ -768,6 +1065,7 @@ static int dspi_remove(struct platform_device *pdev) struct fsl_dspi *dspi = spi_master_get_devdata(master); /* Disconnect from the SPI framework */ + dspi_release_dma(dspi); clk_disable_unprepare(dspi->clk); spi_unregister_master(dspi->master); diff --git a/drivers/spi/spi-fsl-espi.c b/drivers/spi/spi-fsl-espi.c index 2c175b9..1d332e2 100644 --- a/drivers/spi/spi-fsl-espi.c +++ b/drivers/spi/spi-fsl-espi.c @@ -23,8 +23,6 @@ #include <linux/pm_runtime.h> #include <sysdev/fsl_soc.h> -#include "spi-fsl-lib.h" - /* eSPI Controller registers */ #define ESPI_SPMODE 0x00 /* eSPI mode register */ #define ESPI_SPIE 0x04 /* eSPI event register */ @@ -54,8 +52,11 @@ #define CSMODE_AFT(x) ((x) << 8) #define CSMODE_CG(x) ((x) << 3) +#define FSL_ESPI_FIFO_SIZE 32 +#define FSL_ESPI_RXTHR 15 + /* Default mode/csmode for eSPI controller */ -#define SPMODE_INIT_VAL (SPMODE_TXTHR(4) | SPMODE_RXTHR(3)) +#define SPMODE_INIT_VAL (SPMODE_TXTHR(4) | SPMODE_RXTHR(FSL_ESPI_RXTHR)) #define CSMODE_INIT_VAL (CSMODE_POL_1 | CSMODE_BEF(0) \ | CSMODE_AFT(0) | CSMODE_CG(1)) @@ -90,219 +91,342 @@ #define AUTOSUSPEND_TIMEOUT 2000 -static inline u32 fsl_espi_read_reg(struct mpc8xxx_spi *mspi, int offset) +struct fsl_espi { + struct device *dev; + void __iomem *reg_base; + + struct list_head *m_transfers; + struct spi_transfer *tx_t; + unsigned int tx_pos; + bool tx_done; + struct spi_transfer *rx_t; + unsigned int rx_pos; + bool rx_done; + + bool swab; + unsigned int rxskip; + + spinlock_t lock; + + u32 spibrg; /* SPIBRG input clock */ + + struct completion done; +}; + +struct fsl_espi_cs { + u32 hw_mode; +}; + +static inline u32 fsl_espi_read_reg(struct fsl_espi *espi, int offset) { - return ioread32be(mspi->reg_base + offset); + return ioread32be(espi->reg_base + offset); } -static inline u8 fsl_espi_read_reg8(struct mpc8xxx_spi *mspi, int offset) +static inline u16 fsl_espi_read_reg16(struct fsl_espi *espi, int offset) { - return ioread8(mspi->reg_base + offset); + return ioread16be(espi->reg_base + offset); } -static inline void fsl_espi_write_reg(struct mpc8xxx_spi *mspi, int offset, - u32 val) +static inline u8 fsl_espi_read_reg8(struct fsl_espi *espi, int offset) { - iowrite32be(val, mspi->reg_base + offset); + return ioread8(espi->reg_base + offset); } -static inline void fsl_espi_write_reg8(struct mpc8xxx_spi *mspi, int offset, - u8 val) +static inline void fsl_espi_write_reg(struct fsl_espi *espi, int offset, + u32 val) { - iowrite8(val, mspi->reg_base + offset); + iowrite32be(val, espi->reg_base + offset); } -static void fsl_espi_copy_to_buf(struct spi_message *m, - struct mpc8xxx_spi *mspi) +static inline void fsl_espi_write_reg16(struct fsl_espi *espi, int offset, + u16 val) { - struct spi_transfer *t; - u8 *buf = mspi->local_buf; - - list_for_each_entry(t, &m->transfers, transfer_list) { - if (t->tx_buf) - memcpy(buf, t->tx_buf, t->len); - else - memset(buf, 0, t->len); - buf += t->len; - } + iowrite16be(val, espi->reg_base + offset); } -static void fsl_espi_copy_from_buf(struct spi_message *m, - struct mpc8xxx_spi *mspi) +static inline void fsl_espi_write_reg8(struct fsl_espi *espi, int offset, + u8 val) { - struct spi_transfer *t; - u8 *buf = mspi->local_buf; - - list_for_each_entry(t, &m->transfers, transfer_list) { - if (t->rx_buf) - memcpy(t->rx_buf, buf, t->len); - buf += t->len; - } + iowrite8(val, espi->reg_base + offset); } static int fsl_espi_check_message(struct spi_message *m) { - struct mpc8xxx_spi *mspi = spi_master_get_devdata(m->spi->master); + struct fsl_espi *espi = spi_master_get_devdata(m->spi->master); struct spi_transfer *t, *first; if (m->frame_length > SPCOM_TRANLEN_MAX) { - dev_err(mspi->dev, "message too long, size is %u bytes\n", + dev_err(espi->dev, "message too long, size is %u bytes\n", m->frame_length); return -EMSGSIZE; } first = list_first_entry(&m->transfers, struct spi_transfer, transfer_list); + list_for_each_entry(t, &m->transfers, transfer_list) { if (first->bits_per_word != t->bits_per_word || first->speed_hz != t->speed_hz) { - dev_err(mspi->dev, "bits_per_word/speed_hz should be the same for all transfers\n"); + dev_err(espi->dev, "bits_per_word/speed_hz should be the same for all transfers\n"); return -EINVAL; } } + /* ESPI supports MSB-first transfers for word size 8 / 16 only */ + if (!(m->spi->mode & SPI_LSB_FIRST) && first->bits_per_word != 8 && + first->bits_per_word != 16) { + dev_err(espi->dev, + "MSB-first transfer not supported for wordsize %u\n", + first->bits_per_word); + return -EINVAL; + } + return 0; } -static void fsl_espi_change_mode(struct spi_device *spi) +static unsigned int fsl_espi_check_rxskip_mode(struct spi_message *m) { - struct mpc8xxx_spi *mspi = spi_master_get_devdata(spi->master); - struct spi_mpc8xxx_cs *cs = spi->controller_state; - u32 tmp; - unsigned long flags; - - /* Turn off IRQs locally to minimize time that SPI is disabled. */ - local_irq_save(flags); - - /* Turn off SPI unit prior changing mode */ - tmp = fsl_espi_read_reg(mspi, ESPI_SPMODE); - fsl_espi_write_reg(mspi, ESPI_SPMODE, tmp & ~SPMODE_ENABLE); - fsl_espi_write_reg(mspi, ESPI_SPMODEx(spi->chip_select), - cs->hw_mode); - fsl_espi_write_reg(mspi, ESPI_SPMODE, tmp); - - local_irq_restore(flags); + struct spi_transfer *t; + unsigned int i = 0, rxskip = 0; + + /* + * prerequisites for ESPI rxskip mode: + * - message has two transfers + * - first transfer is a write and second is a read + * + * In addition the current low-level transfer mechanism requires + * that the rxskip bytes fit into the TX FIFO. Else the transfer + * would hang because after the first FSL_ESPI_FIFO_SIZE bytes + * the TX FIFO isn't re-filled. + */ + list_for_each_entry(t, &m->transfers, transfer_list) { + if (i == 0) { + if (!t->tx_buf || t->rx_buf || + t->len > FSL_ESPI_FIFO_SIZE) + return 0; + rxskip = t->len; + } else if (i == 1) { + if (t->tx_buf || !t->rx_buf) + return 0; + } + i++; + } + + return i == 2 ? rxskip : 0; } -static u32 fsl_espi_tx_buf_lsb(struct mpc8xxx_spi *mpc8xxx_spi) +static void fsl_espi_fill_tx_fifo(struct fsl_espi *espi, u32 events) { - u32 data; - u16 data_h; - u16 data_l; - const u32 *tx = mpc8xxx_spi->tx; + u32 tx_fifo_avail; + unsigned int tx_left; + const void *tx_buf; + + /* if events is zero transfer has not started and tx fifo is empty */ + tx_fifo_avail = events ? SPIE_TXCNT(events) : FSL_ESPI_FIFO_SIZE; +start: + tx_left = espi->tx_t->len - espi->tx_pos; + tx_buf = espi->tx_t->tx_buf; + while (tx_fifo_avail >= min(4U, tx_left) && tx_left) { + if (tx_left >= 4) { + if (!tx_buf) + fsl_espi_write_reg(espi, ESPI_SPITF, 0); + else if (espi->swab) + fsl_espi_write_reg(espi, ESPI_SPITF, + swahb32p(tx_buf + espi->tx_pos)); + else + fsl_espi_write_reg(espi, ESPI_SPITF, + *(u32 *)(tx_buf + espi->tx_pos)); + espi->tx_pos += 4; + tx_left -= 4; + tx_fifo_avail -= 4; + } else if (tx_left >= 2 && tx_buf && espi->swab) { + fsl_espi_write_reg16(espi, ESPI_SPITF, + swab16p(tx_buf + espi->tx_pos)); + espi->tx_pos += 2; + tx_left -= 2; + tx_fifo_avail -= 2; + } else { + if (!tx_buf) + fsl_espi_write_reg8(espi, ESPI_SPITF, 0); + else + fsl_espi_write_reg8(espi, ESPI_SPITF, + *(u8 *)(tx_buf + espi->tx_pos)); + espi->tx_pos += 1; + tx_left -= 1; + tx_fifo_avail -= 1; + } + } - if (!tx) - return 0; + if (!tx_left) { + /* Last transfer finished, in rxskip mode only one is needed */ + if (list_is_last(&espi->tx_t->transfer_list, + espi->m_transfers) || espi->rxskip) { + espi->tx_done = true; + return; + } + espi->tx_t = list_next_entry(espi->tx_t, transfer_list); + espi->tx_pos = 0; + /* continue with next transfer if tx fifo is not full */ + if (tx_fifo_avail) + goto start; + } +} - data = *tx++ << mpc8xxx_spi->tx_shift; - data_l = data & 0xffff; - data_h = (data >> 16) & 0xffff; - swab16s(&data_l); - swab16s(&data_h); - data = data_h | data_l; +static void fsl_espi_read_rx_fifo(struct fsl_espi *espi, u32 events) +{ + u32 rx_fifo_avail = SPIE_RXCNT(events); + unsigned int rx_left; + void *rx_buf; + +start: + rx_left = espi->rx_t->len - espi->rx_pos; + rx_buf = espi->rx_t->rx_buf; + while (rx_fifo_avail >= min(4U, rx_left) && rx_left) { + if (rx_left >= 4) { + u32 val = fsl_espi_read_reg(espi, ESPI_SPIRF); + + if (rx_buf && espi->swab) + *(u32 *)(rx_buf + espi->rx_pos) = swahb32(val); + else if (rx_buf) + *(u32 *)(rx_buf + espi->rx_pos) = val; + espi->rx_pos += 4; + rx_left -= 4; + rx_fifo_avail -= 4; + } else if (rx_left >= 2 && rx_buf && espi->swab) { + u16 val = fsl_espi_read_reg16(espi, ESPI_SPIRF); + + *(u16 *)(rx_buf + espi->rx_pos) = swab16(val); + espi->rx_pos += 2; + rx_left -= 2; + rx_fifo_avail -= 2; + } else { + u8 val = fsl_espi_read_reg8(espi, ESPI_SPIRF); + + if (rx_buf) + *(u8 *)(rx_buf + espi->rx_pos) = val; + espi->rx_pos += 1; + rx_left -= 1; + rx_fifo_avail -= 1; + } + } - mpc8xxx_spi->tx = tx; - return data; + if (!rx_left) { + if (list_is_last(&espi->rx_t->transfer_list, + espi->m_transfers)) { + espi->rx_done = true; + return; + } + espi->rx_t = list_next_entry(espi->rx_t, transfer_list); + espi->rx_pos = 0; + /* continue with next transfer if rx fifo is not empty */ + if (rx_fifo_avail) + goto start; + } } static void fsl_espi_setup_transfer(struct spi_device *spi, struct spi_transfer *t) { - struct mpc8xxx_spi *mpc8xxx_spi = spi_master_get_devdata(spi->master); + struct fsl_espi *espi = spi_master_get_devdata(spi->master); int bits_per_word = t ? t->bits_per_word : spi->bits_per_word; - u32 hz = t ? t->speed_hz : spi->max_speed_hz; - u8 pm; - struct spi_mpc8xxx_cs *cs = spi->controller_state; - - cs->rx_shift = 0; - cs->tx_shift = 0; - cs->get_rx = mpc8xxx_spi_rx_buf_u32; - cs->get_tx = mpc8xxx_spi_tx_buf_u32; - if (bits_per_word <= 8) { - cs->rx_shift = 8 - bits_per_word; - } else { - cs->rx_shift = 16 - bits_per_word; - if (spi->mode & SPI_LSB_FIRST) - cs->get_tx = fsl_espi_tx_buf_lsb; - } - - mpc8xxx_spi->rx_shift = cs->rx_shift; - mpc8xxx_spi->tx_shift = cs->tx_shift; - mpc8xxx_spi->get_rx = cs->get_rx; - mpc8xxx_spi->get_tx = cs->get_tx; + u32 pm, hz = t ? t->speed_hz : spi->max_speed_hz; + struct fsl_espi_cs *cs = spi_get_ctldata(spi); + u32 hw_mode_old = cs->hw_mode; /* mask out bits we are going to set */ cs->hw_mode &= ~(CSMODE_LEN(0xF) | CSMODE_DIV16 | CSMODE_PM(0xF)); cs->hw_mode |= CSMODE_LEN(bits_per_word - 1); - if ((mpc8xxx_spi->spibrg / hz) > 64) { + pm = DIV_ROUND_UP(espi->spibrg, hz * 4) - 1; + + if (pm > 15) { cs->hw_mode |= CSMODE_DIV16; - pm = DIV_ROUND_UP(mpc8xxx_spi->spibrg, hz * 16 * 4); - - WARN_ONCE(pm > 33, "%s: Requested speed is too low: %d Hz. " - "Will use %d Hz instead.\n", dev_name(&spi->dev), - hz, mpc8xxx_spi->spibrg / (4 * 16 * (32 + 1))); - if (pm > 33) - pm = 33; - } else { - pm = DIV_ROUND_UP(mpc8xxx_spi->spibrg, hz * 4); + pm = DIV_ROUND_UP(espi->spibrg, hz * 16 * 4) - 1; } - if (pm) - pm--; - if (pm < 2) - pm = 2; cs->hw_mode |= CSMODE_PM(pm); - fsl_espi_change_mode(spi); + /* don't write the mode register if the mode doesn't change */ + if (cs->hw_mode != hw_mode_old) + fsl_espi_write_reg(espi, ESPI_SPMODEx(spi->chip_select), + cs->hw_mode); } static int fsl_espi_bufs(struct spi_device *spi, struct spi_transfer *t) { - struct mpc8xxx_spi *mpc8xxx_spi = spi_master_get_devdata(spi->master); - u32 word; + struct fsl_espi *espi = spi_master_get_devdata(spi->master); + unsigned int rx_len = t->len; + u32 mask, spcom; int ret; - mpc8xxx_spi->len = t->len; - mpc8xxx_spi->count = roundup(t->len, 4) / 4; - - mpc8xxx_spi->tx = t->tx_buf; - mpc8xxx_spi->rx = t->rx_buf; - - reinit_completion(&mpc8xxx_spi->done); + reinit_completion(&espi->done); /* Set SPCOM[CS] and SPCOM[TRANLEN] field */ - fsl_espi_write_reg(mpc8xxx_spi, ESPI_SPCOM, - (SPCOM_CS(spi->chip_select) | SPCOM_TRANLEN(t->len - 1))); + spcom = SPCOM_CS(spi->chip_select); + spcom |= SPCOM_TRANLEN(t->len - 1); + + /* configure RXSKIP mode */ + if (espi->rxskip) { + spcom |= SPCOM_RXSKIP(espi->rxskip); + rx_len = t->len - espi->rxskip; + if (t->rx_nbits == SPI_NBITS_DUAL) + spcom |= SPCOM_DO; + } + + fsl_espi_write_reg(espi, ESPI_SPCOM, spcom); - /* enable rx ints */ - fsl_espi_write_reg(mpc8xxx_spi, ESPI_SPIM, SPIM_RNE); + /* enable interrupts */ + mask = SPIM_DON; + if (rx_len > FSL_ESPI_FIFO_SIZE) + mask |= SPIM_RXT; + fsl_espi_write_reg(espi, ESPI_SPIM, mask); - /* transmit word */ - word = mpc8xxx_spi->get_tx(mpc8xxx_spi); - fsl_espi_write_reg(mpc8xxx_spi, ESPI_SPITF, word); + /* Prevent filling the fifo from getting interrupted */ + spin_lock_irq(&espi->lock); + fsl_espi_fill_tx_fifo(espi, 0); + spin_unlock_irq(&espi->lock); /* Won't hang up forever, SPI bus sometimes got lost interrupts... */ - ret = wait_for_completion_timeout(&mpc8xxx_spi->done, 2 * HZ); + ret = wait_for_completion_timeout(&espi->done, 2 * HZ); if (ret == 0) - dev_err(mpc8xxx_spi->dev, - "Transaction hanging up (left %d bytes)\n", - mpc8xxx_spi->count); + dev_err(espi->dev, "Transfer timed out!\n"); /* disable rx ints */ - fsl_espi_write_reg(mpc8xxx_spi, ESPI_SPIM, 0); + fsl_espi_write_reg(espi, ESPI_SPIM, 0); - return mpc8xxx_spi->count > 0 ? -EMSGSIZE : 0; + return ret == 0 ? -ETIMEDOUT : 0; } static int fsl_espi_trans(struct spi_message *m, struct spi_transfer *trans) { - struct mpc8xxx_spi *mspi = spi_master_get_devdata(m->spi->master); + struct fsl_espi *espi = spi_master_get_devdata(m->spi->master); struct spi_device *spi = m->spi; int ret; - fsl_espi_copy_to_buf(m, mspi); + /* In case of LSB-first and bits_per_word > 8 byte-swap all words */ + espi->swab = spi->mode & SPI_LSB_FIRST && trans->bits_per_word > 8; + + espi->m_transfers = &m->transfers; + espi->tx_t = list_first_entry(&m->transfers, struct spi_transfer, + transfer_list); + espi->tx_pos = 0; + espi->tx_done = false; + espi->rx_t = list_first_entry(&m->transfers, struct spi_transfer, + transfer_list); + espi->rx_pos = 0; + espi->rx_done = false; + + espi->rxskip = fsl_espi_check_rxskip_mode(m); + if (trans->rx_nbits == SPI_NBITS_DUAL && !espi->rxskip) { + dev_err(espi->dev, "Dual output mode requires RXSKIP mode!\n"); + return -EINVAL; + } + + /* In RXSKIP mode skip first transfer for reads */ + if (espi->rxskip) + espi->rx_t = list_next_entry(espi->rx_t, transfer_list); + fsl_espi_setup_transfer(spi, trans); ret = fsl_espi_bufs(spi, trans); @@ -310,19 +434,13 @@ static int fsl_espi_trans(struct spi_message *m, struct spi_transfer *trans) if (trans->delay_usecs) udelay(trans->delay_usecs); - fsl_espi_setup_transfer(spi, NULL); - - if (!ret) - fsl_espi_copy_from_buf(m, mspi); - return ret; } static int fsl_espi_do_one_msg(struct spi_master *master, struct spi_message *m) { - struct mpc8xxx_spi *mspi = spi_master_get_devdata(m->spi->master); - unsigned int delay_usecs = 0; + unsigned int delay_usecs = 0, rx_nbits = 0; struct spi_transfer *t, trans = {}; int ret; @@ -333,6 +451,8 @@ static int fsl_espi_do_one_msg(struct spi_master *master, list_for_each_entry(t, &m->transfers, transfer_list) { if (t->delay_usecs > delay_usecs) delay_usecs = t->delay_usecs; + if (t->rx_nbits > rx_nbits) + rx_nbits = t->rx_nbits; } t = list_first_entry(&m->transfers, struct spi_transfer, @@ -342,8 +462,7 @@ static int fsl_espi_do_one_msg(struct spi_master *master, trans.speed_hz = t->speed_hz; trans.bits_per_word = t->bits_per_word; trans.delay_usecs = delay_usecs; - trans.tx_buf = mspi->local_buf; - trans.rx_buf = mspi->local_buf; + trans.rx_nbits = rx_nbits; if (trans.len) ret = fsl_espi_trans(m, &trans); @@ -360,12 +479,9 @@ out: static int fsl_espi_setup(struct spi_device *spi) { - struct mpc8xxx_spi *mpc8xxx_spi; + struct fsl_espi *espi; u32 loop_mode; - struct spi_mpc8xxx_cs *cs = spi_get_ctldata(spi); - - if (!spi->max_speed_hz) - return -EINVAL; + struct fsl_espi_cs *cs = spi_get_ctldata(spi); if (!cs) { cs = kzalloc(sizeof(*cs), GFP_KERNEL); @@ -374,12 +490,11 @@ static int fsl_espi_setup(struct spi_device *spi) spi_set_ctldata(spi, cs); } - mpc8xxx_spi = spi_master_get_devdata(spi->master); + espi = spi_master_get_devdata(spi->master); - pm_runtime_get_sync(mpc8xxx_spi->dev); + pm_runtime_get_sync(espi->dev); - cs->hw_mode = fsl_espi_read_reg(mpc8xxx_spi, - ESPI_SPMODEx(spi->chip_select)); + cs->hw_mode = fsl_espi_read_reg(espi, ESPI_SPMODEx(spi->chip_select)); /* mask out bits we are going to set */ cs->hw_mode &= ~(CSMODE_CP_BEGIN_EDGECLK | CSMODE_CI_INACTIVEHIGH | CSMODE_REV); @@ -392,115 +507,74 @@ static int fsl_espi_setup(struct spi_device *spi) cs->hw_mode |= CSMODE_REV; /* Handle the loop mode */ - loop_mode = fsl_espi_read_reg(mpc8xxx_spi, ESPI_SPMODE); + loop_mode = fsl_espi_read_reg(espi, ESPI_SPMODE); loop_mode &= ~SPMODE_LOOP; if (spi->mode & SPI_LOOP) loop_mode |= SPMODE_LOOP; - fsl_espi_write_reg(mpc8xxx_spi, ESPI_SPMODE, loop_mode); + fsl_espi_write_reg(espi, ESPI_SPMODE, loop_mode); fsl_espi_setup_transfer(spi, NULL); - pm_runtime_mark_last_busy(mpc8xxx_spi->dev); - pm_runtime_put_autosuspend(mpc8xxx_spi->dev); + pm_runtime_mark_last_busy(espi->dev); + pm_runtime_put_autosuspend(espi->dev); return 0; } static void fsl_espi_cleanup(struct spi_device *spi) { - struct spi_mpc8xxx_cs *cs = spi_get_ctldata(spi); + struct fsl_espi_cs *cs = spi_get_ctldata(spi); kfree(cs); spi_set_ctldata(spi, NULL); } -static void fsl_espi_cpu_irq(struct mpc8xxx_spi *mspi, u32 events) +static void fsl_espi_cpu_irq(struct fsl_espi *espi, u32 events) { - /* We need handle RX first */ - if (events & SPIE_RNE) { - u32 rx_data, tmp; - u8 rx_data_8; - int rx_nr_bytes = 4; - int ret; - - /* Spin until RX is done */ - if (SPIE_RXCNT(events) < min(4, mspi->len)) { - ret = spin_event_timeout( - !(SPIE_RXCNT(events = - fsl_espi_read_reg(mspi, ESPI_SPIE)) < - min(4, mspi->len)), - 10000, 0); /* 10 msec */ - if (!ret) - dev_err(mspi->dev, - "tired waiting for SPIE_RXCNT\n"); - } + if (!espi->rx_done) + fsl_espi_read_rx_fifo(espi, events); - if (mspi->len >= 4) { - rx_data = fsl_espi_read_reg(mspi, ESPI_SPIRF); - } else if (mspi->len <= 0) { - dev_err(mspi->dev, - "unexpected RX(SPIE_RNE) interrupt occurred,\n" - "(local rxlen %d bytes, reg rxlen %d bytes)\n", - min(4, mspi->len), SPIE_RXCNT(events)); - rx_nr_bytes = 0; - } else { - rx_nr_bytes = mspi->len; - tmp = mspi->len; - rx_data = 0; - while (tmp--) { - rx_data_8 = fsl_espi_read_reg8(mspi, - ESPI_SPIRF); - rx_data |= (rx_data_8 << (tmp * 8)); - } - - rx_data <<= (4 - mspi->len) * 8; - } + if (!espi->tx_done) + fsl_espi_fill_tx_fifo(espi, events); - mspi->len -= rx_nr_bytes; + if (!espi->tx_done || !espi->rx_done) + return; - if (rx_nr_bytes && mspi->rx) - mspi->get_rx(rx_data, mspi); - } + /* we're done, but check for errors before returning */ + events = fsl_espi_read_reg(espi, ESPI_SPIE); - if (!(events & SPIE_TNF)) { - int ret; - - /* spin until TX is done */ - ret = spin_event_timeout(((events = fsl_espi_read_reg( - mspi, ESPI_SPIE)) & SPIE_TNF), 1000, 0); - if (!ret) { - dev_err(mspi->dev, "tired waiting for SPIE_TNF\n"); - complete(&mspi->done); - return; - } - } + if (!(events & SPIE_DON)) + dev_err(espi->dev, + "Transfer done but SPIE_DON isn't set!\n"); - mspi->count -= 1; - if (mspi->count) { - u32 word = mspi->get_tx(mspi); + if (SPIE_RXCNT(events) || SPIE_TXCNT(events) != FSL_ESPI_FIFO_SIZE) + dev_err(espi->dev, "Transfer done but rx/tx fifo's aren't empty!\n"); - fsl_espi_write_reg(mspi, ESPI_SPITF, word); - } else { - complete(&mspi->done); - } + complete(&espi->done); } static irqreturn_t fsl_espi_irq(s32 irq, void *context_data) { - struct mpc8xxx_spi *mspi = context_data; + struct fsl_espi *espi = context_data; u32 events; + spin_lock(&espi->lock); + /* Get interrupt events(tx/rx) */ - events = fsl_espi_read_reg(mspi, ESPI_SPIE); - if (!events) + events = fsl_espi_read_reg(espi, ESPI_SPIE); + if (!events) { + spin_unlock(&espi->lock); return IRQ_NONE; + } - dev_vdbg(mspi->dev, "%s: events %x\n", __func__, events); + dev_vdbg(espi->dev, "%s: events %x\n", __func__, events); - fsl_espi_cpu_irq(mspi, events); + fsl_espi_cpu_irq(espi, events); /* Clear the events */ - fsl_espi_write_reg(mspi, ESPI_SPIE, events); + fsl_espi_write_reg(espi, ESPI_SPIE, events); + + spin_unlock(&espi->lock); return IRQ_HANDLED; } @@ -509,12 +583,12 @@ static irqreturn_t fsl_espi_irq(s32 irq, void *context_data) static int fsl_espi_runtime_suspend(struct device *dev) { struct spi_master *master = dev_get_drvdata(dev); - struct mpc8xxx_spi *mpc8xxx_spi = spi_master_get_devdata(master); + struct fsl_espi *espi = spi_master_get_devdata(master); u32 regval; - regval = fsl_espi_read_reg(mpc8xxx_spi, ESPI_SPMODE); + regval = fsl_espi_read_reg(espi, ESPI_SPMODE); regval &= ~SPMODE_ENABLE; - fsl_espi_write_reg(mpc8xxx_spi, ESPI_SPMODE, regval); + fsl_espi_write_reg(espi, ESPI_SPMODE, regval); return 0; } @@ -522,12 +596,12 @@ static int fsl_espi_runtime_suspend(struct device *dev) static int fsl_espi_runtime_resume(struct device *dev) { struct spi_master *master = dev_get_drvdata(dev); - struct mpc8xxx_spi *mpc8xxx_spi = spi_master_get_devdata(master); + struct fsl_espi *espi = spi_master_get_devdata(master); u32 regval; - regval = fsl_espi_read_reg(mpc8xxx_spi, ESPI_SPMODE); + regval = fsl_espi_read_reg(espi, ESPI_SPMODE); regval |= SPMODE_ENABLE; - fsl_espi_write_reg(mpc8xxx_spi, ESPI_SPMODE, regval); + fsl_espi_write_reg(espi, ESPI_SPMODE, regval); return 0; } @@ -538,96 +612,105 @@ static size_t fsl_espi_max_message_size(struct spi_device *spi) return SPCOM_TRANLEN_MAX; } +static void fsl_espi_init_regs(struct device *dev, bool initial) +{ + struct spi_master *master = dev_get_drvdata(dev); + struct fsl_espi *espi = spi_master_get_devdata(master); + struct device_node *nc; + u32 csmode, cs, prop; + int ret; + + /* SPI controller initializations */ + fsl_espi_write_reg(espi, ESPI_SPMODE, 0); + fsl_espi_write_reg(espi, ESPI_SPIM, 0); + fsl_espi_write_reg(espi, ESPI_SPCOM, 0); + fsl_espi_write_reg(espi, ESPI_SPIE, 0xffffffff); + + /* Init eSPI CS mode register */ + for_each_available_child_of_node(master->dev.of_node, nc) { + /* get chip select */ + ret = of_property_read_u32(nc, "reg", &cs); + if (ret || cs >= master->num_chipselect) + continue; + + csmode = CSMODE_INIT_VAL; + + /* check if CSBEF is set in device tree */ + ret = of_property_read_u32(nc, "fsl,csbef", &prop); + if (!ret) { + csmode &= ~(CSMODE_BEF(0xf)); + csmode |= CSMODE_BEF(prop); + } + + /* check if CSAFT is set in device tree */ + ret = of_property_read_u32(nc, "fsl,csaft", &prop); + if (!ret) { + csmode &= ~(CSMODE_AFT(0xf)); + csmode |= CSMODE_AFT(prop); + } + + fsl_espi_write_reg(espi, ESPI_SPMODEx(cs), csmode); + + if (initial) + dev_info(dev, "cs=%u, init_csmode=0x%x\n", cs, csmode); + } + + /* Enable SPI interface */ + fsl_espi_write_reg(espi, ESPI_SPMODE, SPMODE_INIT_VAL | SPMODE_ENABLE); +} + static int fsl_espi_probe(struct device *dev, struct resource *mem, - unsigned int irq) + unsigned int irq, unsigned int num_cs) { - struct fsl_spi_platform_data *pdata = dev_get_platdata(dev); struct spi_master *master; - struct mpc8xxx_spi *mpc8xxx_spi; - struct device_node *nc; - const __be32 *prop; - u32 regval, csmode; - int i, len, ret; + struct fsl_espi *espi; + int ret; - master = spi_alloc_master(dev, sizeof(struct mpc8xxx_spi)); + master = spi_alloc_master(dev, sizeof(struct fsl_espi)); if (!master) return -ENOMEM; dev_set_drvdata(dev, master); - mpc8xxx_spi_probe(dev, mem, irq); - + master->mode_bits = SPI_RX_DUAL | SPI_CPOL | SPI_CPHA | SPI_CS_HIGH | + SPI_LSB_FIRST | SPI_LOOP; + master->dev.of_node = dev->of_node; master->bits_per_word_mask = SPI_BPW_RANGE_MASK(4, 16); master->setup = fsl_espi_setup; master->cleanup = fsl_espi_cleanup; master->transfer_one_message = fsl_espi_do_one_msg; master->auto_runtime_pm = true; master->max_message_size = fsl_espi_max_message_size; + master->num_chipselect = num_cs; - mpc8xxx_spi = spi_master_get_devdata(master); + espi = spi_master_get_devdata(master); + spin_lock_init(&espi->lock); - mpc8xxx_spi->local_buf = - devm_kmalloc(dev, SPCOM_TRANLEN_MAX, GFP_KERNEL); - if (!mpc8xxx_spi->local_buf) { - ret = -ENOMEM; + espi->dev = dev; + espi->spibrg = fsl_get_sys_freq(); + if (espi->spibrg == -1) { + dev_err(dev, "Can't get sys frequency!\n"); + ret = -EINVAL; goto err_probe; } + /* determined by clock divider fields DIV16/PM in register SPMODEx */ + master->min_speed_hz = DIV_ROUND_UP(espi->spibrg, 4 * 16 * 16); + master->max_speed_hz = DIV_ROUND_UP(espi->spibrg, 4); - mpc8xxx_spi->reg_base = devm_ioremap_resource(dev, mem); - if (IS_ERR(mpc8xxx_spi->reg_base)) { - ret = PTR_ERR(mpc8xxx_spi->reg_base); + init_completion(&espi->done); + + espi->reg_base = devm_ioremap_resource(dev, mem); + if (IS_ERR(espi->reg_base)) { + ret = PTR_ERR(espi->reg_base); goto err_probe; } /* Register for SPI Interrupt */ - ret = devm_request_irq(dev, mpc8xxx_spi->irq, fsl_espi_irq, - 0, "fsl_espi", mpc8xxx_spi); + ret = devm_request_irq(dev, irq, fsl_espi_irq, 0, "fsl_espi", espi); if (ret) goto err_probe; - if (mpc8xxx_spi->flags & SPI_QE_CPU_MODE) { - mpc8xxx_spi->rx_shift = 16; - mpc8xxx_spi->tx_shift = 24; - } - - /* SPI controller initializations */ - fsl_espi_write_reg(mpc8xxx_spi, ESPI_SPMODE, 0); - fsl_espi_write_reg(mpc8xxx_spi, ESPI_SPIM, 0); - fsl_espi_write_reg(mpc8xxx_spi, ESPI_SPCOM, 0); - fsl_espi_write_reg(mpc8xxx_spi, ESPI_SPIE, 0xffffffff); - - /* Init eSPI CS mode register */ - for_each_available_child_of_node(master->dev.of_node, nc) { - /* get chip select */ - prop = of_get_property(nc, "reg", &len); - if (!prop || len < sizeof(*prop)) - continue; - i = be32_to_cpup(prop); - if (i < 0 || i >= pdata->max_chipselect) - continue; - - csmode = CSMODE_INIT_VAL; - /* check if CSBEF is set in device tree */ - prop = of_get_property(nc, "fsl,csbef", &len); - if (prop && len >= sizeof(*prop)) { - csmode &= ~(CSMODE_BEF(0xf)); - csmode |= CSMODE_BEF(be32_to_cpup(prop)); - } - /* check if CSAFT is set in device tree */ - prop = of_get_property(nc, "fsl,csaft", &len); - if (prop && len >= sizeof(*prop)) { - csmode &= ~(CSMODE_AFT(0xf)); - csmode |= CSMODE_AFT(be32_to_cpup(prop)); - } - fsl_espi_write_reg(mpc8xxx_spi, ESPI_SPMODEx(i), csmode); - - dev_info(dev, "cs=%d, init_csmode=0x%x\n", i, csmode); - } - - /* Enable SPI interface */ - regval = pdata->initial_spmode | SPMODE_INIT_VAL | SPMODE_ENABLE; - - fsl_espi_write_reg(mpc8xxx_spi, ESPI_SPMODE, regval); + fsl_espi_init_regs(dev, true); pm_runtime_set_autosuspend_delay(dev, AUTOSUSPEND_TIMEOUT); pm_runtime_use_autosuspend(dev); @@ -639,8 +722,7 @@ static int fsl_espi_probe(struct device *dev, struct resource *mem, if (ret < 0) goto err_pm; - dev_info(dev, "at 0x%p (irq = %d)\n", mpc8xxx_spi->reg_base, - mpc8xxx_spi->irq); + dev_info(dev, "at 0x%p (irq = %u)\n", espi->reg_base, irq); pm_runtime_mark_last_busy(dev); pm_runtime_put_autosuspend(dev); @@ -659,20 +741,16 @@ err_probe: static int of_fsl_espi_get_chipselects(struct device *dev) { struct device_node *np = dev->of_node; - struct fsl_spi_platform_data *pdata = dev_get_platdata(dev); - const u32 *prop; - int len; + u32 num_cs; + int ret; - prop = of_get_property(np, "fsl,espi-num-chipselects", &len); - if (!prop || len < sizeof(*prop)) { + ret = of_property_read_u32(np, "fsl,espi-num-chipselects", &num_cs); + if (ret) { dev_err(dev, "No 'fsl,espi-num-chipselects' property\n"); - return -EINVAL; + return 0; } - pdata->max_chipselect = *prop; - pdata->cs_control = NULL; - - return 0; + return num_cs; } static int of_fsl_espi_probe(struct platform_device *ofdev) @@ -680,16 +758,17 @@ static int of_fsl_espi_probe(struct platform_device *ofdev) struct device *dev = &ofdev->dev; struct device_node *np = ofdev->dev.of_node; struct resource mem; - unsigned int irq; + unsigned int irq, num_cs; int ret; - ret = of_mpc8xxx_spi_probe(ofdev); - if (ret) - return ret; + if (of_property_read_bool(np, "mode")) { + dev_err(dev, "mode property is not supported on ESPI!\n"); + return -EINVAL; + } - ret = of_fsl_espi_get_chipselects(dev); - if (ret) - return ret; + num_cs = of_fsl_espi_get_chipselects(dev); + if (!num_cs) + return -EINVAL; ret = of_address_to_resource(np, 0, &mem); if (ret) @@ -699,7 +778,7 @@ static int of_fsl_espi_probe(struct platform_device *ofdev) if (!irq) return -EINVAL; - return fsl_espi_probe(dev, &mem, irq); + return fsl_espi_probe(dev, &mem, irq, num_cs); } static int of_fsl_espi_remove(struct platform_device *dev) @@ -721,38 +800,15 @@ static int of_fsl_espi_suspend(struct device *dev) return ret; } - ret = pm_runtime_force_suspend(dev); - if (ret < 0) - return ret; - - return 0; + return pm_runtime_force_suspend(dev); } static int of_fsl_espi_resume(struct device *dev) { - struct fsl_spi_platform_data *pdata = dev_get_platdata(dev); struct spi_master *master = dev_get_drvdata(dev); - struct mpc8xxx_spi *mpc8xxx_spi; - u32 regval; - int i, ret; - - mpc8xxx_spi = spi_master_get_devdata(master); - - /* SPI controller initializations */ - fsl_espi_write_reg(mpc8xxx_spi, ESPI_SPMODE, 0); - fsl_espi_write_reg(mpc8xxx_spi, ESPI_SPIM, 0); - fsl_espi_write_reg(mpc8xxx_spi, ESPI_SPCOM, 0); - fsl_espi_write_reg(mpc8xxx_spi, ESPI_SPIE, 0xffffffff); - - /* Init eSPI CS mode register */ - for (i = 0; i < pdata->max_chipselect; i++) - fsl_espi_write_reg(mpc8xxx_spi, ESPI_SPMODEx(i), - CSMODE_INIT_VAL); - - /* Enable SPI interface */ - regval = pdata->initial_spmode | SPMODE_INIT_VAL | SPMODE_ENABLE; + int ret; - fsl_espi_write_reg(mpc8xxx_spi, ESPI_SPMODE, regval); + fsl_espi_init_regs(dev, false); ret = pm_runtime_force_resume(dev); if (ret < 0) diff --git a/drivers/spi/spi-fsl-lib.h b/drivers/spi/spi-fsl-lib.h index 2925c80..f303f30 100644 --- a/drivers/spi/spi-fsl-lib.h +++ b/drivers/spi/spi-fsl-lib.h @@ -28,10 +28,6 @@ struct mpc8xxx_spi { /* rx & tx bufs from the spi_transfer */ const void *tx; void *rx; -#if IS_ENABLED(CONFIG_SPI_FSL_ESPI) - int len; - u8 *local_buf; -#endif int subblock; struct spi_pram __iomem *pram; diff --git a/drivers/spi/spi-fsl-lpspi.c b/drivers/spi/spi-fsl-lpspi.c new file mode 100644 index 0000000..52551f6 --- /dev/null +++ b/drivers/spi/spi-fsl-lpspi.c @@ -0,0 +1,525 @@ +/* + * Freescale i.MX7ULP LPSPI driver + * + * Copyright 2016 Freescale Semiconductor, Inc. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + */ + +#include <linux/clk.h> +#include <linux/completion.h> +#include <linux/delay.h> +#include <linux/err.h> +#include <linux/interrupt.h> +#include <linux/io.h> +#include <linux/irq.h> +#include <linux/kernel.h> +#include <linux/module.h> +#include <linux/of.h> +#include <linux/of_device.h> +#include <linux/platform_device.h> +#include <linux/slab.h> +#include <linux/spi/spi.h> +#include <linux/spi/spi_bitbang.h> +#include <linux/types.h> + +#define DRIVER_NAME "fsl_lpspi" + +/* i.MX7ULP LPSPI registers */ +#define IMX7ULP_VERID 0x0 +#define IMX7ULP_PARAM 0x4 +#define IMX7ULP_CR 0x10 +#define IMX7ULP_SR 0x14 +#define IMX7ULP_IER 0x18 +#define IMX7ULP_DER 0x1c +#define IMX7ULP_CFGR0 0x20 +#define IMX7ULP_CFGR1 0x24 +#define IMX7ULP_DMR0 0x30 +#define IMX7ULP_DMR1 0x34 +#define IMX7ULP_CCR 0x40 +#define IMX7ULP_FCR 0x58 +#define IMX7ULP_FSR 0x5c +#define IMX7ULP_TCR 0x60 +#define IMX7ULP_TDR 0x64 +#define IMX7ULP_RSR 0x70 +#define IMX7ULP_RDR 0x74 + +/* General control register field define */ +#define CR_RRF BIT(9) +#define CR_RTF BIT(8) +#define CR_RST BIT(1) +#define CR_MEN BIT(0) +#define SR_TCF BIT(10) +#define SR_RDF BIT(1) +#define SR_TDF BIT(0) +#define IER_TCIE BIT(10) +#define IER_RDIE BIT(1) +#define IER_TDIE BIT(0) +#define CFGR1_PCSCFG BIT(27) +#define CFGR1_PCSPOL BIT(8) +#define CFGR1_NOSTALL BIT(3) +#define CFGR1_MASTER BIT(0) +#define RSR_RXEMPTY BIT(1) +#define TCR_CPOL BIT(31) +#define TCR_CPHA BIT(30) +#define TCR_CONT BIT(21) +#define TCR_CONTC BIT(20) +#define TCR_RXMSK BIT(19) +#define TCR_TXMSK BIT(18) + +static int clkdivs[] = {1, 2, 4, 8, 16, 32, 64, 128}; + +struct lpspi_config { + u8 bpw; + u8 chip_select; + u8 prescale; + u16 mode; + u32 speed_hz; +}; + +struct fsl_lpspi_data { + struct device *dev; + void __iomem *base; + struct clk *clk; + + void *rx_buf; + const void *tx_buf; + void (*tx)(struct fsl_lpspi_data *); + void (*rx)(struct fsl_lpspi_data *); + + u32 remain; + u8 txfifosize; + u8 rxfifosize; + + struct lpspi_config config; + struct completion xfer_done; +}; + +static const struct of_device_id fsl_lpspi_dt_ids[] = { + { .compatible = "fsl,imx7ulp-spi", }, + { /* sentinel */ } +}; +MODULE_DEVICE_TABLE(of, fsl_lpspi_dt_ids); + +#define LPSPI_BUF_RX(type) \ +static void fsl_lpspi_buf_rx_##type(struct fsl_lpspi_data *fsl_lpspi) \ +{ \ + unsigned int val = readl(fsl_lpspi->base + IMX7ULP_RDR); \ + \ + if (fsl_lpspi->rx_buf) { \ + *(type *)fsl_lpspi->rx_buf = val; \ + fsl_lpspi->rx_buf += sizeof(type); \ + } \ +} + +#define LPSPI_BUF_TX(type) \ +static void fsl_lpspi_buf_tx_##type(struct fsl_lpspi_data *fsl_lpspi) \ +{ \ + type val = 0; \ + \ + if (fsl_lpspi->tx_buf) { \ + val = *(type *)fsl_lpspi->tx_buf; \ + fsl_lpspi->tx_buf += sizeof(type); \ + } \ + \ + fsl_lpspi->remain -= sizeof(type); \ + writel(val, fsl_lpspi->base + IMX7ULP_TDR); \ +} + +LPSPI_BUF_RX(u8) +LPSPI_BUF_TX(u8) +LPSPI_BUF_RX(u16) +LPSPI_BUF_TX(u16) +LPSPI_BUF_RX(u32) +LPSPI_BUF_TX(u32) + +static void fsl_lpspi_intctrl(struct fsl_lpspi_data *fsl_lpspi, + unsigned int enable) +{ + writel(enable, fsl_lpspi->base + IMX7ULP_IER); +} + +static int lpspi_prepare_xfer_hardware(struct spi_master *master) +{ + struct fsl_lpspi_data *fsl_lpspi = spi_master_get_devdata(master); + + return clk_prepare_enable(fsl_lpspi->clk); +} + +static int lpspi_unprepare_xfer_hardware(struct spi_master *master) +{ + struct fsl_lpspi_data *fsl_lpspi = spi_master_get_devdata(master); + + clk_disable_unprepare(fsl_lpspi->clk); + + return 0; +} + +static int fsl_lpspi_txfifo_empty(struct fsl_lpspi_data *fsl_lpspi) +{ + u32 txcnt; + unsigned long orig_jiffies = jiffies; + + do { + txcnt = readl(fsl_lpspi->base + IMX7ULP_FSR) & 0xff; + + if (time_after(jiffies, orig_jiffies + msecs_to_jiffies(500))) { + dev_dbg(fsl_lpspi->dev, "txfifo empty timeout\n"); + return -ETIMEDOUT; + } + cond_resched(); + + } while (txcnt); + + return 0; +} + +static void fsl_lpspi_write_tx_fifo(struct fsl_lpspi_data *fsl_lpspi) +{ + u8 txfifo_cnt; + + txfifo_cnt = readl(fsl_lpspi->base + IMX7ULP_FSR) & 0xff; + + while (txfifo_cnt < fsl_lpspi->txfifosize) { + if (!fsl_lpspi->remain) + break; + fsl_lpspi->tx(fsl_lpspi); + txfifo_cnt++; + } + + if (!fsl_lpspi->remain && (txfifo_cnt < fsl_lpspi->txfifosize)) + writel(0, fsl_lpspi->base + IMX7ULP_TDR); + else + fsl_lpspi_intctrl(fsl_lpspi, IER_TDIE); +} + +static void fsl_lpspi_read_rx_fifo(struct fsl_lpspi_data *fsl_lpspi) +{ + while (!(readl(fsl_lpspi->base + IMX7ULP_RSR) & RSR_RXEMPTY)) + fsl_lpspi->rx(fsl_lpspi); +} + +static void fsl_lpspi_set_cmd(struct fsl_lpspi_data *fsl_lpspi, + bool is_first_xfer) +{ + u32 temp = 0; + + temp |= fsl_lpspi->config.bpw - 1; + temp |= fsl_lpspi->config.prescale << 27; + temp |= (fsl_lpspi->config.mode & 0x3) << 30; + temp |= (fsl_lpspi->config.chip_select & 0x3) << 24; + + /* + * Set TCR_CONT will keep SS asserted after current transfer. + * For the first transfer, clear TCR_CONTC to assert SS. + * For subsequent transfer, set TCR_CONTC to keep SS asserted. + */ + temp |= TCR_CONT; + if (is_first_xfer) + temp &= ~TCR_CONTC; + else + temp |= TCR_CONTC; + + writel(temp, fsl_lpspi->base + IMX7ULP_TCR); + + dev_dbg(fsl_lpspi->dev, "TCR=0x%x\n", temp); +} + +static void fsl_lpspi_set_watermark(struct fsl_lpspi_data *fsl_lpspi) +{ + u32 temp; + + temp = fsl_lpspi->txfifosize >> 1 | (fsl_lpspi->rxfifosize >> 1) << 16; + + writel(temp, fsl_lpspi->base + IMX7ULP_FCR); + + dev_dbg(fsl_lpspi->dev, "FCR=0x%x\n", temp); +} + +static int fsl_lpspi_set_bitrate(struct fsl_lpspi_data *fsl_lpspi) +{ + struct lpspi_config config = fsl_lpspi->config; + unsigned int perclk_rate, scldiv; + u8 prescale; + + perclk_rate = clk_get_rate(fsl_lpspi->clk); + for (prescale = 0; prescale < 8; prescale++) { + scldiv = perclk_rate / + (clkdivs[prescale] * config.speed_hz) - 2; + if (scldiv < 256) { + fsl_lpspi->config.prescale = prescale; + break; + } + } + + if (prescale == 8 && scldiv >= 256) + return -EINVAL; + + writel(scldiv, fsl_lpspi->base + IMX7ULP_CCR); + + dev_dbg(fsl_lpspi->dev, "perclk=%d, speed=%d, prescale =%d, scldiv=%d\n", + perclk_rate, config.speed_hz, prescale, scldiv); + + return 0; +} + +static int fsl_lpspi_config(struct fsl_lpspi_data *fsl_lpspi) +{ + u32 temp; + int ret; + + temp = CR_RST; + writel(temp, fsl_lpspi->base + IMX7ULP_CR); + writel(0, fsl_lpspi->base + IMX7ULP_CR); + + ret = fsl_lpspi_set_bitrate(fsl_lpspi); + if (ret) + return ret; + + fsl_lpspi_set_watermark(fsl_lpspi); + + temp = CFGR1_PCSCFG | CFGR1_MASTER | CFGR1_NOSTALL; + if (fsl_lpspi->config.mode & SPI_CS_HIGH) + temp |= CFGR1_PCSPOL; + writel(temp, fsl_lpspi->base + IMX7ULP_CFGR1); + + temp = readl(fsl_lpspi->base + IMX7ULP_CR); + temp |= CR_RRF | CR_RTF | CR_MEN; + writel(temp, fsl_lpspi->base + IMX7ULP_CR); + + return 0; +} + +static void fsl_lpspi_setup_transfer(struct spi_device *spi, + struct spi_transfer *t) +{ + struct fsl_lpspi_data *fsl_lpspi = spi_master_get_devdata(spi->master); + + fsl_lpspi->config.mode = spi->mode; + fsl_lpspi->config.bpw = t ? t->bits_per_word : spi->bits_per_word; + fsl_lpspi->config.speed_hz = t ? t->speed_hz : spi->max_speed_hz; + fsl_lpspi->config.chip_select = spi->chip_select; + + if (!fsl_lpspi->config.speed_hz) + fsl_lpspi->config.speed_hz = spi->max_speed_hz; + if (!fsl_lpspi->config.bpw) + fsl_lpspi->config.bpw = spi->bits_per_word; + + /* Initialize the functions for transfer */ + if (fsl_lpspi->config.bpw <= 8) { + fsl_lpspi->rx = fsl_lpspi_buf_rx_u8; + fsl_lpspi->tx = fsl_lpspi_buf_tx_u8; + } else if (fsl_lpspi->config.bpw <= 16) { + fsl_lpspi->rx = fsl_lpspi_buf_rx_u16; + fsl_lpspi->tx = fsl_lpspi_buf_tx_u16; + } else { + fsl_lpspi->rx = fsl_lpspi_buf_rx_u32; + fsl_lpspi->tx = fsl_lpspi_buf_tx_u32; + } + + fsl_lpspi_config(fsl_lpspi); +} + +static int fsl_lpspi_transfer_one(struct spi_master *master, + struct spi_device *spi, + struct spi_transfer *t) +{ + struct fsl_lpspi_data *fsl_lpspi = spi_master_get_devdata(master); + int ret; + + fsl_lpspi->tx_buf = t->tx_buf; + fsl_lpspi->rx_buf = t->rx_buf; + fsl_lpspi->remain = t->len; + + reinit_completion(&fsl_lpspi->xfer_done); + fsl_lpspi_write_tx_fifo(fsl_lpspi); + + ret = wait_for_completion_timeout(&fsl_lpspi->xfer_done, HZ); + if (!ret) { + dev_dbg(fsl_lpspi->dev, "wait for completion timeout\n"); + return -ETIMEDOUT; + } + + ret = fsl_lpspi_txfifo_empty(fsl_lpspi); + if (ret) + return ret; + + fsl_lpspi_read_rx_fifo(fsl_lpspi); + + return 0; +} + +static int fsl_lpspi_transfer_one_msg(struct spi_master *master, + struct spi_message *msg) +{ + struct fsl_lpspi_data *fsl_lpspi = spi_master_get_devdata(master); + struct spi_device *spi = msg->spi; + struct spi_transfer *xfer; + bool is_first_xfer = true; + u32 temp; + int ret; + + msg->status = 0; + msg->actual_length = 0; + + list_for_each_entry(xfer, &msg->transfers, transfer_list) { + fsl_lpspi_setup_transfer(spi, xfer); + fsl_lpspi_set_cmd(fsl_lpspi, is_first_xfer); + + is_first_xfer = false; + + ret = fsl_lpspi_transfer_one(master, spi, xfer); + if (ret < 0) + goto complete; + + msg->actual_length += xfer->len; + } + +complete: + /* de-assert SS, then finalize current message */ + temp = readl(fsl_lpspi->base + IMX7ULP_TCR); + temp &= ~TCR_CONTC; + writel(temp, fsl_lpspi->base + IMX7ULP_TCR); + + msg->status = ret; + spi_finalize_current_message(master); + + return ret; +} + +static irqreturn_t fsl_lpspi_isr(int irq, void *dev_id) +{ + struct fsl_lpspi_data *fsl_lpspi = dev_id; + u32 temp; + + fsl_lpspi_intctrl(fsl_lpspi, 0); + temp = readl(fsl_lpspi->base + IMX7ULP_SR); + + fsl_lpspi_read_rx_fifo(fsl_lpspi); + + if (temp & SR_TDF) { + fsl_lpspi_write_tx_fifo(fsl_lpspi); + + if (!fsl_lpspi->remain) + complete(&fsl_lpspi->xfer_done); + + return IRQ_HANDLED; + } + + return IRQ_NONE; +} + +static int fsl_lpspi_probe(struct platform_device *pdev) +{ + struct fsl_lpspi_data *fsl_lpspi; + struct spi_master *master; + struct resource *res; + int ret, irq; + u32 temp; + + master = spi_alloc_master(&pdev->dev, sizeof(struct fsl_lpspi_data)); + if (!master) + return -ENOMEM; + + platform_set_drvdata(pdev, master); + + master->bits_per_word_mask = SPI_BPW_RANGE_MASK(8, 32); + master->bus_num = pdev->id; + + fsl_lpspi = spi_master_get_devdata(master); + fsl_lpspi->dev = &pdev->dev; + + master->transfer_one_message = fsl_lpspi_transfer_one_msg; + master->prepare_transfer_hardware = lpspi_prepare_xfer_hardware; + master->unprepare_transfer_hardware = lpspi_unprepare_xfer_hardware; + master->mode_bits = SPI_CPOL | SPI_CPHA | SPI_CS_HIGH; + master->flags = SPI_MASTER_MUST_RX | SPI_MASTER_MUST_TX; + master->dev.of_node = pdev->dev.of_node; + master->bus_num = pdev->id; + + init_completion(&fsl_lpspi->xfer_done); + + res = platform_get_resource(pdev, IORESOURCE_MEM, 0); + fsl_lpspi->base = devm_ioremap_resource(&pdev->dev, res); + if (IS_ERR(fsl_lpspi->base)) { + ret = PTR_ERR(fsl_lpspi->base); + goto out_master_put; + } + + irq = platform_get_irq(pdev, 0); + if (irq < 0) { + ret = irq; + goto out_master_put; + } + + ret = devm_request_irq(&pdev->dev, irq, fsl_lpspi_isr, 0, + dev_name(&pdev->dev), fsl_lpspi); + if (ret) { + dev_err(&pdev->dev, "can't get irq%d: %d\n", irq, ret); + goto out_master_put; + } + + fsl_lpspi->clk = devm_clk_get(&pdev->dev, "ipg"); + if (IS_ERR(fsl_lpspi->clk)) { + ret = PTR_ERR(fsl_lpspi->clk); + goto out_master_put; + } + + ret = clk_prepare_enable(fsl_lpspi->clk); + if (ret) { + dev_err(&pdev->dev, "can't enable lpspi clock, ret=%d\n", ret); + goto out_master_put; + } + + temp = readl(fsl_lpspi->base + IMX7ULP_PARAM); + fsl_lpspi->txfifosize = 1 << (temp & 0x0f); + fsl_lpspi->rxfifosize = 1 << ((temp >> 8) & 0x0f); + + clk_disable_unprepare(fsl_lpspi->clk); + + ret = devm_spi_register_master(&pdev->dev, master); + if (ret < 0) { + dev_err(&pdev->dev, "spi_register_master error.\n"); + goto out_master_put; + } + + return 0; + +out_master_put: + spi_master_put(master); + + return ret; +} + +static int fsl_lpspi_remove(struct platform_device *pdev) +{ + struct spi_master *master = platform_get_drvdata(pdev); + struct fsl_lpspi_data *fsl_lpspi = spi_master_get_devdata(master); + + clk_disable_unprepare(fsl_lpspi->clk); + + return 0; +} + +static struct platform_driver fsl_lpspi_driver = { + .driver = { + .name = DRIVER_NAME, + .of_match_table = fsl_lpspi_dt_ids, + }, + .probe = fsl_lpspi_probe, + .remove = fsl_lpspi_remove, +}; +module_platform_driver(fsl_lpspi_driver); + +MODULE_DESCRIPTION("LPSPI Master Controller driver"); +MODULE_AUTHOR("Gao Pan <pandy.gao@nxp.com>"); +MODULE_LICENSE("GPL"); diff --git a/drivers/spi/spi-imx.c b/drivers/spi/spi-imx.c index deb782f..32ced64 100644 --- a/drivers/spi/spi-imx.c +++ b/drivers/spi/spi-imx.c @@ -173,15 +173,16 @@ static int mxc_clkdivs[] = {0, 3, 4, 6, 8, 12, 16, 24, 32, 48, 64, 96, 128, 192, /* MX21, MX27 */ static unsigned int spi_imx_clkdiv_1(unsigned int fin, - unsigned int fspi, unsigned int max) + unsigned int fspi, unsigned int max, unsigned int *fres) { int i; for (i = 2; i < max; i++) if (fspi * mxc_clkdivs[i] >= fin) - return i; + break; - return max; + *fres = fin / mxc_clkdivs[i]; + return i; } /* MX1, MX31, MX35, MX51 CSPI */ @@ -442,6 +443,7 @@ static void mx51_ecspi_reset(struct spi_imx_data *spi_imx) #define MX31_CSPICTRL_ENABLE (1 << 0) #define MX31_CSPICTRL_MASTER (1 << 1) #define MX31_CSPICTRL_XCH (1 << 2) +#define MX31_CSPICTRL_SMC (1 << 3) #define MX31_CSPICTRL_POL (1 << 4) #define MX31_CSPICTRL_PHA (1 << 5) #define MX31_CSPICTRL_SSCTL (1 << 6) @@ -452,6 +454,10 @@ static void mx51_ecspi_reset(struct spi_imx_data *spi_imx) #define MX35_CSPICTRL_CS_SHIFT 12 #define MX31_CSPICTRL_DR_SHIFT 16 +#define MX31_CSPI_DMAREG 0x10 +#define MX31_DMAREG_RH_DEN (1<<4) +#define MX31_DMAREG_TH_DEN (1<<1) + #define MX31_CSPISTATUS 0x14 #define MX31_STATUS_RR (1 << 3) @@ -511,6 +517,9 @@ static int mx31_config(struct spi_device *spi, struct spi_imx_config *config) (is_imx35_cspi(spi_imx) ? MX35_CSPICTRL_CS_SHIFT : MX31_CSPICTRL_CS_SHIFT); + if (spi_imx->usedma) + reg |= MX31_CSPICTRL_SMC; + writel(reg, spi_imx->base + MXC_CSPICTRL); reg = readl(spi_imx->base + MX31_CSPI_TESTREG); @@ -520,6 +529,13 @@ static int mx31_config(struct spi_device *spi, struct spi_imx_config *config) reg &= ~MX31_TEST_LBC; writel(reg, spi_imx->base + MX31_CSPI_TESTREG); + if (spi_imx->usedma) { + /* configure DMA requests when RXFIFO is half full and + when TXFIFO is half empty */ + writel(MX31_DMAREG_RH_DEN | MX31_DMAREG_TH_DEN, + spi_imx->base + MX31_CSPI_DMAREG); + } + return 0; } @@ -574,9 +590,12 @@ static int mx21_config(struct spi_device *spi, struct spi_imx_config *config) struct spi_imx_data *spi_imx = spi_master_get_devdata(spi->master); unsigned int reg = MX21_CSPICTRL_ENABLE | MX21_CSPICTRL_MASTER; unsigned int max = is_imx27_cspi(spi_imx) ? 16 : 18; + unsigned int clk; + + reg |= spi_imx_clkdiv_1(spi_imx->spi_clk, config->speed_hz, max, &clk) + << MX21_CSPICTRL_DR_SHIFT; + spi_imx->spi_bus_clk = clk; - reg |= spi_imx_clkdiv_1(spi_imx->spi_clk, config->speed_hz, max) << - MX21_CSPICTRL_DR_SHIFT; reg |= config->bpw - 1; if (spi->mode & SPI_CPHA) @@ -1244,10 +1263,10 @@ static int spi_imx_probe(struct platform_device *pdev) spi_imx->spi_clk = clk_get_rate(spi_imx->clk_per); /* - * Only validated on i.mx6 now, can remove the constrain if validated on - * other chips. + * Only validated on i.mx35 and i.mx6 now, can remove the constraint + * if validated on other chips. */ - if (is_imx51_ecspi(spi_imx)) { + if (is_imx35_cspi(spi_imx) || is_imx51_ecspi(spi_imx)) { ret = spi_imx_sdma_init(&pdev->dev, spi_imx, master); if (ret == -EPROBE_DEFER) goto out_clk_put; diff --git a/drivers/spi/spi-jcore.c b/drivers/spi/spi-jcore.c index f8117b8..cebfea5 100644 --- a/drivers/spi/spi-jcore.c +++ b/drivers/spi/spi-jcore.c @@ -214,6 +214,7 @@ static const struct of_device_id jcore_spi_of_match[] = { { .compatible = "jcore,spi2" }, {}, }; +MODULE_DEVICE_TABLE(of, jcore_spi_of_match); static struct platform_driver jcore_spi_driver = { .probe = jcore_spi_probe, diff --git a/drivers/spi/spi-omap2-mcspi.c b/drivers/spi/spi-omap2-mcspi.c index d5157b2..79800e9 100644 --- a/drivers/spi/spi-omap2-mcspi.c +++ b/drivers/spi/spi-omap2-mcspi.c @@ -1386,20 +1386,13 @@ static int omap2_mcspi_probe(struct platform_device *pdev) regs_offset = pdata->regs_offset; r = platform_get_resource(pdev, IORESOURCE_MEM, 0); - if (r == NULL) { - status = -ENODEV; - goto free_master; - } - - r->start += regs_offset; - r->end += regs_offset; - mcspi->phys = r->start; - mcspi->base = devm_ioremap_resource(&pdev->dev, r); if (IS_ERR(mcspi->base)) { status = PTR_ERR(mcspi->base); goto free_master; } + mcspi->phys = r->start + regs_offset; + mcspi->base += regs_offset; mcspi->dev = &pdev->dev; diff --git a/drivers/spi/spi-orion.c b/drivers/spi/spi-orion.c index ded3702..6b001c4 100644 --- a/drivers/spi/spi-orion.c +++ b/drivers/spi/spi-orion.c @@ -138,37 +138,62 @@ static int orion_spi_baudrate_set(struct spi_device *spi, unsigned int speed) tclk_hz = clk_get_rate(orion_spi->clk); if (devdata->typ == ARMADA_SPI) { - unsigned int clk, spr, sppr, sppr2, err; - unsigned int best_spr, best_sppr, best_err; - - best_err = speed; - best_spr = 0; - best_sppr = 0; - - /* Iterate over the valid range looking for best fit */ - for (sppr = 0; sppr < 8; sppr++) { - sppr2 = 0x1 << sppr; - - spr = tclk_hz / sppr2; - spr = DIV_ROUND_UP(spr, speed); - if ((spr == 0) || (spr > 15)) - continue; - - clk = tclk_hz / (spr * sppr2); - err = speed - clk; - - if (err < best_err) { - best_spr = spr; - best_sppr = sppr; - best_err = err; - } - } + /* + * Given the core_clk (tclk_hz) and the target rate (speed) we + * determine the best values for SPR (in [0 .. 15]) and SPPR (in + * [0..7]) such that + * + * core_clk / (SPR * 2 ** SPPR) + * + * is as big as possible but not bigger than speed. + */ - if ((best_sppr == 0) && (best_spr == 0)) - return -EINVAL; + /* best integer divider: */ + unsigned divider = DIV_ROUND_UP(tclk_hz, speed); + unsigned spr, sppr; + + if (divider < 16) { + /* This is the easy case, divider is less than 16 */ + spr = divider; + sppr = 0; + + } else { + unsigned two_pow_sppr; + /* + * Find the highest bit set in divider. This and the + * three next bits define SPR (apart from rounding). + * SPPR is then the number of zero bits that must be + * appended: + */ + sppr = fls(divider) - 4; + + /* + * As SPR only has 4 bits, we have to round divider up + * to the next multiple of 2 ** sppr. + */ + two_pow_sppr = 1 << sppr; + divider = (divider + two_pow_sppr - 1) & -two_pow_sppr; + + /* + * recalculate sppr as rounding up divider might have + * increased it enough to change the position of the + * highest set bit. In this case the bit that now + * doesn't make it into SPR is 0, so there is no need to + * round again. + */ + sppr = fls(divider) - 4; + spr = divider >> sppr; + + /* + * Now do range checking. SPR is constructed to have a + * width of 4 bits, so this is fine for sure. So we + * still need to check for sppr to fit into 3 bits: + */ + if (sppr > 7) + return -EINVAL; + } - prescale = ((best_sppr & 0x6) << 5) | - ((best_sppr & 0x1) << 4) | best_spr; + prescale = ((sppr & 0x6) << 5) | ((sppr & 0x1) << 4) | spr; } else { /* * the supported rates are: 4,6,8...30 diff --git a/drivers/spi/spi-pxa2xx.h b/drivers/spi/spi-pxa2xx.h index ce31b81..2823a00 100644 --- a/drivers/spi/spi-pxa2xx.h +++ b/drivers/spi/spi-pxa2xx.h @@ -109,7 +109,6 @@ static inline void pxa2xx_spi_write(const struct driver_data *drv_data, #define DONE_STATE ((void *)2) #define ERROR_STATE ((void *)-1) -#define IS_DMA_ALIGNED(x) IS_ALIGNED((unsigned long)(x), DMA_ALIGNMENT) #define DMA_ALIGNMENT 8 static inline int pxa25x_ssp_comp(struct driver_data *drv_data) diff --git a/drivers/spi/spi-rspi.c b/drivers/spi/spi-rspi.c index a816f07..9daf500 100644 --- a/drivers/spi/spi-rspi.c +++ b/drivers/spi/spi-rspi.c @@ -413,7 +413,7 @@ static unsigned int qspi_set_send_trigger(struct rspi_data *rspi, return n; } -static void qspi_set_receive_trigger(struct rspi_data *rspi, unsigned int len) +static int qspi_set_receive_trigger(struct rspi_data *rspi, unsigned int len) { unsigned int n; @@ -428,6 +428,7 @@ static void qspi_set_receive_trigger(struct rspi_data *rspi, unsigned int len) qspi_update(rspi, SPBFCR_RXTRG_MASK, SPBFCR_RXTRG_1B, QSPI_SPBFCR); } + return n; } #define set_config_register(spi, n) spi->ops->set_config_register(spi, n) @@ -785,6 +786,9 @@ static int qspi_transfer_out_in(struct rspi_data *rspi, static int qspi_transfer_out(struct rspi_data *rspi, struct spi_transfer *xfer) { + const u8 *tx = xfer->tx_buf; + unsigned int n = xfer->len; + unsigned int i, len; int ret; if (rspi->master->can_dma && __rspi_can_dma(rspi, xfer)) { @@ -793,9 +797,23 @@ static int qspi_transfer_out(struct rspi_data *rspi, struct spi_transfer *xfer) return ret; } - ret = rspi_pio_transfer(rspi, xfer->tx_buf, NULL, xfer->len); - if (ret < 0) - return ret; + while (n > 0) { + len = qspi_set_send_trigger(rspi, n); + if (len == QSPI_BUFFER_SIZE) { + ret = rspi_wait_for_tx_empty(rspi); + if (ret < 0) { + dev_err(&rspi->master->dev, "transmit timeout\n"); + return ret; + } + for (i = 0; i < len; i++) + rspi_write_data(rspi, *tx++); + } else { + ret = rspi_pio_transfer(rspi, tx, NULL, n); + if (ret < 0) + return ret; + } + n -= len; + } /* Wait for the last transmission */ rspi_wait_for_tx_empty(rspi); @@ -805,13 +823,37 @@ static int qspi_transfer_out(struct rspi_data *rspi, struct spi_transfer *xfer) static int qspi_transfer_in(struct rspi_data *rspi, struct spi_transfer *xfer) { + u8 *rx = xfer->rx_buf; + unsigned int n = xfer->len; + unsigned int i, len; + int ret; + if (rspi->master->can_dma && __rspi_can_dma(rspi, xfer)) { int ret = rspi_dma_transfer(rspi, NULL, &xfer->rx_sg); if (ret != -EAGAIN) return ret; } - return rspi_pio_transfer(rspi, NULL, xfer->rx_buf, xfer->len); + while (n > 0) { + len = qspi_set_receive_trigger(rspi, n); + if (len == QSPI_BUFFER_SIZE) { + ret = rspi_wait_for_rx_full(rspi); + if (ret < 0) { + dev_err(&rspi->master->dev, "receive timeout\n"); + return ret; + } + for (i = 0; i < len; i++) + *rx++ = rspi_read_data(rspi); + } else { + ret = rspi_pio_transfer(rspi, NULL, rx, n); + if (ret < 0) + return ret; + *rx++ = ret; + } + n -= len; + } + + return 0; } static int qspi_transfer_one(struct spi_master *master, struct spi_device *spi, diff --git a/drivers/spi/spi-sh-msiof.c b/drivers/spi/spi-sh-msiof.c index 1de3a77..0012ad0 100644 --- a/drivers/spi/spi-sh-msiof.c +++ b/drivers/spi/spi-sh-msiof.c @@ -980,6 +980,7 @@ static const struct of_device_id sh_msiof_match[] = { { .compatible = "renesas,msiof-r8a7792", .data = &r8a779x_data }, { .compatible = "renesas,msiof-r8a7793", .data = &r8a779x_data }, { .compatible = "renesas,msiof-r8a7794", .data = &r8a779x_data }, + { .compatible = "renesas,msiof-r8a7796", .data = &r8a779x_data }, {}, }; MODULE_DEVICE_TABLE(of, sh_msiof_match); diff --git a/drivers/spi/spi-sun4i.c b/drivers/spi/spi-sun4i.c index 4969dc1..c5cd635 100644 --- a/drivers/spi/spi-sun4i.c +++ b/drivers/spi/spi-sun4i.c @@ -46,6 +46,8 @@ #define SUN4I_CTL_TP BIT(18) #define SUN4I_INT_CTL_REG 0x0c +#define SUN4I_INT_CTL_RF_F34 BIT(4) +#define SUN4I_INT_CTL_TF_E34 BIT(12) #define SUN4I_INT_CTL_TC BIT(16) #define SUN4I_INT_STA_REG 0x10 @@ -61,11 +63,14 @@ #define SUN4I_CLK_CTL_CDR1(div) (((div) & SUN4I_CLK_CTL_CDR1_MASK) << 8) #define SUN4I_CLK_CTL_DRS BIT(12) +#define SUN4I_MAX_XFER_SIZE 0xffffff + #define SUN4I_BURST_CNT_REG 0x20 -#define SUN4I_BURST_CNT(cnt) ((cnt) & 0xffffff) +#define SUN4I_BURST_CNT(cnt) ((cnt) & SUN4I_MAX_XFER_SIZE) #define SUN4I_XMIT_CNT_REG 0x24 -#define SUN4I_XMIT_CNT(cnt) ((cnt) & 0xffffff) +#define SUN4I_XMIT_CNT(cnt) ((cnt) & SUN4I_MAX_XFER_SIZE) + #define SUN4I_FIFO_STA_REG 0x28 #define SUN4I_FIFO_STA_RF_CNT_MASK 0x7f @@ -96,6 +101,31 @@ static inline void sun4i_spi_write(struct sun4i_spi *sspi, u32 reg, u32 value) writel(value, sspi->base_addr + reg); } +static inline u32 sun4i_spi_get_tx_fifo_count(struct sun4i_spi *sspi) +{ + u32 reg = sun4i_spi_read(sspi, SUN4I_FIFO_STA_REG); + + reg >>= SUN4I_FIFO_STA_TF_CNT_BITS; + + return reg & SUN4I_FIFO_STA_TF_CNT_MASK; +} + +static inline void sun4i_spi_enable_interrupt(struct sun4i_spi *sspi, u32 mask) +{ + u32 reg = sun4i_spi_read(sspi, SUN4I_INT_CTL_REG); + + reg |= mask; + sun4i_spi_write(sspi, SUN4I_INT_CTL_REG, reg); +} + +static inline void sun4i_spi_disable_interrupt(struct sun4i_spi *sspi, u32 mask) +{ + u32 reg = sun4i_spi_read(sspi, SUN4I_INT_CTL_REG); + + reg &= ~mask; + sun4i_spi_write(sspi, SUN4I_INT_CTL_REG, reg); +} + static inline void sun4i_spi_drain_fifo(struct sun4i_spi *sspi, int len) { u32 reg, cnt; @@ -118,10 +148,13 @@ static inline void sun4i_spi_drain_fifo(struct sun4i_spi *sspi, int len) static inline void sun4i_spi_fill_fifo(struct sun4i_spi *sspi, int len) { + u32 cnt; u8 byte; - if (len > sspi->len) - len = sspi->len; + /* See how much data we can fit */ + cnt = SUN4I_FIFO_DEPTH - sun4i_spi_get_tx_fifo_count(sspi); + + len = min3(len, (int)cnt, sspi->len); while (len--) { byte = sspi->tx_buf ? *sspi->tx_buf++ : 0; @@ -184,10 +217,10 @@ static int sun4i_spi_transfer_one(struct spi_master *master, u32 reg; /* We don't support transfer larger than the FIFO */ - if (tfr->len > SUN4I_FIFO_DEPTH) + if (tfr->len > SUN4I_MAX_XFER_SIZE) return -EMSGSIZE; - if (tfr->tx_buf && tfr->len >= SUN4I_FIFO_DEPTH) + if (tfr->tx_buf && tfr->len >= SUN4I_MAX_XFER_SIZE) return -EMSGSIZE; reinit_completion(&sspi->done); @@ -286,7 +319,11 @@ static int sun4i_spi_transfer_one(struct spi_master *master, sun4i_spi_fill_fifo(sspi, SUN4I_FIFO_DEPTH - 1); /* Enable the interrupts */ - sun4i_spi_write(sspi, SUN4I_INT_CTL_REG, SUN4I_INT_CTL_TC); + sun4i_spi_enable_interrupt(sspi, SUN4I_INT_CTL_TC | + SUN4I_INT_CTL_RF_F34); + /* Only enable Tx FIFO interrupt if we really need it */ + if (tx_len > SUN4I_FIFO_DEPTH) + sun4i_spi_enable_interrupt(sspi, SUN4I_INT_CTL_TF_E34); /* Start the transfer */ reg = sun4i_spi_read(sspi, SUN4I_CTL_REG); @@ -306,7 +343,6 @@ static int sun4i_spi_transfer_one(struct spi_master *master, goto out; } - sun4i_spi_drain_fifo(sspi, SUN4I_FIFO_DEPTH); out: sun4i_spi_write(sspi, SUN4I_INT_CTL_REG, 0); @@ -322,10 +358,33 @@ static irqreturn_t sun4i_spi_handler(int irq, void *dev_id) /* Transfer complete */ if (status & SUN4I_INT_CTL_TC) { sun4i_spi_write(sspi, SUN4I_INT_STA_REG, SUN4I_INT_CTL_TC); + sun4i_spi_drain_fifo(sspi, SUN4I_FIFO_DEPTH); complete(&sspi->done); return IRQ_HANDLED; } + /* Receive FIFO 3/4 full */ + if (status & SUN4I_INT_CTL_RF_F34) { + sun4i_spi_drain_fifo(sspi, SUN4I_FIFO_DEPTH); + /* Only clear the interrupt _after_ draining the FIFO */ + sun4i_spi_write(sspi, SUN4I_INT_STA_REG, SUN4I_INT_CTL_RF_F34); + return IRQ_HANDLED; + } + + /* Transmit FIFO 3/4 empty */ + if (status & SUN4I_INT_CTL_TF_E34) { + sun4i_spi_fill_fifo(sspi, SUN4I_FIFO_DEPTH); + + if (!sspi->len) + /* nothing left to transmit */ + sun4i_spi_disable_interrupt(sspi, SUN4I_INT_CTL_TF_E34); + + /* Only clear the interrupt _after_ re-seeding the FIFO */ + sun4i_spi_write(sspi, SUN4I_INT_STA_REG, SUN4I_INT_CTL_TF_E34); + + return IRQ_HANDLED; + } + return IRQ_NONE; } diff --git a/drivers/spi/spi-sun6i.c b/drivers/spi/spi-sun6i.c index 9918a57..e311483 100644 --- a/drivers/spi/spi-sun6i.c +++ b/drivers/spi/spi-sun6i.c @@ -17,6 +17,7 @@ #include <linux/interrupt.h> #include <linux/io.h> #include <linux/module.h> +#include <linux/of_device.h> #include <linux/platform_device.h> #include <linux/pm_runtime.h> #include <linux/reset.h> @@ -24,6 +25,7 @@ #include <linux/spi/spi.h> #define SUN6I_FIFO_DEPTH 128 +#define SUN8I_FIFO_DEPTH 64 #define SUN6I_GBL_CTL_REG 0x04 #define SUN6I_GBL_CTL_BUS_ENABLE BIT(0) @@ -90,6 +92,7 @@ struct sun6i_spi { const u8 *tx_buf; u8 *rx_buf; int len; + unsigned long fifo_depth; }; static inline u32 sun6i_spi_read(struct sun6i_spi *sspi, u32 reg) @@ -155,7 +158,9 @@ static void sun6i_spi_set_cs(struct spi_device *spi, bool enable) static size_t sun6i_spi_max_transfer_size(struct spi_device *spi) { - return SUN6I_FIFO_DEPTH - 1; + struct sun6i_spi *sspi = spi_master_get_devdata(spi->master); + + return sspi->fifo_depth - 1; } static int sun6i_spi_transfer_one(struct spi_master *master, @@ -170,7 +175,7 @@ static int sun6i_spi_transfer_one(struct spi_master *master, u32 reg; /* We don't support transfer larger than the FIFO */ - if (tfr->len > SUN6I_FIFO_DEPTH) + if (tfr->len > sspi->fifo_depth) return -EINVAL; reinit_completion(&sspi->done); @@ -265,7 +270,7 @@ static int sun6i_spi_transfer_one(struct spi_master *master, SUN6I_BURST_CTL_CNT_STC(tx_len)); /* Fill the TX FIFO */ - sun6i_spi_fill_fifo(sspi, SUN6I_FIFO_DEPTH); + sun6i_spi_fill_fifo(sspi, sspi->fifo_depth); /* Enable the interrupts */ sun6i_spi_write(sspi, SUN6I_INT_CTL_REG, SUN6I_INT_CTL_TC); @@ -288,7 +293,7 @@ static int sun6i_spi_transfer_one(struct spi_master *master, goto out; } - sun6i_spi_drain_fifo(sspi, SUN6I_FIFO_DEPTH); + sun6i_spi_drain_fifo(sspi, sspi->fifo_depth); out: sun6i_spi_write(sspi, SUN6I_INT_CTL_REG, 0); @@ -398,6 +403,8 @@ static int sun6i_spi_probe(struct platform_device *pdev) } sspi->master = master; + sspi->fifo_depth = (unsigned long)of_device_get_match_data(&pdev->dev); + master->max_speed_hz = 100 * 1000 * 1000; master->min_speed_hz = 3 * 1000; master->set_cs = sun6i_spi_set_cs; @@ -470,7 +477,8 @@ static int sun6i_spi_remove(struct platform_device *pdev) } static const struct of_device_id sun6i_spi_match[] = { - { .compatible = "allwinner,sun6i-a31-spi", }, + { .compatible = "allwinner,sun6i-a31-spi", .data = (void *)SUN6I_FIFO_DEPTH }, + { .compatible = "allwinner,sun8i-h3-spi", .data = (void *)SUN8I_FIFO_DEPTH }, {} }; MODULE_DEVICE_TABLE(of, sun6i_spi_match); diff --git a/drivers/spi/spi-ti-qspi.c b/drivers/spi/spi-ti-qspi.c index caeac66..ec6fb09 100644 --- a/drivers/spi/spi-ti-qspi.c +++ b/drivers/spi/spi-ti-qspi.c @@ -411,6 +411,7 @@ static int ti_qspi_dma_xfer(struct ti_qspi *qspi, dma_addr_t dma_dst, tx->callback = ti_qspi_dma_callback; tx->callback_param = qspi; cookie = tx->tx_submit(tx); + reinit_completion(&qspi->transfer_complete); ret = dma_submit_error(cookie); if (ret) { diff --git a/drivers/spi/spi-topcliff-pch.c b/drivers/spi/spi-topcliff-pch.c index c54ee66..fcb9910 100644 --- a/drivers/spi/spi-topcliff-pch.c +++ b/drivers/spi/spi-topcliff-pch.c @@ -1268,11 +1268,8 @@ static void pch_spi_free_resources(struct pch_spi_board_data *board_dat, static int pch_spi_get_resources(struct pch_spi_board_data *board_dat, struct pch_spi_data *data) { - int retval = 0; - dev_dbg(&board_dat->pdev->dev, "%s ENTRY\n", __func__); - /* reset PCH SPI h/w */ pch_spi_reset(data->master); dev_dbg(&board_dat->pdev->dev, @@ -1280,15 +1277,7 @@ static int pch_spi_get_resources(struct pch_spi_board_data *board_dat, dev_dbg(&board_dat->pdev->dev, "%s data->irq_reg_sts=true\n", __func__); - if (retval != 0) { - dev_err(&board_dat->pdev->dev, - "%s FAIL:invoking pch_spi_free_resources\n", __func__); - pch_spi_free_resources(board_dat, data); - } - - dev_dbg(&board_dat->pdev->dev, "%s Return=%d\n", __func__, retval); - - return retval; + return 0; } static void pch_free_dma_buf(struct pch_spi_board_data *board_dat, diff --git a/drivers/spi/spi-xlp.c b/drivers/spi/spi-xlp.c index 4071a72..bea7a93 100644 --- a/drivers/spi/spi-xlp.c +++ b/drivers/spi/spi-xlp.c @@ -451,6 +451,7 @@ static const struct of_device_id xlp_spi_dt_id[] = { { .compatible = "netlogic,xlp832-spi" }, { }, }; +MODULE_DEVICE_TABLE(of, xlp_spi_dt_id); static struct platform_driver xlp_spi_driver = { .probe = xlp_spi_probe, diff --git a/drivers/spi/spi.c b/drivers/spi/spi.c index 838783c..656dd3e 100644 --- a/drivers/spi/spi.c +++ b/drivers/spi/spi.c @@ -697,10 +697,15 @@ static void spi_set_cs(struct spi_device *spi, bool enable) if (spi->mode & SPI_CS_HIGH) enable = !enable; - if (gpio_is_valid(spi->cs_gpio)) + if (gpio_is_valid(spi->cs_gpio)) { gpio_set_value(spi->cs_gpio, !enable); - else if (spi->master->set_cs) + /* Some SPI masters need both GPIO CS & slave_select */ + if ((spi->master->flags & SPI_MASTER_GPIO_SS) && + spi->master->set_cs) + spi->master->set_cs(spi, !enable); + } else if (spi->master->set_cs) { spi->master->set_cs(spi, !enable); + } } #ifdef CONFIG_HAS_DMA @@ -720,6 +725,7 @@ static int spi_map_buf(struct spi_master *master, struct device *dev, int desc_len; int sgs; struct page *vm_page; + struct scatterlist *sg; void *sg_buf; size_t min; int i, ret; @@ -738,6 +744,7 @@ static int spi_map_buf(struct spi_master *master, struct device *dev, if (ret != 0) return ret; + sg = &sgt->sgl[0]; for (i = 0; i < sgs; i++) { if (vmalloced_buf || kmap_buf) { @@ -751,16 +758,17 @@ static int spi_map_buf(struct spi_master *master, struct device *dev, sg_free_table(sgt); return -ENOMEM; } - sg_set_page(&sgt->sgl[i], vm_page, + sg_set_page(sg, vm_page, min, offset_in_page(buf)); } else { min = min_t(size_t, len, desc_len); sg_buf = buf; - sg_set_buf(&sgt->sgl[i], sg_buf, min); + sg_set_buf(sg, sg_buf, min); } buf += min; len -= min; + sg = sg_next(sg); } ret = dma_map_sg(dev, sgt->sgl, sgt->nents, dir); @@ -1034,8 +1042,14 @@ static int spi_transfer_one_message(struct spi_master *master, if (msg->status != -EINPROGRESS) goto out; - if (xfer->delay_usecs) - udelay(xfer->delay_usecs); + if (xfer->delay_usecs) { + u16 us = xfer->delay_usecs; + + if (us <= 10) + udelay(us); + else + usleep_range(us, us + DIV_ROUND_UP(us, 10)); + } if (xfer->cs_change) { if (list_is_last(&xfer->transfer_list, diff --git a/drivers/spi/spidev.c b/drivers/spi/spidev.c index 2e05046..9e2e099 100644 --- a/drivers/spi/spidev.c +++ b/drivers/spi/spidev.c @@ -696,6 +696,7 @@ static struct class *spidev_class; static const struct of_device_id spidev_dt_ids[] = { { .compatible = "rohm,dh2228fv" }, { .compatible = "lineartechnology,ltc2488" }, + { .compatible = "ge,achc" }, {}, }; MODULE_DEVICE_TABLE(of, spidev_dt_ids); diff --git a/drivers/target/target_core_fabric_configfs.c b/drivers/target/target_core_fabric_configfs.c index 31a096a..d8a16ca 100644 --- a/drivers/target/target_core_fabric_configfs.c +++ b/drivers/target/target_core_fabric_configfs.c @@ -137,7 +137,7 @@ static int target_fabric_mappedlun_link( return core_dev_add_initiator_node_lun_acl(se_tpg, lacl, lun, lun_access_ro); } -static int target_fabric_mappedlun_unlink( +static void target_fabric_mappedlun_unlink( struct config_item *lun_acl_ci, struct config_item *lun_ci) { @@ -146,7 +146,7 @@ static int target_fabric_mappedlun_unlink( struct se_lun *lun = container_of(to_config_group(lun_ci), struct se_lun, lun_group); - return core_dev_del_initiator_node_lun_acl(lun, lacl); + core_dev_del_initiator_node_lun_acl(lun, lacl); } static struct se_lun_acl *item_to_lun_acl(struct config_item *item) @@ -669,7 +669,7 @@ out: return ret; } -static int target_fabric_port_unlink( +static void target_fabric_port_unlink( struct config_item *lun_ci, struct config_item *se_dev_ci) { @@ -688,7 +688,6 @@ static int target_fabric_port_unlink( } core_dev_del_lun(se_tpg, lun); - return 0; } static void target_fabric_port_release(struct config_item *item) diff --git a/drivers/target/tcm_fc/tfc_cmd.c b/drivers/target/tcm_fc/tfc_cmd.c index ff5de9a..9af7842 100644 --- a/drivers/target/tcm_fc/tfc_cmd.c +++ b/drivers/target/tcm_fc/tfc_cmd.c @@ -92,7 +92,7 @@ static void ft_free_cmd(struct ft_cmd *cmd) fp = cmd->req_frame; lport = fr_dev(fp); if (fr_seq(fp)) - lport->tt.seq_release(fr_seq(fp)); + fc_seq_release(fr_seq(fp)); fc_frame_free(fp); percpu_ida_free(&sess->se_sess->sess_tag_pool, cmd->se_cmd.map_tag); ft_sess_put(sess); /* undo get from lookup at recv */ @@ -161,11 +161,11 @@ int ft_queue_status(struct se_cmd *se_cmd) /* * Send response. */ - cmd->seq = lport->tt.seq_start_next(cmd->seq); + cmd->seq = fc_seq_start_next(cmd->seq); fc_fill_fc_hdr(fp, FC_RCTL_DD_CMD_STATUS, ep->did, ep->sid, FC_TYPE_FCP, FC_FC_EX_CTX | FC_FC_LAST_SEQ | FC_FC_END_SEQ, 0); - rc = lport->tt.seq_send(lport, cmd->seq, fp); + rc = fc_seq_send(lport, cmd->seq, fp); if (rc) { pr_info_ratelimited("%s: Failed to send response frame %p, " "xid <0x%x>\n", __func__, fp, ep->xid); @@ -177,7 +177,7 @@ int ft_queue_status(struct se_cmd *se_cmd) se_cmd->scsi_status = SAM_STAT_TASK_SET_FULL; return -ENOMEM; } - lport->tt.exch_done(cmd->seq); + fc_exch_done(cmd->seq); /* * Drop the extra ACK_KREF reference taken by target_submit_cmd() * ahead of ft_check_stop_free() -> transport_generic_free_cmd() @@ -221,7 +221,7 @@ int ft_write_pending(struct se_cmd *se_cmd) memset(txrdy, 0, sizeof(*txrdy)); txrdy->ft_burst_len = htonl(se_cmd->data_length); - cmd->seq = lport->tt.seq_start_next(cmd->seq); + cmd->seq = fc_seq_start_next(cmd->seq); fc_fill_fc_hdr(fp, FC_RCTL_DD_DATA_DESC, ep->did, ep->sid, FC_TYPE_FCP, FC_FC_EX_CTX | FC_FC_END_SEQ | FC_FC_SEQ_INIT, 0); @@ -242,7 +242,7 @@ int ft_write_pending(struct se_cmd *se_cmd) cmd->was_ddp_setup = 1; } } - lport->tt.seq_send(lport, cmd->seq, fp); + fc_seq_send(lport, cmd->seq, fp); return 0; } @@ -323,8 +323,8 @@ static void ft_send_resp_status(struct fc_lport *lport, fc_fill_reply_hdr(fp, rx_fp, FC_RCTL_DD_CMD_STATUS, 0); sp = fr_seq(fp); if (sp) { - lport->tt.seq_send(lport, sp, fp); - lport->tt.exch_done(sp); + fc_seq_send(lport, sp, fp); + fc_exch_done(sp); } else { lport->tt.frame_send(lport, fp); } @@ -461,7 +461,7 @@ static void ft_recv_cmd(struct ft_sess *sess, struct fc_frame *fp) cmd->se_cmd.map_tag = tag; cmd->sess = sess; - cmd->seq = lport->tt.seq_assign(lport, fp); + cmd->seq = fc_seq_assign(lport, fp); if (!cmd->seq) { percpu_ida_free(&se_sess->sess_tag_pool, tag); goto busy; @@ -563,7 +563,7 @@ static void ft_send_work(struct work_struct *work) task_attr = TCM_SIMPLE_TAG; } - fc_seq_exch(cmd->seq)->lp->tt.seq_set_resp(cmd->seq, ft_recv_seq, cmd); + fc_seq_set_resp(cmd->seq, ft_recv_seq, cmd); cmd->se_cmd.tag = fc_seq_exch(cmd->seq)->rxid; /* * Use a single se_cmd->cmd_kref as we expect to release se_cmd diff --git a/drivers/target/tcm_fc/tfc_io.c b/drivers/target/tcm_fc/tfc_io.c index 6f7c65a..1eb1f58 100644 --- a/drivers/target/tcm_fc/tfc_io.c +++ b/drivers/target/tcm_fc/tfc_io.c @@ -82,7 +82,7 @@ int ft_queue_data_in(struct se_cmd *se_cmd) ep = fc_seq_exch(cmd->seq); lport = ep->lp; - cmd->seq = lport->tt.seq_start_next(cmd->seq); + cmd->seq = fc_seq_start_next(cmd->seq); remaining = se_cmd->data_length; @@ -174,7 +174,7 @@ int ft_queue_data_in(struct se_cmd *se_cmd) f_ctl |= FC_FC_END_SEQ; fc_fill_fc_hdr(fp, FC_RCTL_DD_SOL_DATA, ep->did, ep->sid, FC_TYPE_FCP, f_ctl, fh_off); - error = lport->tt.seq_send(lport, seq, fp); + error = fc_seq_send(lport, seq, fp); if (error) { pr_info_ratelimited("%s: Failed to send frame %p, " "xid <0x%x>, remaining %zu, " diff --git a/drivers/tty/vt/vt.c b/drivers/tty/vt/vt.c index 6232644..4c10a9d 100644 --- a/drivers/tty/vt/vt.c +++ b/drivers/tty/vt/vt.c @@ -3923,10 +3923,6 @@ void unblank_screen(void) */ static void blank_screen_t(unsigned long dummy) { - if (unlikely(!keventd_up())) { - mod_timer(&console_timer, jiffies + (blankinterval * HZ)); - return; - } blank_timer_expired = 1; schedule_work(&console_work); } diff --git a/drivers/usb/gadget/configfs.c b/drivers/usb/gadget/configfs.c index 3984787..78c4497 100644 --- a/drivers/usb/gadget/configfs.c +++ b/drivers/usb/gadget/configfs.c @@ -408,7 +408,7 @@ out: return ret; } -static int config_usb_cfg_unlink( +static void config_usb_cfg_unlink( struct config_item *usb_cfg_ci, struct config_item *usb_func_ci) { @@ -437,12 +437,11 @@ static int config_usb_cfg_unlink( list_del(&f->list); usb_put_function(f); mutex_unlock(&gi->lock); - return 0; + return; } } mutex_unlock(&gi->lock); WARN(1, "Unable to locate function to unbind\n"); - return 0; } static struct configfs_item_operations gadget_config_item_ops = { @@ -865,7 +864,7 @@ out: return ret; } -static int os_desc_unlink(struct config_item *os_desc_ci, +static void os_desc_unlink(struct config_item *os_desc_ci, struct config_item *usb_cfg_ci) { struct gadget_info *gi = container_of(to_config_group(os_desc_ci), @@ -878,7 +877,6 @@ static int os_desc_unlink(struct config_item *os_desc_ci, cdev->os_desc_config = NULL; WARN_ON(gi->composite.gadget_driver.udc_name); mutex_unlock(&gi->lock); - return 0; } static struct configfs_item_operations os_desc_ops = { diff --git a/drivers/usb/gadget/function/uvc_configfs.c b/drivers/usb/gadget/function/uvc_configfs.c index 31125a4..4e037d2 100644 --- a/drivers/usb/gadget/function/uvc_configfs.c +++ b/drivers/usb/gadget/function/uvc_configfs.c @@ -547,7 +547,7 @@ out: return ret; } -static int uvcg_control_class_drop_link(struct config_item *src, +static void uvcg_control_class_drop_link(struct config_item *src, struct config_item *target) { struct config_item *control, *header; @@ -555,7 +555,6 @@ static int uvcg_control_class_drop_link(struct config_item *src, struct mutex *su_mutex = &src->ci_group->cg_subsys->su_mutex; struct uvc_descriptor_header **class_array; struct uvcg_control_header *target_hdr; - int ret = -EINVAL; mutex_lock(su_mutex); /* for navigating configfs hierarchy */ @@ -569,23 +568,17 @@ static int uvcg_control_class_drop_link(struct config_item *src, mutex_lock(&opts->lock); class_array = uvcg_get_ctl_class_arr(src, opts); - if (!class_array) - goto unlock; - if (opts->refcnt) { - ret = -EBUSY; + if (!class_array || opts->refcnt) goto unlock; - } target_hdr = to_uvcg_control_header(target); --target_hdr->linked; class_array[0] = NULL; - ret = 0; unlock: mutex_unlock(&opts->lock); out: mutex_unlock(su_mutex); - return ret; } static struct configfs_item_operations uvcg_control_class_item_ops = { @@ -777,7 +770,7 @@ out: return ret; } -static int uvcg_streaming_header_drop_link(struct config_item *src, +static void uvcg_streaming_header_drop_link(struct config_item *src, struct config_item *target) { struct mutex *su_mutex = &src->ci_group->cg_subsys->su_mutex; @@ -786,7 +779,6 @@ static int uvcg_streaming_header_drop_link(struct config_item *src, struct uvcg_streaming_header *src_hdr; struct uvcg_format *target_fmt = NULL; struct uvcg_format_ptr *format_ptr, *tmp; - int ret = -EINVAL; src_hdr = to_uvcg_streaming_header(src); mutex_lock(su_mutex); /* for navigating configfs hierarchy */ @@ -811,8 +803,6 @@ static int uvcg_streaming_header_drop_link(struct config_item *src, out: mutex_unlock(&opts->lock); mutex_unlock(su_mutex); - return ret; - } static struct configfs_item_operations uvcg_streaming_header_item_ops = { @@ -2051,7 +2041,7 @@ out: return ret; } -static int uvcg_streaming_class_drop_link(struct config_item *src, +static void uvcg_streaming_class_drop_link(struct config_item *src, struct config_item *target) { struct config_item *streaming, *header; @@ -2059,7 +2049,6 @@ static int uvcg_streaming_class_drop_link(struct config_item *src, struct mutex *su_mutex = &src->ci_group->cg_subsys->su_mutex; struct uvc_descriptor_header ***class_array; struct uvcg_streaming_header *target_hdr; - int ret = -EINVAL; mutex_lock(su_mutex); /* for navigating configfs hierarchy */ @@ -2076,23 +2065,19 @@ static int uvcg_streaming_class_drop_link(struct config_item *src, if (!class_array || !*class_array) goto unlock; - if (opts->refcnt) { - ret = -EBUSY; + if (opts->refcnt) goto unlock; - } target_hdr = to_uvcg_streaming_header(target); --target_hdr->linked; kfree(**class_array); kfree(*class_array); *class_array = NULL; - ret = 0; unlock: mutex_unlock(&opts->lock); out: mutex_unlock(su_mutex); - return ret; } static struct configfs_item_operations uvcg_streaming_class_item_ops = { diff --git a/drivers/video/fbdev/xen-fbfront.c b/drivers/video/fbdev/xen-fbfront.c index 0567d51..d0115a7 100644 --- a/drivers/video/fbdev/xen-fbfront.c +++ b/drivers/video/fbdev/xen-fbfront.c @@ -633,7 +633,6 @@ static void xenfb_backend_changed(struct xenbus_device *dev, enum xenbus_state backend_state) { struct xenfb_info *info = dev_get_drvdata(&dev->dev); - int val; switch (backend_state) { case XenbusStateInitialising: @@ -657,16 +656,12 @@ InitWait: if (dev->state != XenbusStateConnected) goto InitWait; /* no InitWait seen yet, fudge it */ - if (xenbus_scanf(XBT_NIL, info->xbdev->otherend, - "request-update", "%d", &val) < 0) - val = 0; - if (val) + if (xenbus_read_unsigned(info->xbdev->otherend, + "request-update", 0)) info->update_wanted = 1; - if (xenbus_scanf(XBT_NIL, dev->otherend, - "feature-resize", "%d", &val) < 0) - val = 0; - info->feature_resize = val; + info->feature_resize = xenbus_read_unsigned(dev->otherend, + "feature-resize", 0); break; case XenbusStateClosed: diff --git a/drivers/xen/balloon.c b/drivers/xen/balloon.c index e4db19e..db107fa 100644 --- a/drivers/xen/balloon.c +++ b/drivers/xen/balloon.c @@ -180,7 +180,6 @@ static void __balloon_append(struct page *page) static void balloon_append(struct page *page) { __balloon_append(page); - adjust_managed_page_count(page, -1); } /* balloon_retrieve: rescue a page from the balloon, if it is not empty. */ @@ -201,8 +200,6 @@ static struct page *balloon_retrieve(bool require_lowmem) else balloon_stats.balloon_low--; - adjust_managed_page_count(page, 1); - return page; } @@ -478,7 +475,7 @@ static enum bp_state increase_reservation(unsigned long nr_pages) #endif /* Relinquish the page back to the allocator. */ - __free_reserved_page(page); + free_reserved_page(page); } balloon_stats.current_pages += rc; @@ -509,6 +506,7 @@ static enum bp_state decrease_reservation(unsigned long nr_pages, gfp_t gfp) state = BP_EAGAIN; break; } + adjust_managed_page_count(page, -1); scrub_page(page); list_add(&page->lru, &pages); } diff --git a/drivers/xen/events/events_base.c b/drivers/xen/events/events_base.c index adc19ce..fd8e872 100644 --- a/drivers/xen/events/events_base.c +++ b/drivers/xen/events/events_base.c @@ -947,7 +947,7 @@ static int find_virq(unsigned int virq, unsigned int cpu) continue; if (status.status != EVTCHNSTAT_virq) continue; - if (status.u.virq == virq && status.vcpu == cpu) { + if (status.u.virq == virq && status.vcpu == xen_vcpu_nr(cpu)) { rc = port; break; } diff --git a/drivers/xen/gntalloc.c b/drivers/xen/gntalloc.c index 7a47c4c..1bf55a3 100644 --- a/drivers/xen/gntalloc.c +++ b/drivers/xen/gntalloc.c @@ -127,18 +127,21 @@ static int add_grefs(struct ioctl_gntalloc_alloc_gref *op, struct gntalloc_gref *gref, *next; readonly = !(op->flags & GNTALLOC_FLAG_WRITABLE); - rc = -ENOMEM; for (i = 0; i < op->count; i++) { gref = kzalloc(sizeof(*gref), GFP_KERNEL); - if (!gref) + if (!gref) { + rc = -ENOMEM; goto undo; + } list_add_tail(&gref->next_gref, &queue_gref); list_add_tail(&gref->next_file, &queue_file); gref->users = 1; gref->file_index = op->index + i * PAGE_SIZE; gref->page = alloc_page(GFP_KERNEL|__GFP_ZERO); - if (!gref->page) + if (!gref->page) { + rc = -ENOMEM; goto undo; + } /* Grant foreign access to the page. */ rc = gnttab_grant_foreign_access(op->domid, diff --git a/drivers/xen/gntdev.c b/drivers/xen/gntdev.c index bb95212..2ef2b61 100644 --- a/drivers/xen/gntdev.c +++ b/drivers/xen/gntdev.c @@ -1007,7 +1007,7 @@ static int gntdev_mmap(struct file *flip, struct vm_area_struct *vma) vma->vm_ops = &gntdev_vmops; - vma->vm_flags |= VM_DONTEXPAND | VM_DONTDUMP | VM_IO; + vma->vm_flags |= VM_DONTEXPAND | VM_DONTDUMP | VM_MIXEDMAP; if (use_ptemod) vma->vm_flags |= VM_DONTCOPY; diff --git a/drivers/xen/platform-pci.c b/drivers/xen/platform-pci.c index b59c9455..112ce42 100644 --- a/drivers/xen/platform-pci.c +++ b/drivers/xen/platform-pci.c @@ -125,8 +125,4 @@ static struct pci_driver platform_driver = { .id_table = platform_pci_tbl, }; -static int __init platform_pci_init(void) -{ - return pci_register_driver(&platform_driver); -} -device_initcall(platform_pci_init); +builtin_pci_driver(platform_driver); diff --git a/drivers/xen/swiotlb-xen.c b/drivers/xen/swiotlb-xen.c index 87e6035..478fb91 100644 --- a/drivers/xen/swiotlb-xen.c +++ b/drivers/xen/swiotlb-xen.c @@ -405,7 +405,8 @@ dma_addr_t xen_swiotlb_map_page(struct device *dev, struct page *page, */ trace_swiotlb_bounced(dev, dev_addr, size, swiotlb_force); - map = swiotlb_tbl_map_single(dev, start_dma_addr, phys, size, dir); + map = swiotlb_tbl_map_single(dev, start_dma_addr, phys, size, dir, + attrs); if (map == SWIOTLB_MAP_ERROR) return DMA_ERROR_CODE; @@ -416,11 +417,13 @@ dma_addr_t xen_swiotlb_map_page(struct device *dev, struct page *page, /* * Ensure that the address returned is DMA'ble */ - if (!dma_capable(dev, dev_addr, size)) { - swiotlb_tbl_unmap_single(dev, map, size, dir); - dev_addr = 0; - } - return dev_addr; + if (dma_capable(dev, dev_addr, size)) + return dev_addr; + + attrs |= DMA_ATTR_SKIP_CPU_SYNC; + swiotlb_tbl_unmap_single(dev, map, size, dir, attrs); + + return DMA_ERROR_CODE; } EXPORT_SYMBOL_GPL(xen_swiotlb_map_page); @@ -444,7 +447,7 @@ static void xen_unmap_single(struct device *hwdev, dma_addr_t dev_addr, /* NOTE: We use dev_addr here, not paddr! */ if (is_xen_swiotlb_buffer(dev_addr)) { - swiotlb_tbl_unmap_single(hwdev, paddr, size, dir); + swiotlb_tbl_unmap_single(hwdev, paddr, size, dir, attrs); return; } @@ -557,11 +560,12 @@ xen_swiotlb_map_sg_attrs(struct device *hwdev, struct scatterlist *sgl, start_dma_addr, sg_phys(sg), sg->length, - dir); + dir, attrs); if (map == SWIOTLB_MAP_ERROR) { dev_warn(hwdev, "swiotlb buffer is full\n"); /* Don't panic here, we expect map_sg users to do proper error handling. */ + attrs |= DMA_ATTR_SKIP_CPU_SYNC; xen_swiotlb_unmap_sg_attrs(hwdev, sgl, i, dir, attrs); sg_dma_len(sgl) = 0; @@ -648,13 +652,6 @@ xen_swiotlb_sync_sg_for_device(struct device *hwdev, struct scatterlist *sg, } EXPORT_SYMBOL_GPL(xen_swiotlb_sync_sg_for_device); -int -xen_swiotlb_dma_mapping_error(struct device *hwdev, dma_addr_t dma_addr) -{ - return !dma_addr; -} -EXPORT_SYMBOL_GPL(xen_swiotlb_dma_mapping_error); - /* * Return whether the given device DMA address mask can be supported * properly. For example, if your device can only drive the low 24-bits diff --git a/drivers/xen/xen-pciback/xenbus.c b/drivers/xen/xen-pciback/xenbus.c index 5ce878c..3f0aee0 100644 --- a/drivers/xen/xen-pciback/xenbus.c +++ b/drivers/xen/xen-pciback/xenbus.c @@ -362,7 +362,7 @@ static int xen_pcibk_reconfigure(struct xen_pcibk_device *pdev) int err = 0; int num_devs; int domain, bus, slot, func; - int substate; + unsigned int substate; int i, len; char state_str[64]; char dev_str[64]; @@ -395,10 +395,8 @@ static int xen_pcibk_reconfigure(struct xen_pcibk_device *pdev) "configuration"); goto out; } - err = xenbus_scanf(XBT_NIL, pdev->xdev->nodename, state_str, - "%d", &substate); - if (err != 1) - substate = XenbusStateUnknown; + substate = xenbus_read_unsigned(pdev->xdev->nodename, state_str, + XenbusStateUnknown); switch (substate) { case XenbusStateInitialising: diff --git a/drivers/xen/xenbus/xenbus_dev_frontend.c b/drivers/xen/xenbus/xenbus_dev_frontend.c index 1e8be12..6c0ead4 100644 --- a/drivers/xen/xenbus/xenbus_dev_frontend.c +++ b/drivers/xen/xenbus/xenbus_dev_frontend.c @@ -538,6 +538,8 @@ static int xenbus_file_open(struct inode *inode, struct file *filp) nonseekable_open(inode, filp); + filp->f_mode &= ~FMODE_ATOMIC_POS; /* cdev-style semantics */ + u = kzalloc(sizeof(*u), GFP_KERNEL); if (u == NULL) return -ENOMEM; diff --git a/drivers/xen/xenbus/xenbus_probe.c b/drivers/xen/xenbus/xenbus_probe.c index 33a31cf..4bdf654 100644 --- a/drivers/xen/xenbus/xenbus_probe.c +++ b/drivers/xen/xenbus/xenbus_probe.c @@ -702,7 +702,7 @@ device_initcall(xenbus_probe_initcall); */ static int __init xenstored_local_init(void) { - int err = 0; + int err = -ENOMEM; unsigned long page = 0; struct evtchn_alloc_unbound alloc_unbound; @@ -826,7 +826,7 @@ static int __init xenbus_init(void) * Create xenfs mountpoint in /proc for compatibility with * utilities that expect to find "xenbus" under "/proc/xen". */ - proc_mkdir("xen", NULL); + proc_create_mount_point("xen"); #endif out_error: diff --git a/drivers/xen/xenbus/xenbus_probe_backend.c b/drivers/xen/xenbus/xenbus_probe_backend.c index 04f7f85..37929df 100644 --- a/drivers/xen/xenbus/xenbus_probe_backend.c +++ b/drivers/xen/xenbus/xenbus_probe_backend.c @@ -224,13 +224,7 @@ static int read_frontend_details(struct xenbus_device *xendev) int xenbus_dev_is_online(struct xenbus_device *dev) { - int rc, val; - - rc = xenbus_scanf(XBT_NIL, dev->nodename, "online", "%d", &val); - if (rc != 1) - val = 0; /* no online node present */ - - return val; + return !!xenbus_read_unsigned(dev->nodename, "online", 0); } EXPORT_SYMBOL_GPL(xenbus_dev_is_online); diff --git a/drivers/xen/xenbus/xenbus_xs.c b/drivers/xen/xenbus/xenbus_xs.c index 22f7cd7..6afb993 100644 --- a/drivers/xen/xenbus/xenbus_xs.c +++ b/drivers/xen/xenbus/xenbus_xs.c @@ -559,6 +559,21 @@ int xenbus_scanf(struct xenbus_transaction t, } EXPORT_SYMBOL_GPL(xenbus_scanf); +/* Read an (optional) unsigned value. */ +unsigned int xenbus_read_unsigned(const char *dir, const char *node, + unsigned int default_val) +{ + unsigned int val; + int ret; + + ret = xenbus_scanf(XBT_NIL, dir, node, "%u", &val); + if (ret <= 0) + val = default_val; + + return val; +} +EXPORT_SYMBOL_GPL(xenbus_read_unsigned); + /* Single printf and write: returns -errno or 0. */ int xenbus_printf(struct xenbus_transaction t, const char *dir, const char *node, const char *fmt, ...) @@ -672,7 +687,7 @@ static bool xen_strict_xenbus_quirk(void) } static void xs_reset_watches(void) { - int err, supported = 0; + int err; if (!xen_hvm_domain() || xen_initial_domain()) return; @@ -680,9 +695,8 @@ static void xs_reset_watches(void) if (xen_strict_xenbus_quirk()) return; - err = xenbus_scanf(XBT_NIL, "control", - "platform-feature-xs_reset_watches", "%d", &supported); - if (err != 1 || !supported) + if (!xenbus_read_unsigned("control", + "platform-feature-xs_reset_watches", 0)) return; err = xs_error(xs_single(XBT_NIL, XS_RESET_WATCHES, "", NULL)); @@ -55,7 +55,6 @@ config FS_DAX_PMD depends on FS_DAX depends on ZONE_DEVICE depends on TRANSPARENT_HUGEPAGE - depends on BROKEN endif # BLOCK diff --git a/fs/crypto/Kconfig b/fs/crypto/Kconfig index 92348fa..f514978 100644 --- a/fs/crypto/Kconfig +++ b/fs/crypto/Kconfig @@ -8,9 +8,7 @@ config FS_ENCRYPTION select CRYPTO_XTS select CRYPTO_CTS select CRYPTO_CTR - select CRYPTO_SHA256 select KEYS - select ENCRYPTED_KEYS help Enable encryption of files and directories. This feature is similar to ecryptfs, but it is more memory diff --git a/fs/crypto/crypto.c b/fs/crypto/crypto.c index 98f87fe..ac8e4f6 100644 --- a/fs/crypto/crypto.c +++ b/fs/crypto/crypto.c @@ -27,7 +27,7 @@ #include <linux/bio.h> #include <linux/dcache.h> #include <linux/namei.h> -#include <linux/fscrypto.h> +#include "fscrypt_private.h" static unsigned int num_prealloc_crypto_pages = 32; static unsigned int num_prealloc_crypto_ctxs = 128; @@ -63,7 +63,7 @@ void fscrypt_release_ctx(struct fscrypt_ctx *ctx) { unsigned long flags; - if (ctx->flags & FS_WRITE_PATH_FL && ctx->w.bounce_page) { + if (ctx->flags & FS_CTX_HAS_BOUNCE_BUFFER_FL && ctx->w.bounce_page) { mempool_free(ctx->w.bounce_page, fscrypt_bounce_page_pool); ctx->w.bounce_page = NULL; } @@ -88,7 +88,7 @@ EXPORT_SYMBOL(fscrypt_release_ctx); * Return: An allocated and initialized encryption context on success; error * value or NULL otherwise. */ -struct fscrypt_ctx *fscrypt_get_ctx(struct inode *inode, gfp_t gfp_flags) +struct fscrypt_ctx *fscrypt_get_ctx(const struct inode *inode, gfp_t gfp_flags) { struct fscrypt_ctx *ctx = NULL; struct fscrypt_info *ci = inode->i_crypt_info; @@ -121,7 +121,7 @@ struct fscrypt_ctx *fscrypt_get_ctx(struct inode *inode, gfp_t gfp_flags) } else { ctx->flags &= ~FS_CTX_REQUIRES_FREE_ENCRYPT_FL; } - ctx->flags &= ~FS_WRITE_PATH_FL; + ctx->flags &= ~FS_CTX_HAS_BOUNCE_BUFFER_FL; return ctx; } EXPORT_SYMBOL(fscrypt_get_ctx); @@ -146,9 +146,10 @@ typedef enum { FS_ENCRYPT, } fscrypt_direction_t; -static int do_page_crypto(struct inode *inode, - fscrypt_direction_t rw, pgoff_t index, +static int do_page_crypto(const struct inode *inode, + fscrypt_direction_t rw, u64 lblk_num, struct page *src_page, struct page *dest_page, + unsigned int len, unsigned int offs, gfp_t gfp_flags) { struct { @@ -162,6 +163,8 @@ static int do_page_crypto(struct inode *inode, struct crypto_skcipher *tfm = ci->ci_ctfm; int res = 0; + BUG_ON(len == 0); + req = skcipher_request_alloc(tfm, gfp_flags); if (!req) { printk_ratelimited(KERN_ERR @@ -175,14 +178,14 @@ static int do_page_crypto(struct inode *inode, page_crypt_complete, &ecr); BUILD_BUG_ON(sizeof(xts_tweak) != FS_XTS_TWEAK_SIZE); - xts_tweak.index = cpu_to_le64(index); + xts_tweak.index = cpu_to_le64(lblk_num); memset(xts_tweak.padding, 0, sizeof(xts_tweak.padding)); sg_init_table(&dst, 1); - sg_set_page(&dst, dest_page, PAGE_SIZE, 0); + sg_set_page(&dst, dest_page, len, offs); sg_init_table(&src, 1); - sg_set_page(&src, src_page, PAGE_SIZE, 0); - skcipher_request_set_crypt(req, &src, &dst, PAGE_SIZE, &xts_tweak); + sg_set_page(&src, src_page, len, offs); + skcipher_request_set_crypt(req, &src, &dst, len, &xts_tweak); if (rw == FS_DECRYPT) res = crypto_skcipher_decrypt(req); else @@ -207,34 +210,66 @@ static struct page *alloc_bounce_page(struct fscrypt_ctx *ctx, gfp_t gfp_flags) ctx->w.bounce_page = mempool_alloc(fscrypt_bounce_page_pool, gfp_flags); if (ctx->w.bounce_page == NULL) return ERR_PTR(-ENOMEM); - ctx->flags |= FS_WRITE_PATH_FL; + ctx->flags |= FS_CTX_HAS_BOUNCE_BUFFER_FL; return ctx->w.bounce_page; } /** * fscypt_encrypt_page() - Encrypts a page - * @inode: The inode for which the encryption should take place - * @plaintext_page: The page to encrypt. Must be locked. - * @gfp_flags: The gfp flag for memory allocation + * @inode: The inode for which the encryption should take place + * @page: The page to encrypt. Must be locked for bounce-page + * encryption. + * @len: Length of data to encrypt in @page and encrypted + * data in returned page. + * @offs: Offset of data within @page and returned + * page holding encrypted data. + * @lblk_num: Logical block number. This must be unique for multiple + * calls with same inode, except when overwriting + * previously written data. + * @gfp_flags: The gfp flag for memory allocation * - * Allocates a ciphertext page and encrypts plaintext_page into it using the ctx - * encryption context. + * Encrypts @page using the ctx encryption context. Performs encryption + * either in-place or into a newly allocated bounce page. + * Called on the page write path. * - * Called on the page write path. The caller must call + * Bounce page allocation is the default. + * In this case, the contents of @page are encrypted and stored in an + * allocated bounce page. @page has to be locked and the caller must call * fscrypt_restore_control_page() on the returned ciphertext page to * release the bounce buffer and the encryption context. * - * Return: An allocated page with the encrypted content on success. Else, an + * In-place encryption is used by setting the FS_CFLG_OWN_PAGES flag in + * fscrypt_operations. Here, the input-page is returned with its content + * encrypted. + * + * Return: A page with the encrypted content on success. Else, an * error value or NULL. */ -struct page *fscrypt_encrypt_page(struct inode *inode, - struct page *plaintext_page, gfp_t gfp_flags) +struct page *fscrypt_encrypt_page(const struct inode *inode, + struct page *page, + unsigned int len, + unsigned int offs, + u64 lblk_num, gfp_t gfp_flags) + { struct fscrypt_ctx *ctx; - struct page *ciphertext_page = NULL; + struct page *ciphertext_page = page; int err; - BUG_ON(!PageLocked(plaintext_page)); + BUG_ON(len % FS_CRYPTO_BLOCK_SIZE != 0); + + if (inode->i_sb->s_cop->flags & FS_CFLG_OWN_PAGES) { + /* with inplace-encryption we just encrypt the page */ + err = do_page_crypto(inode, FS_ENCRYPT, lblk_num, + page, ciphertext_page, + len, offs, gfp_flags); + if (err) + return ERR_PTR(err); + + return ciphertext_page; + } + + BUG_ON(!PageLocked(page)); ctx = fscrypt_get_ctx(inode, gfp_flags); if (IS_ERR(ctx)) @@ -245,10 +280,10 @@ struct page *fscrypt_encrypt_page(struct inode *inode, if (IS_ERR(ciphertext_page)) goto errout; - ctx->w.control_page = plaintext_page; - err = do_page_crypto(inode, FS_ENCRYPT, plaintext_page->index, - plaintext_page, ciphertext_page, - gfp_flags); + ctx->w.control_page = page; + err = do_page_crypto(inode, FS_ENCRYPT, lblk_num, + page, ciphertext_page, + len, offs, gfp_flags); if (err) { ciphertext_page = ERR_PTR(err); goto errout; @@ -265,8 +300,13 @@ errout: EXPORT_SYMBOL(fscrypt_encrypt_page); /** - * f2crypt_decrypt_page() - Decrypts a page in-place - * @page: The page to decrypt. Must be locked. + * fscrypt_decrypt_page() - Decrypts a page in-place + * @inode: The corresponding inode for the page to decrypt. + * @page: The page to decrypt. Must be locked in case + * it is a writeback page (FS_CFLG_OWN_PAGES unset). + * @len: Number of bytes in @page to be decrypted. + * @offs: Start of data in @page. + * @lblk_num: Logical block number. * * Decrypts page in-place using the ctx encryption context. * @@ -274,16 +314,18 @@ EXPORT_SYMBOL(fscrypt_encrypt_page); * * Return: Zero on success, non-zero otherwise. */ -int fscrypt_decrypt_page(struct page *page) +int fscrypt_decrypt_page(const struct inode *inode, struct page *page, + unsigned int len, unsigned int offs, u64 lblk_num) { - BUG_ON(!PageLocked(page)); + if (!(inode->i_sb->s_cop->flags & FS_CFLG_OWN_PAGES)) + BUG_ON(!PageLocked(page)); - return do_page_crypto(page->mapping->host, - FS_DECRYPT, page->index, page, page, GFP_NOFS); + return do_page_crypto(inode, FS_DECRYPT, lblk_num, page, page, len, + offs, GFP_NOFS); } EXPORT_SYMBOL(fscrypt_decrypt_page); -int fscrypt_zeroout_range(struct inode *inode, pgoff_t lblk, +int fscrypt_zeroout_range(const struct inode *inode, pgoff_t lblk, sector_t pblk, unsigned int len) { struct fscrypt_ctx *ctx; @@ -306,7 +348,7 @@ int fscrypt_zeroout_range(struct inode *inode, pgoff_t lblk, while (len--) { err = do_page_crypto(inode, FS_ENCRYPT, lblk, ZERO_PAGE(0), ciphertext_page, - GFP_NOFS); + PAGE_SIZE, 0, GFP_NOFS); if (err) goto errout; @@ -414,7 +456,8 @@ static void completion_pages(struct work_struct *work) bio_for_each_segment_all(bv, bio, i) { struct page *page = bv->bv_page; - int ret = fscrypt_decrypt_page(page); + int ret = fscrypt_decrypt_page(page->mapping->host, page, + PAGE_SIZE, 0, page->index); if (ret) { WARN_ON_ONCE(1); @@ -482,17 +525,22 @@ static void fscrypt_destroy(void) /** * fscrypt_initialize() - allocate major buffers for fs encryption. + * @cop_flags: fscrypt operations flags * * We only call this when we start accessing encrypted files, since it * results in memory getting allocated that wouldn't otherwise be used. * * Return: Zero on success, non-zero otherwise. */ -int fscrypt_initialize(void) +int fscrypt_initialize(unsigned int cop_flags) { int i, res = -ENOMEM; - if (fscrypt_bounce_page_pool) + /* + * No need to allocate a bounce page pool if there already is one or + * this FS won't use it. + */ + if (cop_flags & FS_CFLG_OWN_PAGES || fscrypt_bounce_page_pool) return 0; mutex_lock(&fscrypt_init_mutex); @@ -521,7 +569,6 @@ fail: mutex_unlock(&fscrypt_init_mutex); return res; } -EXPORT_SYMBOL(fscrypt_initialize); /** * fscrypt_init() - Set up for fs encryption. diff --git a/fs/crypto/fname.c b/fs/crypto/fname.c index 9b774f4..56ad9d1 100644 --- a/fs/crypto/fname.c +++ b/fs/crypto/fname.c @@ -12,7 +12,7 @@ #include <linux/scatterlist.h> #include <linux/ratelimit.h> -#include <linux/fscrypto.h> +#include "fscrypt_private.h" /** * fname_crypt_complete() - completion callback for filename crypto @@ -209,7 +209,7 @@ static int digest_decode(const char *src, int len, char *dst) return cp - dst; } -u32 fscrypt_fname_encrypted_size(struct inode *inode, u32 ilen) +u32 fscrypt_fname_encrypted_size(const struct inode *inode, u32 ilen) { int padding = 32; struct fscrypt_info *ci = inode->i_crypt_info; @@ -227,7 +227,7 @@ EXPORT_SYMBOL(fscrypt_fname_encrypted_size); * Allocates an output buffer that is sufficient for the crypto operation * specified by the context and the direction. */ -int fscrypt_fname_alloc_buffer(struct inode *inode, +int fscrypt_fname_alloc_buffer(const struct inode *inode, u32 ilen, struct fscrypt_str *crypto_str) { unsigned int olen = fscrypt_fname_encrypted_size(inode, ilen); @@ -350,7 +350,7 @@ int fscrypt_setup_filename(struct inode *dir, const struct qstr *iname, fname->disk_name.len = iname->len; return 0; } - ret = get_crypt_info(dir); + ret = fscrypt_get_crypt_info(dir); if (ret && ret != -EOPNOTSUPP) return ret; diff --git a/fs/crypto/fscrypt_private.h b/fs/crypto/fscrypt_private.h new file mode 100644 index 0000000..aeab032 --- /dev/null +++ b/fs/crypto/fscrypt_private.h @@ -0,0 +1,93 @@ +/* + * fscrypt_private.h + * + * Copyright (C) 2015, Google, Inc. + * + * This contains encryption key functions. + * + * Written by Michael Halcrow, Ildar Muslukhov, and Uday Savagaonkar, 2015. + */ + +#ifndef _FSCRYPT_PRIVATE_H +#define _FSCRYPT_PRIVATE_H + +#include <linux/fscrypto.h> + +#define FS_FNAME_CRYPTO_DIGEST_SIZE 32 + +/* Encryption parameters */ +#define FS_XTS_TWEAK_SIZE 16 +#define FS_AES_128_ECB_KEY_SIZE 16 +#define FS_AES_256_GCM_KEY_SIZE 32 +#define FS_AES_256_CBC_KEY_SIZE 32 +#define FS_AES_256_CTS_KEY_SIZE 32 +#define FS_AES_256_XTS_KEY_SIZE 64 +#define FS_MAX_KEY_SIZE 64 + +#define FS_KEY_DESC_PREFIX "fscrypt:" +#define FS_KEY_DESC_PREFIX_SIZE 8 + +#define FS_KEY_DERIVATION_NONCE_SIZE 16 + +/** + * Encryption context for inode + * + * Protector format: + * 1 byte: Protector format (1 = this version) + * 1 byte: File contents encryption mode + * 1 byte: File names encryption mode + * 1 byte: Flags + * 8 bytes: Master Key descriptor + * 16 bytes: Encryption Key derivation nonce + */ +struct fscrypt_context { + u8 format; + u8 contents_encryption_mode; + u8 filenames_encryption_mode; + u8 flags; + u8 master_key_descriptor[FS_KEY_DESCRIPTOR_SIZE]; + u8 nonce[FS_KEY_DERIVATION_NONCE_SIZE]; +} __packed; + +#define FS_ENCRYPTION_CONTEXT_FORMAT_V1 1 + +/* This is passed in from userspace into the kernel keyring */ +struct fscrypt_key { + u32 mode; + u8 raw[FS_MAX_KEY_SIZE]; + u32 size; +} __packed; + +/* + * A pointer to this structure is stored in the file system's in-core + * representation of an inode. + */ +struct fscrypt_info { + u8 ci_data_mode; + u8 ci_filename_mode; + u8 ci_flags; + struct crypto_skcipher *ci_ctfm; + struct key *ci_keyring_key; + u8 ci_master_key[FS_KEY_DESCRIPTOR_SIZE]; +}; + +#define FS_CTX_REQUIRES_FREE_ENCRYPT_FL 0x00000001 +#define FS_CTX_HAS_BOUNCE_BUFFER_FL 0x00000002 + +struct fscrypt_completion_result { + struct completion completion; + int res; +}; + +#define DECLARE_FS_COMPLETION_RESULT(ecr) \ + struct fscrypt_completion_result ecr = { \ + COMPLETION_INITIALIZER((ecr).completion), 0 } + + +/* crypto.c */ +int fscrypt_initialize(unsigned int cop_flags); + +/* keyinfo.c */ +extern int fscrypt_get_crypt_info(struct inode *); + +#endif /* _FSCRYPT_PRIVATE_H */ diff --git a/fs/crypto/keyinfo.c b/fs/crypto/keyinfo.c index 67fb6d8..6eeea1d 100644 --- a/fs/crypto/keyinfo.c +++ b/fs/crypto/keyinfo.c @@ -10,7 +10,7 @@ #include <keys/user-type.h> #include <linux/scatterlist.h> -#include <linux/fscrypto.h> +#include "fscrypt_private.h" static void derive_crypt_complete(struct crypto_async_request *req, int rc) { @@ -178,7 +178,7 @@ static void put_crypt_info(struct fscrypt_info *ci) kmem_cache_free(fscrypt_info_cachep, ci); } -int get_crypt_info(struct inode *inode) +int fscrypt_get_crypt_info(struct inode *inode) { struct fscrypt_info *crypt_info; struct fscrypt_context ctx; @@ -188,7 +188,7 @@ int get_crypt_info(struct inode *inode) u8 *raw_key = NULL; int res; - res = fscrypt_initialize(); + res = fscrypt_initialize(inode->i_sb->s_cop->flags); if (res) return res; @@ -327,7 +327,7 @@ int fscrypt_get_encryption_info(struct inode *inode) (ci->ci_keyring_key->flags & ((1 << KEY_FLAG_INVALIDATED) | (1 << KEY_FLAG_REVOKED) | (1 << KEY_FLAG_DEAD))))) - return get_crypt_info(inode); + return fscrypt_get_crypt_info(inode); return 0; } EXPORT_SYMBOL(fscrypt_get_encryption_info); diff --git a/fs/crypto/policy.c b/fs/crypto/policy.c index 6865663..6ed7c2e 100644 --- a/fs/crypto/policy.c +++ b/fs/crypto/policy.c @@ -10,8 +10,8 @@ #include <linux/random.h> #include <linux/string.h> -#include <linux/fscrypto.h> #include <linux/mount.h> +#include "fscrypt_private.h" static int inode_has_encryption_context(struct inode *inode) { @@ -93,16 +93,19 @@ static int create_encryption_context_from_policy(struct inode *inode, return inode->i_sb->s_cop->set_context(inode, &ctx, sizeof(ctx), NULL); } -int fscrypt_process_policy(struct file *filp, - const struct fscrypt_policy *policy) +int fscrypt_ioctl_set_policy(struct file *filp, const void __user *arg) { + struct fscrypt_policy policy; struct inode *inode = file_inode(filp); int ret; + if (copy_from_user(&policy, arg, sizeof(policy))) + return -EFAULT; + if (!inode_owner_or_capable(inode)) return -EACCES; - if (policy->version != 0) + if (policy.version != 0) return -EINVAL; ret = mnt_want_write_file(filp); @@ -120,9 +123,9 @@ int fscrypt_process_policy(struct file *filp, ret = -ENOTEMPTY; else ret = create_encryption_context_from_policy(inode, - policy); + &policy); } else if (!is_encryption_context_consistent_with_policy(inode, - policy)) { + &policy)) { printk(KERN_WARNING "%s: Policy inconsistent with encryption context\n", __func__); @@ -134,11 +137,13 @@ int fscrypt_process_policy(struct file *filp, mnt_drop_write_file(filp); return ret; } -EXPORT_SYMBOL(fscrypt_process_policy); +EXPORT_SYMBOL(fscrypt_ioctl_set_policy); -int fscrypt_get_policy(struct inode *inode, struct fscrypt_policy *policy) +int fscrypt_ioctl_get_policy(struct file *filp, void __user *arg) { + struct inode *inode = file_inode(filp); struct fscrypt_context ctx; + struct fscrypt_policy policy; int res; if (!inode->i_sb->s_cop->get_context || @@ -151,15 +156,18 @@ int fscrypt_get_policy(struct inode *inode, struct fscrypt_policy *policy) if (ctx.format != FS_ENCRYPTION_CONTEXT_FORMAT_V1) return -EINVAL; - policy->version = 0; - policy->contents_encryption_mode = ctx.contents_encryption_mode; - policy->filenames_encryption_mode = ctx.filenames_encryption_mode; - policy->flags = ctx.flags; - memcpy(&policy->master_key_descriptor, ctx.master_key_descriptor, + policy.version = 0; + policy.contents_encryption_mode = ctx.contents_encryption_mode; + policy.filenames_encryption_mode = ctx.filenames_encryption_mode; + policy.flags = ctx.flags; + memcpy(policy.master_key_descriptor, ctx.master_key_descriptor, FS_KEY_DESCRIPTOR_SIZE); + + if (copy_to_user(arg, &policy, sizeof(policy))) + return -EFAULT; return 0; } -EXPORT_SYMBOL(fscrypt_get_policy); +EXPORT_SYMBOL(fscrypt_ioctl_get_policy); int fscrypt_has_permitted_context(struct inode *parent, struct inode *child) { @@ -34,25 +34,11 @@ #include <linux/iomap.h> #include "internal.h" -/* - * We use lowest available bit in exceptional entry for locking, other two - * bits to determine entry type. In total 3 special bits. - */ -#define RADIX_DAX_SHIFT (RADIX_TREE_EXCEPTIONAL_SHIFT + 3) -#define RADIX_DAX_PTE (1 << (RADIX_TREE_EXCEPTIONAL_SHIFT + 1)) -#define RADIX_DAX_PMD (1 << (RADIX_TREE_EXCEPTIONAL_SHIFT + 2)) -#define RADIX_DAX_TYPE_MASK (RADIX_DAX_PTE | RADIX_DAX_PMD) -#define RADIX_DAX_TYPE(entry) ((unsigned long)entry & RADIX_DAX_TYPE_MASK) -#define RADIX_DAX_SECTOR(entry) (((unsigned long)entry >> RADIX_DAX_SHIFT)) -#define RADIX_DAX_ENTRY(sector, pmd) ((void *)((unsigned long)sector << \ - RADIX_DAX_SHIFT | (pmd ? RADIX_DAX_PMD : RADIX_DAX_PTE) | \ - RADIX_TREE_EXCEPTIONAL_ENTRY)) - /* We choose 4096 entries - same as per-zone page wait tables */ #define DAX_WAIT_TABLE_BITS 12 #define DAX_WAIT_TABLE_ENTRIES (1 << DAX_WAIT_TABLE_BITS) -wait_queue_head_t wait_table[DAX_WAIT_TABLE_ENTRIES]; +static wait_queue_head_t wait_table[DAX_WAIT_TABLE_ENTRIES]; static int __init init_dax_wait_table(void) { @@ -64,14 +50,6 @@ static int __init init_dax_wait_table(void) } fs_initcall(init_dax_wait_table); -static wait_queue_head_t *dax_entry_waitqueue(struct address_space *mapping, - pgoff_t index) -{ - unsigned long hash = hash_long((unsigned long)mapping ^ index, - DAX_WAIT_TABLE_BITS); - return wait_table + hash; -} - static long dax_map_atomic(struct block_device *bdev, struct blk_dax_ctl *dax) { struct request_queue *q = bdev->bd_queue; @@ -98,209 +76,52 @@ static void dax_unmap_atomic(struct block_device *bdev, blk_queue_exit(bdev->bd_queue); } -struct page *read_dax_sector(struct block_device *bdev, sector_t n) +static int dax_is_pmd_entry(void *entry) { - struct page *page = alloc_pages(GFP_KERNEL, 0); - struct blk_dax_ctl dax = { - .size = PAGE_SIZE, - .sector = n & ~((((int) PAGE_SIZE) / 512) - 1), - }; - long rc; - - if (!page) - return ERR_PTR(-ENOMEM); - - rc = dax_map_atomic(bdev, &dax); - if (rc < 0) - return ERR_PTR(rc); - memcpy_from_pmem(page_address(page), dax.addr, PAGE_SIZE); - dax_unmap_atomic(bdev, &dax); - return page; + return (unsigned long)entry & RADIX_DAX_PMD; } -static bool buffer_written(struct buffer_head *bh) +static int dax_is_pte_entry(void *entry) { - return buffer_mapped(bh) && !buffer_unwritten(bh); + return !((unsigned long)entry & RADIX_DAX_PMD); } -/* - * When ext4 encounters a hole, it returns without modifying the buffer_head - * which means that we can't trust b_size. To cope with this, we set b_state - * to 0 before calling get_block and, if any bit is set, we know we can trust - * b_size. Unfortunate, really, since ext4 knows precisely how long a hole is - * and would save us time calling get_block repeatedly. - */ -static bool buffer_size_valid(struct buffer_head *bh) +static int dax_is_zero_entry(void *entry) { - return bh->b_state != 0; + return (unsigned long)entry & RADIX_DAX_HZP; } - -static sector_t to_sector(const struct buffer_head *bh, - const struct inode *inode) +static int dax_is_empty_entry(void *entry) { - sector_t sector = bh->b_blocknr << (inode->i_blkbits - 9); - - return sector; + return (unsigned long)entry & RADIX_DAX_EMPTY; } -static ssize_t dax_io(struct inode *inode, struct iov_iter *iter, - loff_t start, loff_t end, get_block_t get_block, - struct buffer_head *bh) +struct page *read_dax_sector(struct block_device *bdev, sector_t n) { - loff_t pos = start, max = start, bh_max = start; - bool hole = false; - struct block_device *bdev = NULL; - int rw = iov_iter_rw(iter), rc; - long map_len = 0; + struct page *page = alloc_pages(GFP_KERNEL, 0); struct blk_dax_ctl dax = { - .addr = ERR_PTR(-EIO), + .size = PAGE_SIZE, + .sector = n & ~((((int) PAGE_SIZE) / 512) - 1), }; - unsigned blkbits = inode->i_blkbits; - sector_t file_blks = (i_size_read(inode) + (1 << blkbits) - 1) - >> blkbits; - - if (rw == READ) - end = min(end, i_size_read(inode)); - - while (pos < end) { - size_t len; - if (pos == max) { - long page = pos >> PAGE_SHIFT; - sector_t block = page << (PAGE_SHIFT - blkbits); - unsigned first = pos - (block << blkbits); - long size; - - if (pos == bh_max) { - bh->b_size = PAGE_ALIGN(end - pos); - bh->b_state = 0; - rc = get_block(inode, block, bh, rw == WRITE); - if (rc) - break; - if (!buffer_size_valid(bh)) - bh->b_size = 1 << blkbits; - bh_max = pos - first + bh->b_size; - bdev = bh->b_bdev; - /* - * We allow uninitialized buffers for writes - * beyond EOF as those cannot race with faults - */ - WARN_ON_ONCE( - (buffer_new(bh) && block < file_blks) || - (rw == WRITE && buffer_unwritten(bh))); - } else { - unsigned done = bh->b_size - - (bh_max - (pos - first)); - bh->b_blocknr += done >> blkbits; - bh->b_size -= done; - } - - hole = rw == READ && !buffer_written(bh); - if (hole) { - size = bh->b_size - first; - } else { - dax_unmap_atomic(bdev, &dax); - dax.sector = to_sector(bh, inode); - dax.size = bh->b_size; - map_len = dax_map_atomic(bdev, &dax); - if (map_len < 0) { - rc = map_len; - break; - } - dax.addr += first; - size = map_len - first; - } - /* - * pos + size is one past the last offset for IO, - * so pos + size can overflow loff_t at extreme offsets. - * Cast to u64 to catch this and get the true minimum. - */ - max = min_t(u64, pos + size, end); - } - - if (iov_iter_rw(iter) == WRITE) { - len = copy_from_iter_pmem(dax.addr, max - pos, iter); - } else if (!hole) - len = copy_to_iter((void __force *) dax.addr, max - pos, - iter); - else - len = iov_iter_zero(max - pos, iter); - - if (!len) { - rc = -EFAULT; - break; - } + long rc; - pos += len; - if (!IS_ERR(dax.addr)) - dax.addr += len; - } + if (!page) + return ERR_PTR(-ENOMEM); + rc = dax_map_atomic(bdev, &dax); + if (rc < 0) + return ERR_PTR(rc); + memcpy_from_pmem(page_address(page), dax.addr, PAGE_SIZE); dax_unmap_atomic(bdev, &dax); - - return (pos == start) ? rc : pos - start; -} - -/** - * dax_do_io - Perform I/O to a DAX file - * @iocb: The control block for this I/O - * @inode: The file which the I/O is directed at - * @iter: The addresses to do I/O from or to - * @get_block: The filesystem method used to translate file offsets to blocks - * @end_io: A filesystem callback for I/O completion - * @flags: See below - * - * This function uses the same locking scheme as do_blockdev_direct_IO: - * If @flags has DIO_LOCKING set, we assume that the i_mutex is held by the - * caller for writes. For reads, we take and release the i_mutex ourselves. - * If DIO_LOCKING is not set, the filesystem takes care of its own locking. - * As with do_blockdev_direct_IO(), we increment i_dio_count while the I/O - * is in progress. - */ -ssize_t dax_do_io(struct kiocb *iocb, struct inode *inode, - struct iov_iter *iter, get_block_t get_block, - dio_iodone_t end_io, int flags) -{ - struct buffer_head bh; - ssize_t retval = -EINVAL; - loff_t pos = iocb->ki_pos; - loff_t end = pos + iov_iter_count(iter); - - memset(&bh, 0, sizeof(bh)); - bh.b_bdev = inode->i_sb->s_bdev; - - if ((flags & DIO_LOCKING) && iov_iter_rw(iter) == READ) - inode_lock(inode); - - /* Protects against truncate */ - if (!(flags & DIO_SKIP_DIO_COUNT)) - inode_dio_begin(inode); - - retval = dax_io(inode, iter, pos, end, get_block, &bh); - - if ((flags & DIO_LOCKING) && iov_iter_rw(iter) == READ) - inode_unlock(inode); - - if (end_io) { - int err; - - err = end_io(iocb, pos, retval, bh.b_private); - if (err) - retval = err; - } - - if (!(flags & DIO_SKIP_DIO_COUNT)) - inode_dio_end(inode); - return retval; + return page; } -EXPORT_SYMBOL_GPL(dax_do_io); /* * DAX radix tree locking */ struct exceptional_entry_key { struct address_space *mapping; - unsigned long index; + pgoff_t entry_start; }; struct wait_exceptional_entry_queue { @@ -308,6 +129,26 @@ struct wait_exceptional_entry_queue { struct exceptional_entry_key key; }; +static wait_queue_head_t *dax_entry_waitqueue(struct address_space *mapping, + pgoff_t index, void *entry, struct exceptional_entry_key *key) +{ + unsigned long hash; + + /* + * If 'entry' is a PMD, align the 'index' that we use for the wait + * queue to the start of that PMD. This ensures that all offsets in + * the range covered by the PMD map to the same bit lock. + */ + if (dax_is_pmd_entry(entry)) + index &= ~((1UL << (PMD_SHIFT - PAGE_SHIFT)) - 1); + + key->mapping = mapping; + key->entry_start = index; + + hash = hash_long((unsigned long)mapping ^ index, DAX_WAIT_TABLE_BITS); + return wait_table + hash; +} + static int wake_exceptional_entry_func(wait_queue_t *wait, unsigned int mode, int sync, void *keyp) { @@ -316,7 +157,7 @@ static int wake_exceptional_entry_func(wait_queue_t *wait, unsigned int mode, container_of(wait, struct wait_exceptional_entry_queue, wait); if (key->mapping != ewait->key.mapping || - key->index != ewait->key.index) + key->entry_start != ewait->key.entry_start) return 0; return autoremove_wake_function(wait, mode, sync, NULL); } @@ -372,24 +213,24 @@ static inline void *unlock_slot(struct address_space *mapping, void **slot) static void *get_unlocked_mapping_entry(struct address_space *mapping, pgoff_t index, void ***slotp) { - void *ret, **slot; + void *entry, **slot; struct wait_exceptional_entry_queue ewait; - wait_queue_head_t *wq = dax_entry_waitqueue(mapping, index); + wait_queue_head_t *wq; init_wait(&ewait.wait); ewait.wait.func = wake_exceptional_entry_func; - ewait.key.mapping = mapping; - ewait.key.index = index; for (;;) { - ret = __radix_tree_lookup(&mapping->page_tree, index, NULL, + entry = __radix_tree_lookup(&mapping->page_tree, index, NULL, &slot); - if (!ret || !radix_tree_exceptional_entry(ret) || + if (!entry || !radix_tree_exceptional_entry(entry) || !slot_locked(mapping, slot)) { if (slotp) *slotp = slot; - return ret; + return entry; } + + wq = dax_entry_waitqueue(mapping, index, entry, &ewait.key); prepare_to_wait_exclusive(wq, &ewait.wait, TASK_UNINTERRUPTIBLE); spin_unlock_irq(&mapping->tree_lock); @@ -399,52 +240,156 @@ static void *get_unlocked_mapping_entry(struct address_space *mapping, } } +static void put_locked_mapping_entry(struct address_space *mapping, + pgoff_t index, void *entry) +{ + if (!radix_tree_exceptional_entry(entry)) { + unlock_page(entry); + put_page(entry); + } else { + dax_unlock_mapping_entry(mapping, index); + } +} + +/* + * Called when we are done with radix tree entry we looked up via + * get_unlocked_mapping_entry() and which we didn't lock in the end. + */ +static void put_unlocked_mapping_entry(struct address_space *mapping, + pgoff_t index, void *entry) +{ + if (!radix_tree_exceptional_entry(entry)) + return; + + /* We have to wake up next waiter for the radix tree entry lock */ + dax_wake_mapping_entry_waiter(mapping, index, entry, false); +} + /* * Find radix tree entry at given index. If it points to a page, return with * the page locked. If it points to the exceptional entry, return with the * radix tree entry locked. If the radix tree doesn't contain given index, * create empty exceptional entry for the index and return with it locked. * + * When requesting an entry with size RADIX_DAX_PMD, grab_mapping_entry() will + * either return that locked entry or will return an error. This error will + * happen if there are any 4k entries (either zero pages or DAX entries) + * within the 2MiB range that we are requesting. + * + * We always favor 4k entries over 2MiB entries. There isn't a flow where we + * evict 4k entries in order to 'upgrade' them to a 2MiB entry. A 2MiB + * insertion will fail if it finds any 4k entries already in the tree, and a + * 4k insertion will cause an existing 2MiB entry to be unmapped and + * downgraded to 4k entries. This happens for both 2MiB huge zero pages as + * well as 2MiB empty entries. + * + * The exception to this downgrade path is for 2MiB DAX PMD entries that have + * real storage backing them. We will leave these real 2MiB DAX entries in + * the tree, and PTE writes will simply dirty the entire 2MiB DAX entry. + * * Note: Unlike filemap_fault() we don't honor FAULT_FLAG_RETRY flags. For * persistent memory the benefit is doubtful. We can add that later if we can * show it helps. */ -static void *grab_mapping_entry(struct address_space *mapping, pgoff_t index) +static void *grab_mapping_entry(struct address_space *mapping, pgoff_t index, + unsigned long size_flag) { - void *ret, **slot; + bool pmd_downgrade = false; /* splitting 2MiB entry into 4k entries? */ + void *entry, **slot; restart: spin_lock_irq(&mapping->tree_lock); - ret = get_unlocked_mapping_entry(mapping, index, &slot); + entry = get_unlocked_mapping_entry(mapping, index, &slot); + + if (entry) { + if (size_flag & RADIX_DAX_PMD) { + if (!radix_tree_exceptional_entry(entry) || + dax_is_pte_entry(entry)) { + put_unlocked_mapping_entry(mapping, index, + entry); + entry = ERR_PTR(-EEXIST); + goto out_unlock; + } + } else { /* trying to grab a PTE entry */ + if (radix_tree_exceptional_entry(entry) && + dax_is_pmd_entry(entry) && + (dax_is_zero_entry(entry) || + dax_is_empty_entry(entry))) { + pmd_downgrade = true; + } + } + } + /* No entry for given index? Make sure radix tree is big enough. */ - if (!ret) { + if (!entry || pmd_downgrade) { int err; + if (pmd_downgrade) { + /* + * Make sure 'entry' remains valid while we drop + * mapping->tree_lock. + */ + entry = lock_slot(mapping, slot); + } + spin_unlock_irq(&mapping->tree_lock); + /* + * Besides huge zero pages the only other thing that gets + * downgraded are empty entries which don't need to be + * unmapped. + */ + if (pmd_downgrade && dax_is_zero_entry(entry)) + unmap_mapping_range(mapping, + (index << PAGE_SHIFT) & PMD_MASK, PMD_SIZE, 0); + err = radix_tree_preload( mapping_gfp_mask(mapping) & ~__GFP_HIGHMEM); - if (err) + if (err) { + if (pmd_downgrade) + put_locked_mapping_entry(mapping, index, entry); return ERR_PTR(err); - ret = (void *)(RADIX_TREE_EXCEPTIONAL_ENTRY | - RADIX_DAX_ENTRY_LOCK); + } spin_lock_irq(&mapping->tree_lock); - err = radix_tree_insert(&mapping->page_tree, index, ret); + + if (pmd_downgrade) { + radix_tree_delete(&mapping->page_tree, index); + mapping->nrexceptional--; + dax_wake_mapping_entry_waiter(mapping, index, entry, + true); + } + + entry = dax_radix_locked_entry(0, size_flag | RADIX_DAX_EMPTY); + + err = __radix_tree_insert(&mapping->page_tree, index, + dax_radix_order(entry), entry); radix_tree_preload_end(); if (err) { spin_unlock_irq(&mapping->tree_lock); - /* Someone already created the entry? */ - if (err == -EEXIST) + /* + * Someone already created the entry? This is a + * normal failure when inserting PMDs in a range + * that already contains PTEs. In that case we want + * to return -EEXIST immediately. + */ + if (err == -EEXIST && !(size_flag & RADIX_DAX_PMD)) goto restart; + /* + * Our insertion of a DAX PMD entry failed, most + * likely because it collided with a PTE sized entry + * at a different index in the PMD range. We haven't + * inserted anything into the radix tree and have no + * waiters to wake. + */ return ERR_PTR(err); } /* Good, we have inserted empty locked entry into the tree. */ mapping->nrexceptional++; spin_unlock_irq(&mapping->tree_lock); - return ret; + return entry; } /* Normal page in radix tree? */ - if (!radix_tree_exceptional_entry(ret)) { - struct page *page = ret; + if (!radix_tree_exceptional_entry(entry)) { + struct page *page = entry; get_page(page); spin_unlock_irq(&mapping->tree_lock); @@ -457,15 +402,26 @@ restart: } return page; } - ret = lock_slot(mapping, slot); + entry = lock_slot(mapping, slot); + out_unlock: spin_unlock_irq(&mapping->tree_lock); - return ret; + return entry; } +/* + * We do not necessarily hold the mapping->tree_lock when we call this + * function so it is possible that 'entry' is no longer a valid item in the + * radix tree. This is okay because all we really need to do is to find the + * correct waitqueue where tasks might be waiting for that old 'entry' and + * wake them. + */ void dax_wake_mapping_entry_waiter(struct address_space *mapping, - pgoff_t index, bool wake_all) + pgoff_t index, void *entry, bool wake_all) { - wait_queue_head_t *wq = dax_entry_waitqueue(mapping, index); + struct exceptional_entry_key key; + wait_queue_head_t *wq; + + wq = dax_entry_waitqueue(mapping, index, entry, &key); /* * Checking for locked entry and prepare_to_wait_exclusive() happens @@ -473,54 +429,24 @@ void dax_wake_mapping_entry_waiter(struct address_space *mapping, * So at this point all tasks that could have seen our entry locked * must be in the waitqueue and the following check will see them. */ - if (waitqueue_active(wq)) { - struct exceptional_entry_key key; - - key.mapping = mapping; - key.index = index; + if (waitqueue_active(wq)) __wake_up(wq, TASK_NORMAL, wake_all ? 0 : 1, &key); - } } void dax_unlock_mapping_entry(struct address_space *mapping, pgoff_t index) { - void *ret, **slot; + void *entry, **slot; spin_lock_irq(&mapping->tree_lock); - ret = __radix_tree_lookup(&mapping->page_tree, index, NULL, &slot); - if (WARN_ON_ONCE(!ret || !radix_tree_exceptional_entry(ret) || + entry = __radix_tree_lookup(&mapping->page_tree, index, NULL, &slot); + if (WARN_ON_ONCE(!entry || !radix_tree_exceptional_entry(entry) || !slot_locked(mapping, slot))) { spin_unlock_irq(&mapping->tree_lock); return; } unlock_slot(mapping, slot); spin_unlock_irq(&mapping->tree_lock); - dax_wake_mapping_entry_waiter(mapping, index, false); -} - -static void put_locked_mapping_entry(struct address_space *mapping, - pgoff_t index, void *entry) -{ - if (!radix_tree_exceptional_entry(entry)) { - unlock_page(entry); - put_page(entry); - } else { - dax_unlock_mapping_entry(mapping, index); - } -} - -/* - * Called when we are done with radix tree entry we looked up via - * get_unlocked_mapping_entry() and which we didn't lock in the end. - */ -static void put_unlocked_mapping_entry(struct address_space *mapping, - pgoff_t index, void *entry) -{ - if (!radix_tree_exceptional_entry(entry)) - return; - - /* We have to wake up next waiter for the radix tree entry lock */ - dax_wake_mapping_entry_waiter(mapping, index, false); + dax_wake_mapping_entry_waiter(mapping, index, entry, false); } /* @@ -547,7 +473,7 @@ int dax_delete_mapping_entry(struct address_space *mapping, pgoff_t index) radix_tree_delete(&mapping->page_tree, index); mapping->nrexceptional--; spin_unlock_irq(&mapping->tree_lock); - dax_wake_mapping_entry_waiter(mapping, index, true); + dax_wake_mapping_entry_waiter(mapping, index, entry, true); return 1; } @@ -600,11 +526,17 @@ static int copy_user_dax(struct block_device *bdev, sector_t sector, size_t size return 0; } -#define DAX_PMD_INDEX(page_index) (page_index & (PMD_MASK >> PAGE_SHIFT)) - +/* + * By this point grab_mapping_entry() has ensured that we have a locked entry + * of the appropriate size so we don't have to worry about downgrading PMDs to + * PTEs. If we happen to be trying to insert a PTE and there is a PMD + * already in the tree, we will skip the insertion and just dirty the PMD as + * appropriate. + */ static void *dax_insert_mapping_entry(struct address_space *mapping, struct vm_fault *vmf, - void *entry, sector_t sector) + void *entry, sector_t sector, + unsigned long flags) { struct radix_tree_root *page_tree = &mapping->page_tree; int error = 0; @@ -627,22 +559,35 @@ static void *dax_insert_mapping_entry(struct address_space *mapping, error = radix_tree_preload(vmf->gfp_mask & ~__GFP_HIGHMEM); if (error) return ERR_PTR(error); + } else if (dax_is_zero_entry(entry) && !(flags & RADIX_DAX_HZP)) { + /* replacing huge zero page with PMD block mapping */ + unmap_mapping_range(mapping, + (vmf->pgoff << PAGE_SHIFT) & PMD_MASK, PMD_SIZE, 0); } spin_lock_irq(&mapping->tree_lock); - new_entry = (void *)((unsigned long)RADIX_DAX_ENTRY(sector, false) | - RADIX_DAX_ENTRY_LOCK); + new_entry = dax_radix_locked_entry(sector, flags); + if (hole_fill) { __delete_from_page_cache(entry, NULL); /* Drop pagecache reference */ put_page(entry); - error = radix_tree_insert(page_tree, index, new_entry); + error = __radix_tree_insert(page_tree, index, + dax_radix_order(new_entry), new_entry); if (error) { new_entry = ERR_PTR(error); goto unlock; } mapping->nrexceptional++; - } else { + } else if (dax_is_zero_entry(entry) || dax_is_empty_entry(entry)) { + /* + * Only swap our new entry into the radix tree if the current + * entry is a zero page or an empty entry. If a normal PTE or + * PMD entry is already in the tree, we leave it alone. This + * means that if we are trying to insert a PTE and the + * existing entry is a PMD, we will just leave the PMD in the + * tree and dirty it if necessary. + */ struct radix_tree_node *node; void **slot; void *ret; @@ -674,7 +619,6 @@ static int dax_writeback_one(struct block_device *bdev, struct address_space *mapping, pgoff_t index, void *entry) { struct radix_tree_root *page_tree = &mapping->page_tree; - int type = RADIX_DAX_TYPE(entry); struct radix_tree_node *node; struct blk_dax_ctl dax; void **slot; @@ -695,13 +639,21 @@ static int dax_writeback_one(struct block_device *bdev, if (!radix_tree_tag_get(page_tree, index, PAGECACHE_TAG_TOWRITE)) goto unlock; - if (WARN_ON_ONCE(type != RADIX_DAX_PTE && type != RADIX_DAX_PMD)) { + if (WARN_ON_ONCE(dax_is_empty_entry(entry) || + dax_is_zero_entry(entry))) { ret = -EIO; goto unlock; } - dax.sector = RADIX_DAX_SECTOR(entry); - dax.size = (type == RADIX_DAX_PMD ? PMD_SIZE : PAGE_SIZE); + /* + * Even if dax_writeback_mapping_range() was given a wbc->range_start + * in the middle of a PMD, the 'index' we are given will be aligned to + * the start index of the PMD, as will the sector we pull from + * 'entry'. This allows us to flush for PMD_SIZE and not have to + * worry about partial PMD writebacks. + */ + dax.sector = dax_radix_sector(entry); + dax.size = PAGE_SIZE << dax_radix_order(entry); spin_unlock_irq(&mapping->tree_lock); /* @@ -740,12 +692,11 @@ int dax_writeback_mapping_range(struct address_space *mapping, struct block_device *bdev, struct writeback_control *wbc) { struct inode *inode = mapping->host; - pgoff_t start_index, end_index, pmd_index; + pgoff_t start_index, end_index; pgoff_t indices[PAGEVEC_SIZE]; struct pagevec pvec; bool done = false; int i, ret = 0; - void *entry; if (WARN_ON_ONCE(inode->i_blkbits != PAGE_SHIFT)) return -EIO; @@ -755,15 +706,6 @@ int dax_writeback_mapping_range(struct address_space *mapping, start_index = wbc->range_start >> PAGE_SHIFT; end_index = wbc->range_end >> PAGE_SHIFT; - pmd_index = DAX_PMD_INDEX(start_index); - - rcu_read_lock(); - entry = radix_tree_lookup(&mapping->page_tree, pmd_index); - rcu_read_unlock(); - - /* see if the start of our range is covered by a PMD entry */ - if (entry && RADIX_DAX_TYPE(entry) == RADIX_DAX_PMD) - start_index = pmd_index; tag_pages_for_writeback(mapping, start_index, end_index); @@ -808,7 +750,7 @@ static int dax_insert_mapping(struct address_space *mapping, return PTR_ERR(dax.addr); dax_unmap_atomic(bdev, &dax); - ret = dax_insert_mapping_entry(mapping, vmf, entry, dax.sector); + ret = dax_insert_mapping_entry(mapping, vmf, entry, dax.sector, 0); if (IS_ERR(ret)) return PTR_ERR(ret); *entryp = ret; @@ -817,323 +759,6 @@ static int dax_insert_mapping(struct address_space *mapping, } /** - * dax_fault - handle a page fault on a DAX file - * @vma: The virtual memory area where the fault occurred - * @vmf: The description of the fault - * @get_block: The filesystem method used to translate file offsets to blocks - * - * When a page fault occurs, filesystems may call this helper in their - * fault handler for DAX files. dax_fault() assumes the caller has done all - * the necessary locking for the page fault to proceed successfully. - */ -int dax_fault(struct vm_area_struct *vma, struct vm_fault *vmf, - get_block_t get_block) -{ - struct file *file = vma->vm_file; - struct address_space *mapping = file->f_mapping; - struct inode *inode = mapping->host; - void *entry; - struct buffer_head bh; - unsigned long vaddr = (unsigned long)vmf->virtual_address; - unsigned blkbits = inode->i_blkbits; - sector_t block; - pgoff_t size; - int error; - int major = 0; - - /* - * Check whether offset isn't beyond end of file now. Caller is supposed - * to hold locks serializing us with truncate / punch hole so this is - * a reliable test. - */ - size = (i_size_read(inode) + PAGE_SIZE - 1) >> PAGE_SHIFT; - if (vmf->pgoff >= size) - return VM_FAULT_SIGBUS; - - memset(&bh, 0, sizeof(bh)); - block = (sector_t)vmf->pgoff << (PAGE_SHIFT - blkbits); - bh.b_bdev = inode->i_sb->s_bdev; - bh.b_size = PAGE_SIZE; - - entry = grab_mapping_entry(mapping, vmf->pgoff); - if (IS_ERR(entry)) { - error = PTR_ERR(entry); - goto out; - } - - error = get_block(inode, block, &bh, 0); - if (!error && (bh.b_size < PAGE_SIZE)) - error = -EIO; /* fs corruption? */ - if (error) - goto unlock_entry; - - if (vmf->cow_page) { - struct page *new_page = vmf->cow_page; - if (buffer_written(&bh)) - error = copy_user_dax(bh.b_bdev, to_sector(&bh, inode), - bh.b_size, new_page, vaddr); - else - clear_user_highpage(new_page, vaddr); - if (error) - goto unlock_entry; - if (!radix_tree_exceptional_entry(entry)) { - vmf->page = entry; - return VM_FAULT_LOCKED; - } - vmf->entry = entry; - return VM_FAULT_DAX_LOCKED; - } - - if (!buffer_mapped(&bh)) { - if (vmf->flags & FAULT_FLAG_WRITE) { - error = get_block(inode, block, &bh, 1); - count_vm_event(PGMAJFAULT); - mem_cgroup_count_vm_event(vma->vm_mm, PGMAJFAULT); - major = VM_FAULT_MAJOR; - if (!error && (bh.b_size < PAGE_SIZE)) - error = -EIO; - if (error) - goto unlock_entry; - } else { - return dax_load_hole(mapping, entry, vmf); - } - } - - /* Filesystem should not return unwritten buffers to us! */ - WARN_ON_ONCE(buffer_unwritten(&bh) || buffer_new(&bh)); - error = dax_insert_mapping(mapping, bh.b_bdev, to_sector(&bh, inode), - bh.b_size, &entry, vma, vmf); - unlock_entry: - put_locked_mapping_entry(mapping, vmf->pgoff, entry); - out: - if (error == -ENOMEM) - return VM_FAULT_OOM | major; - /* -EBUSY is fine, somebody else faulted on the same PTE */ - if ((error < 0) && (error != -EBUSY)) - return VM_FAULT_SIGBUS | major; - return VM_FAULT_NOPAGE | major; -} -EXPORT_SYMBOL_GPL(dax_fault); - -#if defined(CONFIG_TRANSPARENT_HUGEPAGE) -/* - * The 'colour' (ie low bits) within a PMD of a page offset. This comes up - * more often than one might expect in the below function. - */ -#define PG_PMD_COLOUR ((PMD_SIZE >> PAGE_SHIFT) - 1) - -static void __dax_dbg(struct buffer_head *bh, unsigned long address, - const char *reason, const char *fn) -{ - if (bh) { - char bname[BDEVNAME_SIZE]; - bdevname(bh->b_bdev, bname); - pr_debug("%s: %s addr: %lx dev %s state %lx start %lld " - "length %zd fallback: %s\n", fn, current->comm, - address, bname, bh->b_state, (u64)bh->b_blocknr, - bh->b_size, reason); - } else { - pr_debug("%s: %s addr: %lx fallback: %s\n", fn, - current->comm, address, reason); - } -} - -#define dax_pmd_dbg(bh, address, reason) __dax_dbg(bh, address, reason, "dax_pmd") - -/** - * dax_pmd_fault - handle a PMD fault on a DAX file - * @vma: The virtual memory area where the fault occurred - * @vmf: The description of the fault - * @get_block: The filesystem method used to translate file offsets to blocks - * - * When a page fault occurs, filesystems may call this helper in their - * pmd_fault handler for DAX files. - */ -int dax_pmd_fault(struct vm_area_struct *vma, unsigned long address, - pmd_t *pmd, unsigned int flags, get_block_t get_block) -{ - struct file *file = vma->vm_file; - struct address_space *mapping = file->f_mapping; - struct inode *inode = mapping->host; - struct buffer_head bh; - unsigned blkbits = inode->i_blkbits; - unsigned long pmd_addr = address & PMD_MASK; - bool write = flags & FAULT_FLAG_WRITE; - struct block_device *bdev; - pgoff_t size, pgoff; - sector_t block; - int result = 0; - bool alloc = false; - - /* dax pmd mappings require pfn_t_devmap() */ - if (!IS_ENABLED(CONFIG_FS_DAX_PMD)) - return VM_FAULT_FALLBACK; - - /* Fall back to PTEs if we're going to COW */ - if (write && !(vma->vm_flags & VM_SHARED)) { - split_huge_pmd(vma, pmd, address); - dax_pmd_dbg(NULL, address, "cow write"); - return VM_FAULT_FALLBACK; - } - /* If the PMD would extend outside the VMA */ - if (pmd_addr < vma->vm_start) { - dax_pmd_dbg(NULL, address, "vma start unaligned"); - return VM_FAULT_FALLBACK; - } - if ((pmd_addr + PMD_SIZE) > vma->vm_end) { - dax_pmd_dbg(NULL, address, "vma end unaligned"); - return VM_FAULT_FALLBACK; - } - - pgoff = linear_page_index(vma, pmd_addr); - size = (i_size_read(inode) + PAGE_SIZE - 1) >> PAGE_SHIFT; - if (pgoff >= size) - return VM_FAULT_SIGBUS; - /* If the PMD would cover blocks out of the file */ - if ((pgoff | PG_PMD_COLOUR) >= size) { - dax_pmd_dbg(NULL, address, - "offset + huge page size > file size"); - return VM_FAULT_FALLBACK; - } - - memset(&bh, 0, sizeof(bh)); - bh.b_bdev = inode->i_sb->s_bdev; - block = (sector_t)pgoff << (PAGE_SHIFT - blkbits); - - bh.b_size = PMD_SIZE; - - if (get_block(inode, block, &bh, 0) != 0) - return VM_FAULT_SIGBUS; - - if (!buffer_mapped(&bh) && write) { - if (get_block(inode, block, &bh, 1) != 0) - return VM_FAULT_SIGBUS; - alloc = true; - WARN_ON_ONCE(buffer_unwritten(&bh) || buffer_new(&bh)); - } - - bdev = bh.b_bdev; - - /* - * If the filesystem isn't willing to tell us the length of a hole, - * just fall back to PTEs. Calling get_block 512 times in a loop - * would be silly. - */ - if (!buffer_size_valid(&bh) || bh.b_size < PMD_SIZE) { - dax_pmd_dbg(&bh, address, "allocated block too small"); - return VM_FAULT_FALLBACK; - } - - /* - * If we allocated new storage, make sure no process has any - * zero pages covering this hole - */ - if (alloc) { - loff_t lstart = pgoff << PAGE_SHIFT; - loff_t lend = lstart + PMD_SIZE - 1; /* inclusive */ - - truncate_pagecache_range(inode, lstart, lend); - } - - if (!write && !buffer_mapped(&bh)) { - spinlock_t *ptl; - pmd_t entry; - struct page *zero_page = mm_get_huge_zero_page(vma->vm_mm); - - if (unlikely(!zero_page)) { - dax_pmd_dbg(&bh, address, "no zero page"); - goto fallback; - } - - ptl = pmd_lock(vma->vm_mm, pmd); - if (!pmd_none(*pmd)) { - spin_unlock(ptl); - dax_pmd_dbg(&bh, address, "pmd already present"); - goto fallback; - } - - dev_dbg(part_to_dev(bdev->bd_part), - "%s: %s addr: %lx pfn: <zero> sect: %llx\n", - __func__, current->comm, address, - (unsigned long long) to_sector(&bh, inode)); - - entry = mk_pmd(zero_page, vma->vm_page_prot); - entry = pmd_mkhuge(entry); - set_pmd_at(vma->vm_mm, pmd_addr, pmd, entry); - result = VM_FAULT_NOPAGE; - spin_unlock(ptl); - } else { - struct blk_dax_ctl dax = { - .sector = to_sector(&bh, inode), - .size = PMD_SIZE, - }; - long length = dax_map_atomic(bdev, &dax); - - if (length < 0) { - dax_pmd_dbg(&bh, address, "dax-error fallback"); - goto fallback; - } - if (length < PMD_SIZE) { - dax_pmd_dbg(&bh, address, "dax-length too small"); - dax_unmap_atomic(bdev, &dax); - goto fallback; - } - if (pfn_t_to_pfn(dax.pfn) & PG_PMD_COLOUR) { - dax_pmd_dbg(&bh, address, "pfn unaligned"); - dax_unmap_atomic(bdev, &dax); - goto fallback; - } - - if (!pfn_t_devmap(dax.pfn)) { - dax_unmap_atomic(bdev, &dax); - dax_pmd_dbg(&bh, address, "pfn not in memmap"); - goto fallback; - } - dax_unmap_atomic(bdev, &dax); - - /* - * For PTE faults we insert a radix tree entry for reads, and - * leave it clean. Then on the first write we dirty the radix - * tree entry via the dax_pfn_mkwrite() path. This sequence - * allows the dax_pfn_mkwrite() call to be simpler and avoid a - * call into get_block() to translate the pgoff to a sector in - * order to be able to create a new radix tree entry. - * - * The PMD path doesn't have an equivalent to - * dax_pfn_mkwrite(), though, so for a read followed by a - * write we traverse all the way through dax_pmd_fault() - * twice. This means we can just skip inserting a radix tree - * entry completely on the initial read and just wait until - * the write to insert a dirty entry. - */ - if (write) { - /* - * We should insert radix-tree entry and dirty it here. - * For now this is broken... - */ - } - - dev_dbg(part_to_dev(bdev->bd_part), - "%s: %s addr: %lx pfn: %lx sect: %llx\n", - __func__, current->comm, address, - pfn_t_to_pfn(dax.pfn), - (unsigned long long) dax.sector); - result |= vmf_insert_pfn_pmd(vma, address, pmd, - dax.pfn, write); - } - - out: - return result; - - fallback: - count_vm_event(THP_FAULT_FALLBACK); - result = VM_FAULT_FALLBACK; - goto out; -} -EXPORT_SYMBOL_GPL(dax_pmd_fault); -#endif /* CONFIG_TRANSPARENT_HUGEPAGE */ - -/** * dax_pfn_mkwrite - handle first write to DAX page * @vma: The virtual memory area where the fault occurred * @vmf: The description of the fault @@ -1193,62 +818,14 @@ int __dax_zero_page_range(struct block_device *bdev, sector_t sector, } EXPORT_SYMBOL_GPL(__dax_zero_page_range); -/** - * dax_zero_page_range - zero a range within a page of a DAX file - * @inode: The file being truncated - * @from: The file offset that is being truncated to - * @length: The number of bytes to zero - * @get_block: The filesystem method used to translate file offsets to blocks - * - * This function can be called by a filesystem when it is zeroing part of a - * page in a DAX file. This is intended for hole-punch operations. If - * you are truncating a file, the helper function dax_truncate_page() may be - * more convenient. - */ -int dax_zero_page_range(struct inode *inode, loff_t from, unsigned length, - get_block_t get_block) -{ - struct buffer_head bh; - pgoff_t index = from >> PAGE_SHIFT; - unsigned offset = from & (PAGE_SIZE-1); - int err; - - /* Block boundary? Nothing to do */ - if (!length) - return 0; - BUG_ON((offset + length) > PAGE_SIZE); - - memset(&bh, 0, sizeof(bh)); - bh.b_bdev = inode->i_sb->s_bdev; - bh.b_size = PAGE_SIZE; - err = get_block(inode, index, &bh, 0); - if (err < 0 || !buffer_written(&bh)) - return err; - - return __dax_zero_page_range(bh.b_bdev, to_sector(&bh, inode), - offset, length); -} -EXPORT_SYMBOL_GPL(dax_zero_page_range); - -/** - * dax_truncate_page - handle a partial page being truncated in a DAX file - * @inode: The file being truncated - * @from: The file offset that is being truncated to - * @get_block: The filesystem method used to translate file offsets to blocks - * - * Similar to block_truncate_page(), this function can be called by a - * filesystem when it is truncating a DAX file to handle the partial page. - */ -int dax_truncate_page(struct inode *inode, loff_t from, get_block_t get_block) +#ifdef CONFIG_FS_IOMAP +static sector_t dax_iomap_sector(struct iomap *iomap, loff_t pos) { - unsigned length = PAGE_ALIGN(from) - from; - return dax_zero_page_range(inode, from, length, get_block); + return iomap->blkno + (((pos & PAGE_MASK) - iomap->offset) >> 9); } -EXPORT_SYMBOL_GPL(dax_truncate_page); -#ifdef CONFIG_FS_IOMAP static loff_t -iomap_dax_actor(struct inode *inode, loff_t pos, loff_t length, void *data, +dax_iomap_actor(struct inode *inode, loff_t pos, loff_t length, void *data, struct iomap *iomap) { struct iov_iter *iter = data; @@ -1272,8 +849,7 @@ iomap_dax_actor(struct inode *inode, loff_t pos, loff_t length, void *data, struct blk_dax_ctl dax = { 0 }; ssize_t map_len; - dax.sector = iomap->blkno + - (((pos & PAGE_MASK) - iomap->offset) >> 9); + dax.sector = dax_iomap_sector(iomap, pos); dax.size = (length + offset + PAGE_SIZE - 1) & PAGE_MASK; map_len = dax_map_atomic(iomap->bdev, &dax); if (map_len < 0) { @@ -1305,7 +881,7 @@ iomap_dax_actor(struct inode *inode, loff_t pos, loff_t length, void *data, } /** - * iomap_dax_rw - Perform I/O to a DAX file + * dax_iomap_rw - Perform I/O to a DAX file * @iocb: The control block for this I/O * @iter: The addresses to do I/O from or to * @ops: iomap ops passed from the file system @@ -1315,7 +891,7 @@ iomap_dax_actor(struct inode *inode, loff_t pos, loff_t length, void *data, * and evicting any page cache pages in the region under I/O. */ ssize_t -iomap_dax_rw(struct kiocb *iocb, struct iov_iter *iter, +dax_iomap_rw(struct kiocb *iocb, struct iov_iter *iter, struct iomap_ops *ops) { struct address_space *mapping = iocb->ki_filp->f_mapping; @@ -1345,7 +921,7 @@ iomap_dax_rw(struct kiocb *iocb, struct iov_iter *iter, while (iov_iter_count(iter)) { ret = iomap_apply(inode, pos, iov_iter_count(iter), flags, ops, - iter, iomap_dax_actor); + iter, dax_iomap_actor); if (ret <= 0) break; pos += ret; @@ -1355,10 +931,10 @@ iomap_dax_rw(struct kiocb *iocb, struct iov_iter *iter, iocb->ki_pos += done; return done ? done : ret; } -EXPORT_SYMBOL_GPL(iomap_dax_rw); +EXPORT_SYMBOL_GPL(dax_iomap_rw); /** - * iomap_dax_fault - handle a page fault on a DAX file + * dax_iomap_fault - handle a page fault on a DAX file * @vma: The virtual memory area where the fault occurred * @vmf: The description of the fault * @ops: iomap ops passed from the file system @@ -1367,7 +943,7 @@ EXPORT_SYMBOL_GPL(iomap_dax_rw); * or mkwrite handler for DAX files. Assumes the caller has done all the * necessary locking for the page fault to proceed successfully. */ -int iomap_dax_fault(struct vm_area_struct *vma, struct vm_fault *vmf, +int dax_iomap_fault(struct vm_area_struct *vma, struct vm_fault *vmf, struct iomap_ops *ops) { struct address_space *mapping = vma->vm_file->f_mapping; @@ -1376,8 +952,9 @@ int iomap_dax_fault(struct vm_area_struct *vma, struct vm_fault *vmf, loff_t pos = (loff_t)vmf->pgoff << PAGE_SHIFT; sector_t sector; struct iomap iomap = { 0 }; - unsigned flags = 0; + unsigned flags = IOMAP_FAULT; int error, major = 0; + int locked_status = 0; void *entry; /* @@ -1388,7 +965,7 @@ int iomap_dax_fault(struct vm_area_struct *vma, struct vm_fault *vmf, if (pos >= i_size_read(inode)) return VM_FAULT_SIGBUS; - entry = grab_mapping_entry(mapping, vmf->pgoff); + entry = grab_mapping_entry(mapping, vmf->pgoff, 0); if (IS_ERR(entry)) { error = PTR_ERR(entry); goto out; @@ -1407,10 +984,10 @@ int iomap_dax_fault(struct vm_area_struct *vma, struct vm_fault *vmf, goto unlock_entry; if (WARN_ON_ONCE(iomap.offset + iomap.length < pos + PAGE_SIZE)) { error = -EIO; /* fs corruption? */ - goto unlock_entry; + goto finish_iomap; } - sector = iomap.blkno + (((pos & PAGE_MASK) - iomap.offset) >> 9); + sector = dax_iomap_sector(&iomap, pos); if (vmf->cow_page) { switch (iomap.type) { @@ -1429,13 +1006,15 @@ int iomap_dax_fault(struct vm_area_struct *vma, struct vm_fault *vmf, } if (error) - goto unlock_entry; + goto finish_iomap; if (!radix_tree_exceptional_entry(entry)) { vmf->page = entry; - return VM_FAULT_LOCKED; + locked_status = VM_FAULT_LOCKED; + } else { + vmf->entry = entry; + locked_status = VM_FAULT_DAX_LOCKED; } - vmf->entry = entry; - return VM_FAULT_DAX_LOCKED; + goto finish_iomap; } switch (iomap.type) { @@ -1450,8 +1029,10 @@ int iomap_dax_fault(struct vm_area_struct *vma, struct vm_fault *vmf, break; case IOMAP_UNWRITTEN: case IOMAP_HOLE: - if (!(vmf->flags & FAULT_FLAG_WRITE)) - return dax_load_hole(mapping, entry, vmf); + if (!(vmf->flags & FAULT_FLAG_WRITE)) { + locked_status = dax_load_hole(mapping, entry, vmf); + break; + } /*FALLTHRU*/ default: WARN_ON_ONCE(1); @@ -1459,15 +1040,218 @@ int iomap_dax_fault(struct vm_area_struct *vma, struct vm_fault *vmf, break; } + finish_iomap: + if (ops->iomap_end) { + if (error) { + /* keep previous error */ + ops->iomap_end(inode, pos, PAGE_SIZE, 0, flags, + &iomap); + } else { + error = ops->iomap_end(inode, pos, PAGE_SIZE, + PAGE_SIZE, flags, &iomap); + } + } unlock_entry: - put_locked_mapping_entry(mapping, vmf->pgoff, entry); + if (!locked_status || error) + put_locked_mapping_entry(mapping, vmf->pgoff, entry); out: if (error == -ENOMEM) return VM_FAULT_OOM | major; /* -EBUSY is fine, somebody else faulted on the same PTE */ if (error < 0 && error != -EBUSY) return VM_FAULT_SIGBUS | major; + if (locked_status) { + WARN_ON_ONCE(error); /* -EBUSY from ops->iomap_end? */ + return locked_status; + } return VM_FAULT_NOPAGE | major; } -EXPORT_SYMBOL_GPL(iomap_dax_fault); +EXPORT_SYMBOL_GPL(dax_iomap_fault); + +#ifdef CONFIG_FS_DAX_PMD +/* + * The 'colour' (ie low bits) within a PMD of a page offset. This comes up + * more often than one might expect in the below functions. + */ +#define PG_PMD_COLOUR ((PMD_SIZE >> PAGE_SHIFT) - 1) + +static int dax_pmd_insert_mapping(struct vm_area_struct *vma, pmd_t *pmd, + struct vm_fault *vmf, unsigned long address, + struct iomap *iomap, loff_t pos, bool write, void **entryp) +{ + struct address_space *mapping = vma->vm_file->f_mapping; + struct block_device *bdev = iomap->bdev; + struct blk_dax_ctl dax = { + .sector = dax_iomap_sector(iomap, pos), + .size = PMD_SIZE, + }; + long length = dax_map_atomic(bdev, &dax); + void *ret; + + if (length < 0) /* dax_map_atomic() failed */ + return VM_FAULT_FALLBACK; + if (length < PMD_SIZE) + goto unmap_fallback; + if (pfn_t_to_pfn(dax.pfn) & PG_PMD_COLOUR) + goto unmap_fallback; + if (!pfn_t_devmap(dax.pfn)) + goto unmap_fallback; + + dax_unmap_atomic(bdev, &dax); + + ret = dax_insert_mapping_entry(mapping, vmf, *entryp, dax.sector, + RADIX_DAX_PMD); + if (IS_ERR(ret)) + return VM_FAULT_FALLBACK; + *entryp = ret; + + return vmf_insert_pfn_pmd(vma, address, pmd, dax.pfn, write); + + unmap_fallback: + dax_unmap_atomic(bdev, &dax); + return VM_FAULT_FALLBACK; +} + +static int dax_pmd_load_hole(struct vm_area_struct *vma, pmd_t *pmd, + struct vm_fault *vmf, unsigned long address, + struct iomap *iomap, void **entryp) +{ + struct address_space *mapping = vma->vm_file->f_mapping; + unsigned long pmd_addr = address & PMD_MASK; + struct page *zero_page; + spinlock_t *ptl; + pmd_t pmd_entry; + void *ret; + + zero_page = mm_get_huge_zero_page(vma->vm_mm); + + if (unlikely(!zero_page)) + return VM_FAULT_FALLBACK; + + ret = dax_insert_mapping_entry(mapping, vmf, *entryp, 0, + RADIX_DAX_PMD | RADIX_DAX_HZP); + if (IS_ERR(ret)) + return VM_FAULT_FALLBACK; + *entryp = ret; + + ptl = pmd_lock(vma->vm_mm, pmd); + if (!pmd_none(*pmd)) { + spin_unlock(ptl); + return VM_FAULT_FALLBACK; + } + + pmd_entry = mk_pmd(zero_page, vma->vm_page_prot); + pmd_entry = pmd_mkhuge(pmd_entry); + set_pmd_at(vma->vm_mm, pmd_addr, pmd, pmd_entry); + spin_unlock(ptl); + return VM_FAULT_NOPAGE; +} + +int dax_iomap_pmd_fault(struct vm_area_struct *vma, unsigned long address, + pmd_t *pmd, unsigned int flags, struct iomap_ops *ops) +{ + struct address_space *mapping = vma->vm_file->f_mapping; + unsigned long pmd_addr = address & PMD_MASK; + bool write = flags & FAULT_FLAG_WRITE; + unsigned int iomap_flags = (write ? IOMAP_WRITE : 0) | IOMAP_FAULT; + struct inode *inode = mapping->host; + int result = VM_FAULT_FALLBACK; + struct iomap iomap = { 0 }; + pgoff_t max_pgoff, pgoff; + struct vm_fault vmf; + void *entry; + loff_t pos; + int error; + + /* Fall back to PTEs if we're going to COW */ + if (write && !(vma->vm_flags & VM_SHARED)) + goto fallback; + + /* If the PMD would extend outside the VMA */ + if (pmd_addr < vma->vm_start) + goto fallback; + if ((pmd_addr + PMD_SIZE) > vma->vm_end) + goto fallback; + + /* + * Check whether offset isn't beyond end of file now. Caller is + * supposed to hold locks serializing us with truncate / punch hole so + * this is a reliable test. + */ + pgoff = linear_page_index(vma, pmd_addr); + max_pgoff = (i_size_read(inode) - 1) >> PAGE_SHIFT; + + if (pgoff > max_pgoff) + return VM_FAULT_SIGBUS; + + /* If the PMD would extend beyond the file size */ + if ((pgoff | PG_PMD_COLOUR) > max_pgoff) + goto fallback; + + /* + * grab_mapping_entry() will make sure we get a 2M empty entry, a DAX + * PMD or a HZP entry. If it can't (because a 4k page is already in + * the tree, for instance), it will return -EEXIST and we just fall + * back to 4k entries. + */ + entry = grab_mapping_entry(mapping, pgoff, RADIX_DAX_PMD); + if (IS_ERR(entry)) + goto fallback; + + /* + * Note that we don't use iomap_apply here. We aren't doing I/O, only + * setting up a mapping, so really we're using iomap_begin() as a way + * to look up our filesystem block. + */ + pos = (loff_t)pgoff << PAGE_SHIFT; + error = ops->iomap_begin(inode, pos, PMD_SIZE, iomap_flags, &iomap); + if (error) + goto unlock_entry; + if (iomap.offset + iomap.length < pos + PMD_SIZE) + goto finish_iomap; + + vmf.pgoff = pgoff; + vmf.flags = flags; + vmf.gfp_mask = mapping_gfp_mask(mapping) | __GFP_IO; + + switch (iomap.type) { + case IOMAP_MAPPED: + result = dax_pmd_insert_mapping(vma, pmd, &vmf, address, + &iomap, pos, write, &entry); + break; + case IOMAP_UNWRITTEN: + case IOMAP_HOLE: + if (WARN_ON_ONCE(write)) + goto finish_iomap; + result = dax_pmd_load_hole(vma, pmd, &vmf, address, &iomap, + &entry); + break; + default: + WARN_ON_ONCE(1); + break; + } + + finish_iomap: + if (ops->iomap_end) { + if (result == VM_FAULT_FALLBACK) { + ops->iomap_end(inode, pos, PMD_SIZE, 0, iomap_flags, + &iomap); + } else { + error = ops->iomap_end(inode, pos, PMD_SIZE, PMD_SIZE, + iomap_flags, &iomap); + if (error) + result = VM_FAULT_FALLBACK; + } + } + unlock_entry: + put_locked_mapping_entry(mapping, pgoff, entry); + fallback: + if (result == VM_FAULT_FALLBACK) { + split_huge_pmd(vma, pmd, address); + count_vm_event(THP_FAULT_FALLBACK); + } + return result; +} +EXPORT_SYMBOL_GPL(dax_iomap_pmd_fault); +#endif /* CONFIG_FS_DAX_PMD */ #endif /* CONFIG_FS_IOMAP */ diff --git a/fs/dlm/ast.c b/fs/dlm/ast.c index dcea1e3..07fed83 100644 --- a/fs/dlm/ast.c +++ b/fs/dlm/ast.c @@ -268,7 +268,7 @@ void dlm_callback_work(struct work_struct *work) int dlm_callback_start(struct dlm_ls *ls) { ls->ls_callback_wq = alloc_workqueue("dlm_callback", - WQ_UNBOUND | WQ_MEM_RECLAIM, 0); + WQ_HIGHPRI | WQ_MEM_RECLAIM, 0); if (!ls->ls_callback_wq) { log_print("can't start dlm_callback workqueue"); return -ENOMEM; diff --git a/fs/dlm/config.c b/fs/dlm/config.c index df955d2..7211e82 100644 --- a/fs/dlm/config.c +++ b/fs/dlm/config.c @@ -12,7 +12,7 @@ ******************************************************************************/ #include <linux/kernel.h> -#include <linux/module.h> +#include <linux/init.h> #include <linux/configfs.h> #include <linux/slab.h> #include <linux/in.h> diff --git a/fs/dlm/debug_fs.c b/fs/dlm/debug_fs.c index 466f7d6..ca7089a 100644 --- a/fs/dlm/debug_fs.c +++ b/fs/dlm/debug_fs.c @@ -12,7 +12,7 @@ #include <linux/pagemap.h> #include <linux/seq_file.h> -#include <linux/module.h> +#include <linux/init.h> #include <linux/ctype.h> #include <linux/debugfs.h> #include <linux/slab.h> diff --git a/fs/dlm/dlm_internal.h b/fs/dlm/dlm_internal.h index 216b616..b670f56 100644 --- a/fs/dlm/dlm_internal.h +++ b/fs/dlm/dlm_internal.h @@ -18,7 +18,6 @@ * This is the main header file to be included in each DLM source file. */ -#include <linux/module.h> #include <linux/slab.h> #include <linux/sched.h> #include <linux/types.h> diff --git a/fs/dlm/lockspace.c b/fs/dlm/lockspace.c index f3e7278..91592b7 100644 --- a/fs/dlm/lockspace.c +++ b/fs/dlm/lockspace.c @@ -11,6 +11,8 @@ ******************************************************************************* ******************************************************************************/ +#include <linux/module.h> + #include "dlm_internal.h" #include "lockspace.h" #include "member.h" diff --git a/fs/dlm/lowcomms.c b/fs/dlm/lowcomms.c index 609998d..7d398d3 100644 --- a/fs/dlm/lowcomms.c +++ b/fs/dlm/lowcomms.c @@ -519,29 +519,25 @@ out: /* Note: sk_callback_lock must be locked before calling this function. */ static void save_callbacks(struct connection *con, struct sock *sk) { - lock_sock(sk); con->orig_data_ready = sk->sk_data_ready; con->orig_state_change = sk->sk_state_change; con->orig_write_space = sk->sk_write_space; con->orig_error_report = sk->sk_error_report; - release_sock(sk); } static void restore_callbacks(struct connection *con, struct sock *sk) { write_lock_bh(&sk->sk_callback_lock); - lock_sock(sk); sk->sk_user_data = NULL; sk->sk_data_ready = con->orig_data_ready; sk->sk_state_change = con->orig_state_change; sk->sk_write_space = con->orig_write_space; sk->sk_error_report = con->orig_error_report; - release_sock(sk); write_unlock_bh(&sk->sk_callback_lock); } /* Make a socket active */ -static void add_sock(struct socket *sock, struct connection *con) +static void add_sock(struct socket *sock, struct connection *con, bool save_cb) { struct sock *sk = sock->sk; @@ -549,7 +545,7 @@ static void add_sock(struct socket *sock, struct connection *con) con->sock = sock; sk->sk_user_data = con; - if (!test_bit(CF_IS_OTHERCON, &con->flags)) + if (save_cb) save_callbacks(con, sk); /* Install a data_ready callback */ sk->sk_data_ready = lowcomms_data_ready; @@ -806,7 +802,7 @@ static int tcp_accept_from_sock(struct connection *con) newcon->othercon = othercon; othercon->sock = newsock; newsock->sk->sk_user_data = othercon; - add_sock(newsock, othercon); + add_sock(newsock, othercon, false); addcon = othercon; } else { @@ -819,7 +815,10 @@ static int tcp_accept_from_sock(struct connection *con) else { newsock->sk->sk_user_data = newcon; newcon->rx_action = receive_from_sock; - add_sock(newsock, newcon); + /* accept copies the sk after we've saved the callbacks, so we + don't want to save them a second time or comm errors will + result in calling sk_error_report recursively. */ + add_sock(newsock, newcon, false); addcon = newcon; } @@ -880,7 +879,8 @@ static int sctp_accept_from_sock(struct connection *con) } make_sockaddr(&prim.ssp_addr, 0, &addr_len); - if (addr_to_nodeid(&prim.ssp_addr, &nodeid)) { + ret = addr_to_nodeid(&prim.ssp_addr, &nodeid); + if (ret) { unsigned char *b = (unsigned char *)&prim.ssp_addr; log_print("reject connect from unknown addr"); @@ -919,7 +919,7 @@ static int sctp_accept_from_sock(struct connection *con) newcon->othercon = othercon; othercon->sock = newsock; newsock->sk->sk_user_data = othercon; - add_sock(newsock, othercon); + add_sock(newsock, othercon, false); addcon = othercon; } else { printk("Extra connection from node %d attempted\n", nodeid); @@ -930,7 +930,7 @@ static int sctp_accept_from_sock(struct connection *con) } else { newsock->sk->sk_user_data = newcon; newcon->rx_action = receive_from_sock; - add_sock(newsock, newcon); + add_sock(newsock, newcon, false); addcon = newcon; } @@ -1058,7 +1058,7 @@ static void sctp_connect_to_sock(struct connection *con) sock->sk->sk_user_data = con; con->rx_action = receive_from_sock; con->connect_action = sctp_connect_to_sock; - add_sock(sock, con); + add_sock(sock, con, true); /* Bind to all addresses. */ if (sctp_bind_addrs(con, 0)) @@ -1146,7 +1146,7 @@ static void tcp_connect_to_sock(struct connection *con) sock->sk->sk_user_data = con; con->rx_action = receive_from_sock; con->connect_action = tcp_connect_to_sock; - add_sock(sock, con); + add_sock(sock, con, true); /* Bind to our cluster-known address connecting to avoid routing problems */ @@ -1366,7 +1366,7 @@ static int tcp_listen_for_all(void) sock = tcp_create_listen_sock(con, dlm_local_addr[0]); if (sock) { - add_sock(sock, con); + add_sock(sock, con, true); result = 0; } else { diff --git a/fs/dlm/main.c b/fs/dlm/main.c index 079c0bd..8e1b618 100644 --- a/fs/dlm/main.c +++ b/fs/dlm/main.c @@ -11,6 +11,8 @@ ******************************************************************************* ******************************************************************************/ +#include <linux/module.h> + #include "dlm_internal.h" #include "lockspace.h" #include "lock.h" diff --git a/fs/dlm/netlink.c b/fs/dlm/netlink.c index 0643ae4..43a96c3 100644 --- a/fs/dlm/netlink.c +++ b/fs/dlm/netlink.c @@ -65,7 +65,7 @@ static int user_cmd(struct sk_buff *skb, struct genl_info *info) return 0; } -static struct genl_ops dlm_nl_ops[] = { +static const struct genl_ops dlm_nl_ops[] = { { .cmd = DLM_CMD_HELLO, .doit = user_cmd, diff --git a/fs/dlm/user.c b/fs/dlm/user.c index 58c2f4a..1ce908c 100644 --- a/fs/dlm/user.c +++ b/fs/dlm/user.c @@ -9,7 +9,6 @@ #include <linux/miscdevice.h> #include <linux/init.h> #include <linux/wait.h> -#include <linux/module.h> #include <linux/file.h> #include <linux/fs.h> #include <linux/poll.h> diff --git a/fs/ext2/file.c b/fs/ext2/file.c index a0e1478..b0f2415 100644 --- a/fs/ext2/file.c +++ b/fs/ext2/file.c @@ -38,7 +38,7 @@ static ssize_t ext2_dax_read_iter(struct kiocb *iocb, struct iov_iter *to) return 0; /* skip atime */ inode_lock_shared(inode); - ret = iomap_dax_rw(iocb, to, &ext2_iomap_ops); + ret = dax_iomap_rw(iocb, to, &ext2_iomap_ops); inode_unlock_shared(inode); file_accessed(iocb->ki_filp); @@ -62,7 +62,7 @@ static ssize_t ext2_dax_write_iter(struct kiocb *iocb, struct iov_iter *from) if (ret) goto out_unlock; - ret = iomap_dax_rw(iocb, from, &ext2_iomap_ops); + ret = dax_iomap_rw(iocb, from, &ext2_iomap_ops); if (ret > 0 && iocb->ki_pos > i_size_read(inode)) { i_size_write(inode, iocb->ki_pos); mark_inode_dirty(inode); @@ -99,7 +99,7 @@ static int ext2_dax_fault(struct vm_area_struct *vma, struct vm_fault *vmf) } down_read(&ei->dax_sem); - ret = iomap_dax_fault(vma, vmf, &ext2_iomap_ops); + ret = dax_iomap_fault(vma, vmf, &ext2_iomap_ops); up_read(&ei->dax_sem); if (vmf->flags & FAULT_FLAG_WRITE) @@ -107,27 +107,6 @@ static int ext2_dax_fault(struct vm_area_struct *vma, struct vm_fault *vmf) return ret; } -static int ext2_dax_pmd_fault(struct vm_area_struct *vma, unsigned long addr, - pmd_t *pmd, unsigned int flags) -{ - struct inode *inode = file_inode(vma->vm_file); - struct ext2_inode_info *ei = EXT2_I(inode); - int ret; - - if (flags & FAULT_FLAG_WRITE) { - sb_start_pagefault(inode->i_sb); - file_update_time(vma->vm_file); - } - down_read(&ei->dax_sem); - - ret = dax_pmd_fault(vma, addr, pmd, flags, ext2_get_block); - - up_read(&ei->dax_sem); - if (flags & FAULT_FLAG_WRITE) - sb_end_pagefault(inode->i_sb); - return ret; -} - static int ext2_dax_pfn_mkwrite(struct vm_area_struct *vma, struct vm_fault *vmf) { @@ -154,7 +133,11 @@ static int ext2_dax_pfn_mkwrite(struct vm_area_struct *vma, static const struct vm_operations_struct ext2_dax_vm_ops = { .fault = ext2_dax_fault, - .pmd_fault = ext2_dax_pmd_fault, + /* + * .pmd_fault is not supported for DAX because allocation in ext2 + * cannot be reliably aligned to huge page sizes and so pmd faults + * will always fail and fail back to regular faults. + */ .page_mkwrite = ext2_dax_fault, .pfn_mkwrite = ext2_dax_pfn_mkwrite, }; @@ -166,7 +149,7 @@ static int ext2_file_mmap(struct file *file, struct vm_area_struct *vma) file_accessed(file); vma->vm_ops = &ext2_dax_vm_ops; - vma->vm_flags |= VM_MIXEDMAP | VM_HUGEPAGE; + vma->vm_flags |= VM_MIXEDMAP; return 0; } #else diff --git a/fs/ext2/inode.c b/fs/ext2/inode.c index 41b8b44..046b642 100644 --- a/fs/ext2/inode.c +++ b/fs/ext2/inode.c @@ -850,6 +850,9 @@ struct iomap_ops ext2_iomap_ops = { .iomap_begin = ext2_iomap_begin, .iomap_end = ext2_iomap_end, }; +#else +/* Define empty ops for !CONFIG_FS_DAX case to avoid ugly ifdefs */ +struct iomap_ops ext2_iomap_ops; #endif /* CONFIG_FS_DAX */ int ext2_fiemap(struct inode *inode, struct fiemap_extent_info *fieinfo, @@ -1293,9 +1296,11 @@ static int ext2_setsize(struct inode *inode, loff_t newsize) inode_dio_wait(inode); - if (IS_DAX(inode)) - error = dax_truncate_page(inode, newsize, ext2_get_block); - else if (test_opt(inode->i_sb, NOBH)) + if (IS_DAX(inode)) { + error = iomap_zero_range(inode, newsize, + PAGE_ALIGN(newsize) - newsize, NULL, + &ext2_iomap_ops); + } else if (test_opt(inode->i_sb, NOBH)) error = nobh_truncate_page(inode->i_mapping, newsize, ext2_get_block); else diff --git a/fs/ext4/Kconfig b/fs/ext4/Kconfig index e38039f..7b90691 100644 --- a/fs/ext4/Kconfig +++ b/fs/ext4/Kconfig @@ -37,6 +37,7 @@ config EXT4_FS select CRC16 select CRYPTO select CRYPTO_CRC32C + select FS_IOMAP if FS_DAX help This is the next generation of the ext3 filesystem. diff --git a/fs/ext4/acl.c b/fs/ext4/acl.c index dfa5199..fd38993 100644 --- a/fs/ext4/acl.c +++ b/fs/ext4/acl.c @@ -196,7 +196,7 @@ __ext4_set_acl(handle_t *handle, struct inode *inode, int type, error = posix_acl_update_mode(inode, &inode->i_mode, &acl); if (error) return error; - inode->i_ctime = ext4_current_time(inode); + inode->i_ctime = current_time(inode); ext4_mark_inode_dirty(handle, inode); } break; diff --git a/fs/ext4/ext4.h b/fs/ext4/ext4.h index a8a750f..2163c1e 100644 --- a/fs/ext4/ext4.h +++ b/fs/ext4/ext4.h @@ -397,8 +397,9 @@ struct flex_groups { #define EXT4_RESERVED_FL 0x80000000 /* reserved for ext4 lib */ #define EXT4_FL_USER_VISIBLE 0x304BDFFF /* User visible flags */ -#define EXT4_FL_USER_MODIFIABLE 0x204380FF /* User modifiable flags */ +#define EXT4_FL_USER_MODIFIABLE 0x204BC0FF /* User modifiable flags */ +/* Flags we can manipulate with through EXT4_IOC_FSSETXATTR */ #define EXT4_FL_XFLAG_VISIBLE (EXT4_SYNC_FL | \ EXT4_IMMUTABLE_FL | \ EXT4_APPEND_FL | \ @@ -1533,12 +1534,6 @@ static inline struct ext4_inode_info *EXT4_I(struct inode *inode) return container_of(inode, struct ext4_inode_info, vfs_inode); } -static inline struct timespec ext4_current_time(struct inode *inode) -{ - return (inode->i_sb->s_time_gran < NSEC_PER_SEC) ? - current_fs_time(inode->i_sb) : CURRENT_TIME_SEC; -} - static inline int ext4_valid_inum(struct super_block *sb, unsigned long ino) { return ino == EXT4_ROOT_INO || @@ -2277,11 +2272,6 @@ extern unsigned ext4_free_clusters_after_init(struct super_block *sb, struct ext4_group_desc *gdp); ext4_fsblk_t ext4_inode_to_goal_block(struct inode *); -static inline int ext4_sb_has_crypto(struct super_block *sb) -{ - return ext4_has_feature_encrypt(sb); -} - static inline bool ext4_encrypted_inode(struct inode *inode) { return ext4_test_inode_flag(inode, EXT4_INODE_ENCRYPT); @@ -2339,8 +2329,8 @@ static inline void ext4_fname_free_filename(struct ext4_filename *fname) { } #define fscrypt_pullback_bio_page fscrypt_notsupp_pullback_bio_page #define fscrypt_restore_control_page fscrypt_notsupp_restore_control_page #define fscrypt_zeroout_range fscrypt_notsupp_zeroout_range -#define fscrypt_process_policy fscrypt_notsupp_process_policy -#define fscrypt_get_policy fscrypt_notsupp_get_policy +#define fscrypt_ioctl_set_policy fscrypt_notsupp_ioctl_set_policy +#define fscrypt_ioctl_get_policy fscrypt_notsupp_ioctl_get_policy #define fscrypt_has_permitted_context fscrypt_notsupp_has_permitted_context #define fscrypt_inherit_context fscrypt_notsupp_inherit_context #define fscrypt_get_encryption_info fscrypt_notsupp_get_encryption_info @@ -2458,8 +2448,6 @@ struct buffer_head *ext4_getblk(handle_t *, struct inode *, ext4_lblk_t, int); struct buffer_head *ext4_bread(handle_t *, struct inode *, ext4_lblk_t, int); int ext4_get_block_unwritten(struct inode *inode, sector_t iblock, struct buffer_head *bh_result, int create); -int ext4_dax_get_block(struct inode *inode, sector_t iblock, - struct buffer_head *bh_result, int create); int ext4_get_block(struct inode *inode, sector_t iblock, struct buffer_head *bh_result, int create); int ext4_dio_get_block(struct inode *inode, sector_t iblock, @@ -2492,7 +2480,7 @@ extern int ext4_change_inode_journal_flag(struct inode *, int); extern int ext4_get_inode_loc(struct inode *, struct ext4_iloc *); extern int ext4_inode_attach_jinode(struct inode *inode); extern int ext4_can_truncate(struct inode *inode); -extern void ext4_truncate(struct inode *); +extern int ext4_truncate(struct inode *); extern int ext4_punch_hole(struct inode *inode, loff_t offset, loff_t length); extern int ext4_truncate_restart_trans(handle_t *, struct inode *, int nblocks); extern void ext4_set_inode_flags(struct inode *); @@ -3129,7 +3117,7 @@ extern int ext4_ext_writepage_trans_blocks(struct inode *, int); extern int ext4_ext_index_trans_blocks(struct inode *inode, int extents); extern int ext4_ext_map_blocks(handle_t *handle, struct inode *inode, struct ext4_map_blocks *map, int flags); -extern void ext4_ext_truncate(handle_t *, struct inode *); +extern int ext4_ext_truncate(handle_t *, struct inode *); extern int ext4_ext_remove_space(struct inode *inode, ext4_lblk_t start, ext4_lblk_t end); extern void ext4_ext_init(struct super_block *); @@ -3265,12 +3253,7 @@ static inline void ext4_clear_io_unwritten_flag(ext4_io_end_t *io_end) } } -static inline bool ext4_aligned_io(struct inode *inode, loff_t off, loff_t len) -{ - int blksize = 1 << inode->i_blkbits; - - return IS_ALIGNED(off, blksize) && IS_ALIGNED(len, blksize); -} +extern struct iomap_ops ext4_iomap_ops; #endif /* __KERNEL__ */ diff --git a/fs/ext4/ext4_jbd2.h b/fs/ext4/ext4_jbd2.h index b1d52c1..f976111 100644 --- a/fs/ext4/ext4_jbd2.h +++ b/fs/ext4/ext4_jbd2.h @@ -414,17 +414,19 @@ static inline int ext4_inode_journal_mode(struct inode *inode) return EXT4_INODE_WRITEBACK_DATA_MODE; /* writeback */ /* We do not support data journalling with delayed allocation */ if (!S_ISREG(inode->i_mode) || - test_opt(inode->i_sb, DATA_FLAGS) == EXT4_MOUNT_JOURNAL_DATA) - return EXT4_INODE_JOURNAL_DATA_MODE; /* journal data */ - if (ext4_test_inode_flag(inode, EXT4_INODE_JOURNAL_DATA) && - !test_opt(inode->i_sb, DELALLOC)) + test_opt(inode->i_sb, DATA_FLAGS) == EXT4_MOUNT_JOURNAL_DATA || + (ext4_test_inode_flag(inode, EXT4_INODE_JOURNAL_DATA) && + !test_opt(inode->i_sb, DELALLOC))) { + /* We do not support data journalling for encrypted data */ + if (S_ISREG(inode->i_mode) && ext4_encrypted_inode(inode)) + return EXT4_INODE_ORDERED_DATA_MODE; /* ordered */ return EXT4_INODE_JOURNAL_DATA_MODE; /* journal data */ + } if (test_opt(inode->i_sb, DATA_FLAGS) == EXT4_MOUNT_ORDERED_DATA) return EXT4_INODE_ORDERED_DATA_MODE; /* ordered */ if (test_opt(inode->i_sb, DATA_FLAGS) == EXT4_MOUNT_WRITEBACK_DATA) return EXT4_INODE_WRITEBACK_DATA_MODE; /* writeback */ - else - BUG(); + BUG(); } static inline int ext4_should_journal_data(struct inode *inode) diff --git a/fs/ext4/extents.c b/fs/ext4/extents.c index c930a01..3e1014f 100644 --- a/fs/ext4/extents.c +++ b/fs/ext4/extents.c @@ -4631,7 +4631,7 @@ out2: return err ? err : allocated; } -void ext4_ext_truncate(handle_t *handle, struct inode *inode) +int ext4_ext_truncate(handle_t *handle, struct inode *inode) { struct super_block *sb = inode->i_sb; ext4_lblk_t last_block; @@ -4645,7 +4645,9 @@ void ext4_ext_truncate(handle_t *handle, struct inode *inode) /* we have to know where to truncate from in crash case */ EXT4_I(inode)->i_disksize = inode->i_size; - ext4_mark_inode_dirty(handle, inode); + err = ext4_mark_inode_dirty(handle, inode); + if (err) + return err; last_block = (inode->i_size + sb->s_blocksize - 1) >> EXT4_BLOCK_SIZE_BITS(sb); @@ -4657,12 +4659,9 @@ retry: congestion_wait(BLK_RW_ASYNC, HZ/50); goto retry; } - if (err) { - ext4_std_error(inode->i_sb, err); - return; - } - err = ext4_ext_remove_space(inode, last_block, EXT_MAX_BLOCKS - 1); - ext4_std_error(inode->i_sb, err); + if (err) + return err; + return ext4_ext_remove_space(inode, last_block, EXT_MAX_BLOCKS - 1); } static int ext4_alloc_file_blocks(struct file *file, ext4_lblk_t offset, @@ -4701,7 +4700,7 @@ retry: /* * Recalculate credits when extent tree depth changes. */ - if (depth >= 0 && depth != ext_depth(inode)) { + if (depth != ext_depth(inode)) { credits = ext4_chunk_trans_blocks(inode, len); depth = ext_depth(inode); } @@ -4725,7 +4724,7 @@ retry: map.m_lblk += ret; map.m_len = len = len - ret; epos = (loff_t)map.m_lblk << inode->i_blkbits; - inode->i_ctime = ext4_current_time(inode); + inode->i_ctime = current_time(inode); if (new_size) { if (epos > new_size) epos = new_size; @@ -4853,7 +4852,7 @@ static long ext4_zero_range(struct file *file, loff_t offset, } /* Now release the pages and zero block aligned part of pages */ truncate_pagecache_range(inode, start, end - 1); - inode->i_mtime = inode->i_ctime = ext4_current_time(inode); + inode->i_mtime = inode->i_ctime = current_time(inode); ret = ext4_alloc_file_blocks(file, lblk, max_blocks, new_size, flags, mode); @@ -4878,7 +4877,7 @@ static long ext4_zero_range(struct file *file, loff_t offset, goto out_dio; } - inode->i_mtime = inode->i_ctime = ext4_current_time(inode); + inode->i_mtime = inode->i_ctime = current_time(inode); if (new_size) { ext4_update_inode_size(inode, new_size); } else { @@ -5568,7 +5567,7 @@ int ext4_collapse_range(struct inode *inode, loff_t offset, loff_t len) up_write(&EXT4_I(inode)->i_data_sem); if (IS_SYNC(inode)) ext4_handle_sync(handle); - inode->i_mtime = inode->i_ctime = ext4_current_time(inode); + inode->i_mtime = inode->i_ctime = current_time(inode); ext4_mark_inode_dirty(handle, inode); out_stop: @@ -5678,7 +5677,7 @@ int ext4_insert_range(struct inode *inode, loff_t offset, loff_t len) /* Expand file to avoid data loss if there is error while shifting */ inode->i_size += len; EXT4_I(inode)->i_disksize += len; - inode->i_mtime = inode->i_ctime = ext4_current_time(inode); + inode->i_mtime = inode->i_ctime = current_time(inode); ret = ext4_mark_inode_dirty(handle, inode); if (ret) goto out_stop; diff --git a/fs/ext4/file.c b/fs/ext4/file.c index 2a822d3..b5f1844 100644 --- a/fs/ext4/file.c +++ b/fs/ext4/file.c @@ -31,6 +31,42 @@ #include "xattr.h" #include "acl.h" +#ifdef CONFIG_FS_DAX +static ssize_t ext4_dax_read_iter(struct kiocb *iocb, struct iov_iter *to) +{ + struct inode *inode = file_inode(iocb->ki_filp); + ssize_t ret; + + inode_lock_shared(inode); + /* + * Recheck under inode lock - at this point we are sure it cannot + * change anymore + */ + if (!IS_DAX(inode)) { + inode_unlock_shared(inode); + /* Fallback to buffered IO in case we cannot support DAX */ + return generic_file_read_iter(iocb, to); + } + ret = dax_iomap_rw(iocb, to, &ext4_iomap_ops); + inode_unlock_shared(inode); + + file_accessed(iocb->ki_filp); + return ret; +} +#endif + +static ssize_t ext4_file_read_iter(struct kiocb *iocb, struct iov_iter *to) +{ + if (!iov_iter_count(to)) + return 0; /* skip atime */ + +#ifdef CONFIG_FS_DAX + if (IS_DAX(file_inode(iocb->ki_filp))) + return ext4_dax_read_iter(iocb, to); +#endif + return generic_file_read_iter(iocb, to); +} + /* * Called when an inode is released. Note that this is different * from ext4_file_open: open gets called at every open, but release @@ -88,6 +124,86 @@ ext4_unaligned_aio(struct inode *inode, struct iov_iter *from, loff_t pos) return 0; } +/* Is IO overwriting allocated and initialized blocks? */ +static bool ext4_overwrite_io(struct inode *inode, loff_t pos, loff_t len) +{ + struct ext4_map_blocks map; + unsigned int blkbits = inode->i_blkbits; + int err, blklen; + + if (pos + len > i_size_read(inode)) + return false; + + map.m_lblk = pos >> blkbits; + map.m_len = EXT4_MAX_BLOCKS(len, pos, blkbits); + blklen = map.m_len; + + err = ext4_map_blocks(NULL, inode, &map, 0); + /* + * 'err==len' means that all of the blocks have been preallocated, + * regardless of whether they have been initialized or not. To exclude + * unwritten extents, we need to check m_flags. + */ + return err == blklen && (map.m_flags & EXT4_MAP_MAPPED); +} + +static ssize_t ext4_write_checks(struct kiocb *iocb, struct iov_iter *from) +{ + struct inode *inode = file_inode(iocb->ki_filp); + ssize_t ret; + + ret = generic_write_checks(iocb, from); + if (ret <= 0) + return ret; + /* + * If we have encountered a bitmap-format file, the size limit + * is smaller than s_maxbytes, which is for extent-mapped files. + */ + if (!(ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS))) { + struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb); + + if (iocb->ki_pos >= sbi->s_bitmap_maxbytes) + return -EFBIG; + iov_iter_truncate(from, sbi->s_bitmap_maxbytes - iocb->ki_pos); + } + return iov_iter_count(from); +} + +#ifdef CONFIG_FS_DAX +static ssize_t +ext4_dax_write_iter(struct kiocb *iocb, struct iov_iter *from) +{ + struct inode *inode = file_inode(iocb->ki_filp); + ssize_t ret; + bool overwrite = false; + + inode_lock(inode); + ret = ext4_write_checks(iocb, from); + if (ret <= 0) + goto out; + ret = file_remove_privs(iocb->ki_filp); + if (ret) + goto out; + ret = file_update_time(iocb->ki_filp); + if (ret) + goto out; + + if (ext4_overwrite_io(inode, iocb->ki_pos, iov_iter_count(from))) { + overwrite = true; + downgrade_write(&inode->i_rwsem); + } + ret = dax_iomap_rw(iocb, from, &ext4_iomap_ops); +out: + if (!overwrite) + inode_unlock(inode); + else + inode_unlock_shared(inode); + if (ret > 0) + ret = generic_write_sync(iocb, ret); + return ret; +} +#endif + static ssize_t ext4_file_write_iter(struct kiocb *iocb, struct iov_iter *from) { @@ -97,8 +213,13 @@ ext4_file_write_iter(struct kiocb *iocb, struct iov_iter *from) int overwrite = 0; ssize_t ret; +#ifdef CONFIG_FS_DAX + if (IS_DAX(inode)) + return ext4_dax_write_iter(iocb, from); +#endif + inode_lock(inode); - ret = generic_write_checks(iocb, from); + ret = ext4_write_checks(iocb, from); if (ret <= 0) goto out; @@ -114,53 +235,11 @@ ext4_file_write_iter(struct kiocb *iocb, struct iov_iter *from) ext4_unwritten_wait(inode); } - /* - * If we have encountered a bitmap-format file, the size limit - * is smaller than s_maxbytes, which is for extent-mapped files. - */ - if (!(ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS))) { - struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb); - - if (iocb->ki_pos >= sbi->s_bitmap_maxbytes) { - ret = -EFBIG; - goto out; - } - iov_iter_truncate(from, sbi->s_bitmap_maxbytes - iocb->ki_pos); - } - iocb->private = &overwrite; - if (o_direct) { - size_t length = iov_iter_count(from); - loff_t pos = iocb->ki_pos; - - /* check whether we do a DIO overwrite or not */ - if (ext4_should_dioread_nolock(inode) && !unaligned_aio && - pos + length <= i_size_read(inode)) { - struct ext4_map_blocks map; - unsigned int blkbits = inode->i_blkbits; - int err, len; - - map.m_lblk = pos >> blkbits; - map.m_len = EXT4_MAX_BLOCKS(length, pos, blkbits); - len = map.m_len; - - err = ext4_map_blocks(NULL, inode, &map, 0); - /* - * 'err==len' means that all of blocks has - * been preallocated no matter they are - * initialized or not. For excluding - * unwritten extents, we need to check - * m_flags. There are two conditions that - * indicate for initialized extents. 1) If we - * hit extent cache, EXT4_MAP_MAPPED flag is - * returned; 2) If we do a real lookup, - * non-flags are returned. So we should check - * these two conditions. - */ - if (err == len && (map.m_flags & EXT4_MAP_MAPPED)) - overwrite = 1; - } - } + /* Check whether we do a DIO overwrite or not */ + if (o_direct && ext4_should_dioread_nolock(inode) && !unaligned_aio && + ext4_overwrite_io(inode, iocb->ki_pos, iov_iter_count(from))) + overwrite = 1; ret = __generic_file_write_iter(iocb, from); inode_unlock(inode); @@ -196,7 +275,7 @@ static int ext4_dax_fault(struct vm_area_struct *vma, struct vm_fault *vmf) if (IS_ERR(handle)) result = VM_FAULT_SIGBUS; else - result = dax_fault(vma, vmf, ext4_dax_get_block); + result = dax_iomap_fault(vma, vmf, &ext4_iomap_ops); if (write) { if (!IS_ERR(handle)) @@ -230,9 +309,10 @@ static int ext4_dax_pmd_fault(struct vm_area_struct *vma, unsigned long addr, if (IS_ERR(handle)) result = VM_FAULT_SIGBUS; - else - result = dax_pmd_fault(vma, addr, pmd, flags, - ext4_dax_get_block); + else { + result = dax_iomap_pmd_fault(vma, addr, pmd, flags, + &ext4_iomap_ops); + } if (write) { if (!IS_ERR(handle)) @@ -687,7 +767,7 @@ loff_t ext4_llseek(struct file *file, loff_t offset, int whence) const struct file_operations ext4_file_operations = { .llseek = ext4_llseek, - .read_iter = generic_file_read_iter, + .read_iter = ext4_file_read_iter, .write_iter = ext4_file_write_iter, .unlocked_ioctl = ext4_ioctl, #ifdef CONFIG_COMPAT diff --git a/fs/ext4/ialloc.c b/fs/ext4/ialloc.c index 170421e..e57e8d9 100644 --- a/fs/ext4/ialloc.c +++ b/fs/ext4/ialloc.c @@ -1039,7 +1039,7 @@ got: /* This is the optimal IO size (for stat), not the fs block size */ inode->i_blocks = 0; inode->i_mtime = inode->i_atime = inode->i_ctime = ei->i_crtime = - ext4_current_time(inode); + current_time(inode); memset(ei->i_data, 0, sizeof(ei->i_data)); ei->i_dir_start_lookup = 0; @@ -1115,8 +1115,7 @@ got: } if (encrypt) { - /* give pointer to avoid set_context with journal ops. */ - err = fscrypt_inherit_context(dir, inode, &encrypt, true); + err = fscrypt_inherit_context(dir, inode, handle, true); if (err) goto fail_free_drop; } diff --git a/fs/ext4/inline.c b/fs/ext4/inline.c index f74d5ee..437df6a 100644 --- a/fs/ext4/inline.c +++ b/fs/ext4/inline.c @@ -299,6 +299,11 @@ static int ext4_create_inline_data(handle_t *handle, EXT4_I(inode)->i_inline_size = len + EXT4_MIN_INLINE_DATA_SIZE; ext4_clear_inode_flag(inode, EXT4_INODE_EXTENTS); ext4_set_inode_flag(inode, EXT4_INODE_INLINE_DATA); + /* + * Propagate changes to inode->i_flags as well - e.g. S_DAX may + * get cleared + */ + ext4_set_inode_flags(inode); get_bh(is.iloc.bh); error = ext4_mark_iloc_dirty(handle, inode, &is.iloc); @@ -336,8 +341,10 @@ static int ext4_update_inline_data(handle_t *handle, struct inode *inode, len -= EXT4_MIN_INLINE_DATA_SIZE; value = kzalloc(len, GFP_NOFS); - if (!value) + if (!value) { + error = -ENOMEM; goto out; + } error = ext4_xattr_ibody_get(inode, i.name_index, i.name, value, len); @@ -442,6 +449,11 @@ static int ext4_destroy_inline_data_nolock(handle_t *handle, } } ext4_clear_inode_flag(inode, EXT4_INODE_INLINE_DATA); + /* + * Propagate changes to inode->i_flags as well - e.g. S_DAX may + * get set. + */ + ext4_set_inode_flags(inode); get_bh(is.iloc.bh); error = ext4_mark_iloc_dirty(handle, inode, &is.iloc); @@ -1028,7 +1040,7 @@ static int ext4_add_dirent_to_inline(handle_t *handle, * happen is that the times are slightly out of date * and/or different from the directory change time. */ - dir->i_mtime = dir->i_ctime = ext4_current_time(dir); + dir->i_mtime = dir->i_ctime = current_time(dir); ext4_update_dx_flag(dir); dir->i_version++; ext4_mark_inode_dirty(handle, dir); @@ -1971,7 +1983,7 @@ out: if (inode->i_nlink) ext4_orphan_del(handle, inode); - inode->i_mtime = inode->i_ctime = ext4_current_time(inode); + inode->i_mtime = inode->i_ctime = current_time(inode); ext4_mark_inode_dirty(handle, inode); if (IS_SYNC(inode)) ext4_handle_sync(handle); diff --git a/fs/ext4/inode.c b/fs/ext4/inode.c index 9c06472..72d593f 100644 --- a/fs/ext4/inode.c +++ b/fs/ext4/inode.c @@ -37,6 +37,7 @@ #include <linux/printk.h> #include <linux/slab.h> #include <linux/bitops.h> +#include <linux/iomap.h> #include "ext4_jbd2.h" #include "xattr.h" @@ -71,10 +72,9 @@ static __u32 ext4_inode_csum(struct inode *inode, struct ext4_inode *raw, csum = ext4_chksum(sbi, csum, (__u8 *)&dummy_csum, csum_size); offset += csum_size; - csum = ext4_chksum(sbi, csum, (__u8 *)raw + offset, - EXT4_INODE_SIZE(inode->i_sb) - - offset); } + csum = ext4_chksum(sbi, csum, (__u8 *)raw + offset, + EXT4_INODE_SIZE(inode->i_sb) - offset); } return csum; @@ -261,8 +261,15 @@ void ext4_evict_inode(struct inode *inode) "couldn't mark inode dirty (err %d)", err); goto stop_handle; } - if (inode->i_blocks) - ext4_truncate(inode); + if (inode->i_blocks) { + err = ext4_truncate(inode); + if (err) { + ext4_error(inode->i_sb, + "couldn't truncate inode %lu (err %d)", + inode->i_ino, err); + goto stop_handle; + } + } /* * ext4_ext_truncate() doesn't reserve any slop when it @@ -767,6 +774,9 @@ static int _ext4_get_block(struct inode *inode, sector_t iblock, ext4_update_bh_state(bh, map.m_flags); bh->b_size = inode->i_sb->s_blocksize * map.m_len; ret = 0; + } else if (ret == 0) { + /* hole case, need to fill in bh->b_size */ + bh->b_size = inode->i_sb->s_blocksize * map.m_len; } return ret; } @@ -1166,7 +1176,8 @@ static int ext4_block_write_begin(struct page *page, loff_t pos, unsigned len, if (unlikely(err)) page_zero_new_buffers(page, from, to); else if (decrypt) - err = fscrypt_decrypt_page(page); + err = fscrypt_decrypt_page(page->mapping->host, page, + PAGE_SIZE, 0, page->index); return err; } #endif @@ -2891,7 +2902,8 @@ static int ext4_da_write_begin(struct file *file, struct address_space *mapping, index = pos >> PAGE_SHIFT; - if (ext4_nonda_switch(inode->i_sb)) { + if (ext4_nonda_switch(inode->i_sb) || + S_ISLNK(inode->i_mode)) { *fsdata = (void *)FALL_BACK_TO_NONDELALLOC; return ext4_write_begin(file, mapping, pos, len, flags, pagep, fsdata); @@ -3268,53 +3280,159 @@ static int ext4_releasepage(struct page *page, gfp_t wait) } #ifdef CONFIG_FS_DAX -/* - * Get block function for DAX IO and mmap faults. It takes care of converting - * unwritten extents to written ones and initializes new / converted blocks - * to zeros. - */ -int ext4_dax_get_block(struct inode *inode, sector_t iblock, - struct buffer_head *bh_result, int create) +static int ext4_iomap_begin(struct inode *inode, loff_t offset, loff_t length, + unsigned flags, struct iomap *iomap) { + unsigned int blkbits = inode->i_blkbits; + unsigned long first_block = offset >> blkbits; + unsigned long last_block = (offset + length - 1) >> blkbits; + struct ext4_map_blocks map; int ret; - ext4_debug("inode %lu, create flag %d\n", inode->i_ino, create); - if (!create) - return _ext4_get_block(inode, iblock, bh_result, 0); + if (WARN_ON_ONCE(ext4_has_inline_data(inode))) + return -ERANGE; - ret = ext4_get_block_trans(inode, iblock, bh_result, - EXT4_GET_BLOCKS_PRE_IO | - EXT4_GET_BLOCKS_CREATE_ZERO); - if (ret < 0) - return ret; + map.m_lblk = first_block; + map.m_len = last_block - first_block + 1; + + if (!(flags & IOMAP_WRITE)) { + ret = ext4_map_blocks(NULL, inode, &map, 0); + } else { + int dio_credits; + handle_t *handle; + int retries = 0; - if (buffer_unwritten(bh_result)) { + /* Trim mapping request to maximum we can map at once for DIO */ + if (map.m_len > DIO_MAX_BLOCKS) + map.m_len = DIO_MAX_BLOCKS; + dio_credits = ext4_chunk_trans_blocks(inode, map.m_len); +retry: /* - * We are protected by i_mmap_sem or i_mutex so we know block - * cannot go away from under us even though we dropped - * i_data_sem. Convert extent to written and write zeros there. + * Either we allocate blocks and then we don't get unwritten + * extent so we have reserved enough credits, or the blocks + * are already allocated and unwritten and in that case + * extent conversion fits in the credits as well. */ - ret = ext4_get_block_trans(inode, iblock, bh_result, - EXT4_GET_BLOCKS_CONVERT | - EXT4_GET_BLOCKS_CREATE_ZERO); - if (ret < 0) + handle = ext4_journal_start(inode, EXT4_HT_MAP_BLOCKS, + dio_credits); + if (IS_ERR(handle)) + return PTR_ERR(handle); + + ret = ext4_map_blocks(handle, inode, &map, + EXT4_GET_BLOCKS_CREATE_ZERO); + if (ret < 0) { + ext4_journal_stop(handle); + if (ret == -ENOSPC && + ext4_should_retry_alloc(inode->i_sb, &retries)) + goto retry; return ret; + } + + /* + * If we added blocks beyond i_size, we need to make sure they + * will get truncated if we crash before updating i_size in + * ext4_iomap_end(). For faults we don't need to do that (and + * even cannot because for orphan list operations inode_lock is + * required) - if we happen to instantiate block beyond i_size, + * it is because we race with truncate which has already added + * the inode to the orphan list. + */ + if (!(flags & IOMAP_FAULT) && first_block + map.m_len > + (i_size_read(inode) + (1 << blkbits) - 1) >> blkbits) { + int err; + + err = ext4_orphan_add(handle, inode); + if (err < 0) { + ext4_journal_stop(handle); + return err; + } + } + ext4_journal_stop(handle); } - /* - * At least for now we have to clear BH_New so that DAX code - * doesn't attempt to zero blocks again in a racy way. - */ - clear_buffer_new(bh_result); + + iomap->flags = 0; + iomap->bdev = inode->i_sb->s_bdev; + iomap->offset = first_block << blkbits; + + if (ret == 0) { + iomap->type = IOMAP_HOLE; + iomap->blkno = IOMAP_NULL_BLOCK; + iomap->length = (u64)map.m_len << blkbits; + } else { + if (map.m_flags & EXT4_MAP_MAPPED) { + iomap->type = IOMAP_MAPPED; + } else if (map.m_flags & EXT4_MAP_UNWRITTEN) { + iomap->type = IOMAP_UNWRITTEN; + } else { + WARN_ON_ONCE(1); + return -EIO; + } + iomap->blkno = (sector_t)map.m_pblk << (blkbits - 9); + iomap->length = (u64)map.m_len << blkbits; + } + + if (map.m_flags & EXT4_MAP_NEW) + iomap->flags |= IOMAP_F_NEW; return 0; } -#else -/* Just define empty function, it will never get called. */ -int ext4_dax_get_block(struct inode *inode, sector_t iblock, - struct buffer_head *bh_result, int create) + +static int ext4_iomap_end(struct inode *inode, loff_t offset, loff_t length, + ssize_t written, unsigned flags, struct iomap *iomap) { - BUG(); - return 0; + int ret = 0; + handle_t *handle; + int blkbits = inode->i_blkbits; + bool truncate = false; + + if (!(flags & IOMAP_WRITE) || (flags & IOMAP_FAULT)) + return 0; + + handle = ext4_journal_start(inode, EXT4_HT_INODE, 2); + if (IS_ERR(handle)) { + ret = PTR_ERR(handle); + goto orphan_del; + } + if (ext4_update_inode_size(inode, offset + written)) + ext4_mark_inode_dirty(handle, inode); + /* + * We may need to truncate allocated but not written blocks beyond EOF. + */ + if (iomap->offset + iomap->length > + ALIGN(inode->i_size, 1 << blkbits)) { + ext4_lblk_t written_blk, end_blk; + + written_blk = (offset + written) >> blkbits; + end_blk = (offset + length) >> blkbits; + if (written_blk < end_blk && ext4_can_truncate(inode)) + truncate = true; + } + /* + * Remove inode from orphan list if we were extending a inode and + * everything went fine. + */ + if (!truncate && inode->i_nlink && + !list_empty(&EXT4_I(inode)->i_orphan)) + ext4_orphan_del(handle, inode); + ext4_journal_stop(handle); + if (truncate) { + ext4_truncate_failed_write(inode); +orphan_del: + /* + * If truncate failed early the inode might still be on the + * orphan list; we need to make sure the inode is removed from + * the orphan list in that case. + */ + if (inode->i_nlink) + ext4_orphan_del(NULL, inode); + } + return ret; } + +struct iomap_ops ext4_iomap_ops = { + .iomap_begin = ext4_iomap_begin, + .iomap_end = ext4_iomap_end, +}; + #endif static int ext4_end_io_dio(struct kiocb *iocb, loff_t offset, @@ -3436,19 +3554,7 @@ static ssize_t ext4_direct_IO_write(struct kiocb *iocb, struct iov_iter *iter) iocb->private = NULL; if (overwrite) get_block_func = ext4_dio_get_block_overwrite; - else if (IS_DAX(inode)) { - /* - * We can avoid zeroing for aligned DAX writes beyond EOF. Other - * writes need zeroing either because they can race with page - * faults or because they use partial blocks. - */ - if (round_down(offset, 1<<inode->i_blkbits) >= inode->i_size && - ext4_aligned_io(inode, offset, count)) - get_block_func = ext4_dio_get_block; - else - get_block_func = ext4_dax_get_block; - dio_flags = DIO_LOCKING; - } else if (!ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS) || + else if (!ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS) || round_down(offset, 1 << inode->i_blkbits) >= inode->i_size) { get_block_func = ext4_dio_get_block; dio_flags = DIO_LOCKING | DIO_SKIP_HOLES; @@ -3462,14 +3568,9 @@ static ssize_t ext4_direct_IO_write(struct kiocb *iocb, struct iov_iter *iter) #ifdef CONFIG_EXT4_FS_ENCRYPTION BUG_ON(ext4_encrypted_inode(inode) && S_ISREG(inode->i_mode)); #endif - if (IS_DAX(inode)) { - ret = dax_do_io(iocb, inode, iter, get_block_func, - ext4_end_io_dio, dio_flags); - } else - ret = __blockdev_direct_IO(iocb, inode, - inode->i_sb->s_bdev, iter, - get_block_func, - ext4_end_io_dio, NULL, dio_flags); + ret = __blockdev_direct_IO(iocb, inode, inode->i_sb->s_bdev, iter, + get_block_func, ext4_end_io_dio, NULL, + dio_flags); if (ret > 0 && !overwrite && ext4_test_inode_state(inode, EXT4_STATE_DIO_UNWRITTEN)) { @@ -3538,6 +3639,7 @@ static ssize_t ext4_direct_IO_read(struct kiocb *iocb, struct iov_iter *iter) { struct address_space *mapping = iocb->ki_filp->f_mapping; struct inode *inode = mapping->host; + size_t count = iov_iter_count(iter); ssize_t ret; /* @@ -3546,19 +3648,12 @@ static ssize_t ext4_direct_IO_read(struct kiocb *iocb, struct iov_iter *iter) * we are protected against page writeback as well. */ inode_lock_shared(inode); - if (IS_DAX(inode)) { - ret = dax_do_io(iocb, inode, iter, ext4_dio_get_block, NULL, 0); - } else { - size_t count = iov_iter_count(iter); - - ret = filemap_write_and_wait_range(mapping, iocb->ki_pos, - iocb->ki_pos + count); - if (ret) - goto out_unlock; - ret = __blockdev_direct_IO(iocb, inode, inode->i_sb->s_bdev, - iter, ext4_dio_get_block, - NULL, NULL, 0); - } + ret = filemap_write_and_wait_range(mapping, iocb->ki_pos, + iocb->ki_pos + count); + if (ret) + goto out_unlock; + ret = __blockdev_direct_IO(iocb, inode, inode->i_sb->s_bdev, + iter, ext4_dio_get_block, NULL, NULL, 0); out_unlock: inode_unlock_shared(inode); return ret; @@ -3587,6 +3682,10 @@ static ssize_t ext4_direct_IO(struct kiocb *iocb, struct iov_iter *iter) if (ext4_has_inline_data(inode)) return 0; + /* DAX uses iomap path now */ + if (WARN_ON_ONCE(IS_DAX(inode))) + return 0; + trace_ext4_direct_IO_enter(inode, offset, count, iov_iter_rw(iter)); if (iov_iter_rw(iter) == READ) ret = ext4_direct_IO_read(iocb, iter); @@ -3615,6 +3714,13 @@ static int ext4_journalled_set_page_dirty(struct page *page) return __set_page_dirty_nobuffers(page); } +static int ext4_set_page_dirty(struct page *page) +{ + WARN_ON_ONCE(!PageLocked(page) && !PageDirty(page)); + WARN_ON_ONCE(!page_has_buffers(page)); + return __set_page_dirty_buffers(page); +} + static const struct address_space_operations ext4_aops = { .readpage = ext4_readpage, .readpages = ext4_readpages, @@ -3622,6 +3728,7 @@ static const struct address_space_operations ext4_aops = { .writepages = ext4_writepages, .write_begin = ext4_write_begin, .write_end = ext4_write_end, + .set_page_dirty = ext4_set_page_dirty, .bmap = ext4_bmap, .invalidatepage = ext4_invalidatepage, .releasepage = ext4_releasepage, @@ -3654,6 +3761,7 @@ static const struct address_space_operations ext4_da_aops = { .writepages = ext4_writepages, .write_begin = ext4_da_write_begin, .write_end = ext4_da_write_end, + .set_page_dirty = ext4_set_page_dirty, .bmap = ext4_bmap, .invalidatepage = ext4_da_invalidatepage, .releasepage = ext4_releasepage, @@ -3743,7 +3851,8 @@ static int __ext4_block_zero_page_range(handle_t *handle, /* We expect the key to be set. */ BUG_ON(!fscrypt_has_encryption_key(inode)); BUG_ON(blocksize != PAGE_SIZE); - WARN_ON_ONCE(fscrypt_decrypt_page(page)); + WARN_ON_ONCE(fscrypt_decrypt_page(page->mapping->host, + page, PAGE_SIZE, 0, page->index)); } } if (ext4_should_journal_data(inode)) { @@ -3792,8 +3901,10 @@ static int ext4_block_zero_page_range(handle_t *handle, if (length > max || length < 0) length = max; - if (IS_DAX(inode)) - return dax_zero_page_range(inode, from, length, ext4_get_block); + if (IS_DAX(inode)) { + return iomap_zero_range(inode, from, length, NULL, + &ext4_iomap_ops); + } return __ext4_block_zero_page_range(handle, mapping, from, length); } @@ -4026,7 +4137,7 @@ int ext4_punch_hole(struct inode *inode, loff_t offset, loff_t length) if (IS_SYNC(inode)) ext4_handle_sync(handle); - inode->i_mtime = inode->i_ctime = ext4_current_time(inode); + inode->i_mtime = inode->i_ctime = current_time(inode); ext4_mark_inode_dirty(handle, inode); out_stop: ext4_journal_stop(handle); @@ -4091,10 +4202,11 @@ int ext4_inode_attach_jinode(struct inode *inode) * that's fine - as long as they are linked from the inode, the post-crash * ext4_truncate() run will find them and release them. */ -void ext4_truncate(struct inode *inode) +int ext4_truncate(struct inode *inode) { struct ext4_inode_info *ei = EXT4_I(inode); unsigned int credits; + int err = 0; handle_t *handle; struct address_space *mapping = inode->i_mapping; @@ -4108,7 +4220,7 @@ void ext4_truncate(struct inode *inode) trace_ext4_truncate_enter(inode); if (!ext4_can_truncate(inode)) - return; + return 0; ext4_clear_inode_flag(inode, EXT4_INODE_EOFBLOCKS); @@ -4120,13 +4232,13 @@ void ext4_truncate(struct inode *inode) ext4_inline_data_truncate(inode, &has_inline); if (has_inline) - return; + return 0; } /* If we zero-out tail of the page, we have to create jinode for jbd2 */ if (inode->i_size & (inode->i_sb->s_blocksize - 1)) { if (ext4_inode_attach_jinode(inode) < 0) - return; + return 0; } if (ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS)) @@ -4135,10 +4247,8 @@ void ext4_truncate(struct inode *inode) credits = ext4_blocks_for_truncate(inode); handle = ext4_journal_start(inode, EXT4_HT_TRUNCATE, credits); - if (IS_ERR(handle)) { - ext4_std_error(inode->i_sb, PTR_ERR(handle)); - return; - } + if (IS_ERR(handle)) + return PTR_ERR(handle); if (inode->i_size & (inode->i_sb->s_blocksize - 1)) ext4_block_truncate_page(handle, mapping, inode->i_size); @@ -4152,7 +4262,8 @@ void ext4_truncate(struct inode *inode) * Implication: the file must always be in a sane, consistent * truncatable state while each transaction commits. */ - if (ext4_orphan_add(handle, inode)) + err = ext4_orphan_add(handle, inode); + if (err) goto out_stop; down_write(&EXT4_I(inode)->i_data_sem); @@ -4160,11 +4271,13 @@ void ext4_truncate(struct inode *inode) ext4_discard_preallocations(inode); if (ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS)) - ext4_ext_truncate(handle, inode); + err = ext4_ext_truncate(handle, inode); else ext4_ind_truncate(handle, inode); up_write(&ei->i_data_sem); + if (err) + goto out_stop; if (IS_SYNC(inode)) ext4_handle_sync(handle); @@ -4180,11 +4293,12 @@ out_stop: if (inode->i_nlink) ext4_orphan_del(handle, inode); - inode->i_mtime = inode->i_ctime = ext4_current_time(inode); + inode->i_mtime = inode->i_ctime = current_time(inode); ext4_mark_inode_dirty(handle, inode); ext4_journal_stop(handle); trace_ext4_truncate_exit(inode); + return err; } /* @@ -4352,7 +4466,9 @@ void ext4_set_inode_flags(struct inode *inode) new_fl |= S_NOATIME; if (flags & EXT4_DIRSYNC_FL) new_fl |= S_DIRSYNC; - if (test_opt(inode->i_sb, DAX) && S_ISREG(inode->i_mode)) + if (test_opt(inode->i_sb, DAX) && S_ISREG(inode->i_mode) && + !ext4_should_journal_data(inode) && !ext4_has_inline_data(inode) && + !ext4_encrypted_inode(inode)) new_fl |= S_DAX; inode_set_flags(inode, new_fl, S_SYNC|S_APPEND|S_IMMUTABLE|S_NOATIME|S_DIRSYNC|S_DAX); @@ -4411,7 +4527,9 @@ static inline void ext4_iget_extra_inode(struct inode *inode, { __le32 *magic = (void *)raw_inode + EXT4_GOOD_OLD_INODE_SIZE + ei->i_extra_isize; - if (*magic == cpu_to_le32(EXT4_XATTR_MAGIC)) { + if (EXT4_GOOD_OLD_INODE_SIZE + ei->i_extra_isize + sizeof(__le32) <= + EXT4_INODE_SIZE(inode->i_sb) && + *magic == cpu_to_le32(EXT4_XATTR_MAGIC)) { ext4_set_inode_state(inode, EXT4_STATE_XATTR); ext4_find_inline_data_nolock(inode); } else @@ -4434,6 +4552,7 @@ struct inode *ext4_iget(struct super_block *sb, unsigned long ino) struct inode *inode; journal_t *journal = EXT4_SB(sb)->s_journal; long ret; + loff_t size; int block; uid_t i_uid; gid_t i_gid; @@ -4456,10 +4575,12 @@ struct inode *ext4_iget(struct super_block *sb, unsigned long ino) if (EXT4_INODE_SIZE(inode->i_sb) > EXT4_GOOD_OLD_INODE_SIZE) { ei->i_extra_isize = le16_to_cpu(raw_inode->i_extra_isize); if (EXT4_GOOD_OLD_INODE_SIZE + ei->i_extra_isize > - EXT4_INODE_SIZE(inode->i_sb)) { - EXT4_ERROR_INODE(inode, "bad extra_isize (%u != %u)", - EXT4_GOOD_OLD_INODE_SIZE + ei->i_extra_isize, - EXT4_INODE_SIZE(inode->i_sb)); + EXT4_INODE_SIZE(inode->i_sb) || + (ei->i_extra_isize & 3)) { + EXT4_ERROR_INODE(inode, + "bad extra_isize %u (inode size %u)", + ei->i_extra_isize, + EXT4_INODE_SIZE(inode->i_sb)); ret = -EFSCORRUPTED; goto bad_inode; } @@ -4534,6 +4655,11 @@ struct inode *ext4_iget(struct super_block *sb, unsigned long ino) ei->i_file_acl |= ((__u64)le16_to_cpu(raw_inode->i_file_acl_high)) << 32; inode->i_size = ext4_isize(raw_inode); + if ((size = i_size_read(inode)) < 0) { + EXT4_ERROR_INODE(inode, "bad i_size value: %lld", size); + ret = -EFSCORRUPTED; + goto bad_inode; + } ei->i_disksize = inode->i_size; #ifdef CONFIG_QUOTA ei->i_reserved_quota = 0; @@ -4577,6 +4703,7 @@ struct inode *ext4_iget(struct super_block *sb, unsigned long ino) if (EXT4_INODE_SIZE(inode->i_sb) > EXT4_GOOD_OLD_INODE_SIZE) { if (ei->i_extra_isize == 0) { /* The extra space is currently unused. Use it. */ + BUILD_BUG_ON(sizeof(struct ext4_inode) & 3); ei->i_extra_isize = sizeof(struct ext4_inode) - EXT4_GOOD_OLD_INODE_SIZE; } else { @@ -5154,7 +5281,7 @@ int ext4_setattr(struct dentry *dentry, struct iattr *attr) * update c/mtime in shrink case below */ if (!shrink) { - inode->i_mtime = ext4_current_time(inode); + inode->i_mtime = current_time(inode); inode->i_ctime = inode->i_mtime; } down_write(&EXT4_I(inode)->i_data_sem); @@ -5199,12 +5326,15 @@ int ext4_setattr(struct dentry *dentry, struct iattr *attr) * in data=journal mode to make pages freeable. */ truncate_pagecache(inode, inode->i_size); - if (shrink) - ext4_truncate(inode); + if (shrink) { + rc = ext4_truncate(inode); + if (rc) + error = rc; + } up_write(&EXT4_I(inode)->i_mmap_sem); } - if (!rc) { + if (!error) { setattr_copy(inode, attr); mark_inode_dirty(inode); } @@ -5216,7 +5346,7 @@ int ext4_setattr(struct dentry *dentry, struct iattr *attr) if (orphan && inode->i_nlink) ext4_orphan_del(NULL, inode); - if (!rc && (ia_valid & ATTR_MODE)) + if (!error && (ia_valid & ATTR_MODE)) rc = posix_acl_chmod(inode, inode->i_mode); err_out: @@ -5455,18 +5585,20 @@ int ext4_mark_inode_dirty(handle_t *handle, struct inode *inode) err = ext4_reserve_inode_write(handle, inode, &iloc); if (err) return err; - if (ext4_handle_valid(handle) && - EXT4_I(inode)->i_extra_isize < sbi->s_want_extra_isize && + if (EXT4_I(inode)->i_extra_isize < sbi->s_want_extra_isize && !ext4_test_inode_state(inode, EXT4_STATE_NO_EXPAND)) { /* - * We need extra buffer credits since we may write into EA block + * In nojournal mode, we can immediately attempt to expand + * the inode. When journaled, we first need to obtain extra + * buffer credits since we may write into the EA block * with this same handle. If journal_extend fails, then it will * only result in a minor loss of functionality for that inode. * If this is felt to be critical, then e2fsck should be run to * force a large enough s_min_extra_isize. */ - if ((jbd2_journal_extend(handle, - EXT4_DATA_TRANS_BLOCKS(inode->i_sb))) == 0) { + if (!ext4_handle_valid(handle) || + jbd2_journal_extend(handle, + EXT4_DATA_TRANS_BLOCKS(inode->i_sb)) == 0) { ret = ext4_expand_extra_isize(inode, sbi->s_want_extra_isize, iloc, handle); @@ -5620,6 +5752,11 @@ int ext4_change_inode_journal_flag(struct inode *inode, int val) ext4_clear_inode_flag(inode, EXT4_INODE_JOURNAL_DATA); } ext4_set_aops(inode); + /* + * Update inode->i_flags after EXT4_INODE_JOURNAL_DATA was updated. + * E.g. S_DAX may get cleared / set. + */ + ext4_set_inode_flags(inode); jbd2_journal_unlock_updates(journal); percpu_up_write(&sbi->s_journal_flag_rwsem); diff --git a/fs/ext4/ioctl.c b/fs/ext4/ioctl.c index bf5ae8e..49fd137 100644 --- a/fs/ext4/ioctl.c +++ b/fs/ext4/ioctl.c @@ -153,7 +153,7 @@ static long swap_inode_boot_loader(struct super_block *sb, swap_inode_data(inode, inode_bl); - inode->i_ctime = inode_bl->i_ctime = ext4_current_time(inode); + inode->i_ctime = inode_bl->i_ctime = current_time(inode); spin_lock(&sbi->s_next_gen_lock); inode->i_generation = sbi->s_next_generation++; @@ -191,6 +191,7 @@ journal_err_out: return err; } +#ifdef CONFIG_EXT4_FS_ENCRYPTION static int uuid_is_zero(__u8 u[16]) { int i; @@ -200,6 +201,7 @@ static int uuid_is_zero(__u8 u[16]) return 0; return 1; } +#endif static int ext4_ioctl_setflags(struct inode *inode, unsigned int flags) @@ -248,8 +250,11 @@ static int ext4_ioctl_setflags(struct inode *inode, err = -EOPNOTSUPP; goto flags_out; } - } else if (oldflags & EXT4_EOFBLOCKS_FL) - ext4_truncate(inode); + } else if (oldflags & EXT4_EOFBLOCKS_FL) { + err = ext4_truncate(inode); + if (err) + goto flags_out; + } handle = ext4_journal_start(inode, EXT4_HT_INODE, 1); if (IS_ERR(handle)) { @@ -265,6 +270,9 @@ static int ext4_ioctl_setflags(struct inode *inode, for (i = 0, mask = 1; i < 32; i++, mask <<= 1) { if (!(mask & EXT4_FL_USER_MODIFIABLE)) continue; + /* These flags get special treatment later */ + if (mask == EXT4_JOURNAL_DATA_FL || mask == EXT4_EXTENTS_FL) + continue; if (mask & flags) ext4_set_inode_flag(inode, i); else @@ -272,7 +280,7 @@ static int ext4_ioctl_setflags(struct inode *inode, } ext4_set_inode_flags(inode); - inode->i_ctime = ext4_current_time(inode); + inode->i_ctime = current_time(inode); err = ext4_mark_iloc_dirty(handle, inode, &iloc); flags_err: @@ -368,7 +376,7 @@ static int ext4_ioctl_setproject(struct file *filp, __u32 projid) } EXT4_I(inode)->i_projid = kprojid; - inode->i_ctime = ext4_current_time(inode); + inode->i_ctime = current_time(inode); out_dirty: rc = ext4_mark_iloc_dirty(handle, inode, &iloc); if (!err) @@ -409,6 +417,10 @@ static inline __u32 ext4_iflags_to_xflags(unsigned long iflags) return xflags; } +#define EXT4_SUPPORTED_FS_XFLAGS (FS_XFLAG_SYNC | FS_XFLAG_IMMUTABLE | \ + FS_XFLAG_APPEND | FS_XFLAG_NODUMP | \ + FS_XFLAG_NOATIME | FS_XFLAG_PROJINHERIT) + /* Transfer xflags flags to internal */ static inline unsigned long ext4_xflags_to_iflags(__u32 xflags) { @@ -453,12 +465,22 @@ long ext4_ioctl(struct file *filp, unsigned int cmd, unsigned long arg) if (get_user(flags, (int __user *) arg)) return -EFAULT; + if (flags & ~EXT4_FL_USER_VISIBLE) + return -EOPNOTSUPP; + /* + * chattr(1) grabs flags via GETFLAGS, modifies the result and + * passes that to SETFLAGS. So we cannot easily make SETFLAGS + * more restrictive than just silently masking off visible but + * not settable flags as we always did. + */ + flags &= EXT4_FL_USER_MODIFIABLE; + if (ext4_mask_flags(inode->i_mode, flags) != flags) + return -EOPNOTSUPP; + err = mnt_want_write_file(filp); if (err) return err; - flags = ext4_mask_flags(inode->i_mode, flags); - inode_lock(inode); err = ext4_ioctl_setflags(inode, flags); inode_unlock(inode); @@ -500,7 +522,7 @@ long ext4_ioctl(struct file *filp, unsigned int cmd, unsigned long arg) } err = ext4_reserve_inode_write(handle, inode, &iloc); if (err == 0) { - inode->i_ctime = ext4_current_time(inode); + inode->i_ctime = current_time(inode); inode->i_generation = generation; err = ext4_mark_iloc_dirty(handle, inode, &iloc); } @@ -765,28 +787,19 @@ resizefs_out: } case EXT4_IOC_PRECACHE_EXTENTS: return ext4_ext_precache(inode); - case EXT4_IOC_SET_ENCRYPTION_POLICY: { -#ifdef CONFIG_EXT4_FS_ENCRYPTION - struct fscrypt_policy policy; + case EXT4_IOC_SET_ENCRYPTION_POLICY: if (!ext4_has_feature_encrypt(sb)) return -EOPNOTSUPP; + return fscrypt_ioctl_set_policy(filp, (const void __user *)arg); - if (copy_from_user(&policy, - (struct fscrypt_policy __user *)arg, - sizeof(policy))) - return -EFAULT; - return fscrypt_process_policy(filp, &policy); -#else - return -EOPNOTSUPP; -#endif - } case EXT4_IOC_GET_ENCRYPTION_PWSALT: { +#ifdef CONFIG_EXT4_FS_ENCRYPTION int err, err2; struct ext4_sb_info *sbi = EXT4_SB(sb); handle_t *handle; - if (!ext4_sb_has_crypto(sb)) + if (!ext4_has_feature_encrypt(sb)) return -EOPNOTSUPP; if (uuid_is_zero(sbi->s_es->s_encrypt_pw_salt)) { err = mnt_want_write_file(filp); @@ -816,24 +829,13 @@ resizefs_out: sbi->s_es->s_encrypt_pw_salt, 16)) return -EFAULT; return 0; - } - case EXT4_IOC_GET_ENCRYPTION_POLICY: { -#ifdef CONFIG_EXT4_FS_ENCRYPTION - struct fscrypt_policy policy; - int err = 0; - - if (!ext4_encrypted_inode(inode)) - return -ENOENT; - err = fscrypt_get_policy(inode, &policy); - if (err) - return err; - if (copy_to_user((void __user *)arg, &policy, sizeof(policy))) - return -EFAULT; - return 0; #else return -EOPNOTSUPP; #endif } + case EXT4_IOC_GET_ENCRYPTION_POLICY: + return fscrypt_ioctl_get_policy(filp, (void __user *)arg); + case EXT4_IOC_FSGETXATTR: { struct fsxattr fa; @@ -865,13 +867,17 @@ resizefs_out: if (!inode_owner_or_capable(inode)) return -EACCES; + if (fa.fsx_xflags & ~EXT4_SUPPORTED_FS_XFLAGS) + return -EOPNOTSUPP; + + flags = ext4_xflags_to_iflags(fa.fsx_xflags); + if (ext4_mask_flags(inode->i_mode, flags) != flags) + return -EOPNOTSUPP; + err = mnt_want_write_file(filp); if (err) return err; - flags = ext4_xflags_to_iflags(fa.fsx_xflags); - flags = ext4_mask_flags(inode->i_mode, flags); - inode_lock(inode); flags = (ei->i_flags & ~EXT4_FL_XFLAG_VISIBLE) | (flags & EXT4_FL_XFLAG_VISIBLE); diff --git a/fs/ext4/mballoc.c b/fs/ext4/mballoc.c index f418f55..7ae43c5 100644 --- a/fs/ext4/mballoc.c +++ b/fs/ext4/mballoc.c @@ -669,7 +669,7 @@ static void ext4_mb_mark_free_simple(struct super_block *sb, ext4_grpblk_t min; ext4_grpblk_t max; ext4_grpblk_t chunk; - unsigned short border; + unsigned int border; BUG_ON(len > EXT4_CLUSTERS_PER_GROUP(sb)); @@ -2287,7 +2287,7 @@ static int ext4_mb_seq_groups_show(struct seq_file *seq, void *v) struct ext4_group_info *grinfo; struct sg { struct ext4_group_info info; - ext4_grpblk_t counters[16]; + ext4_grpblk_t counters[EXT4_MAX_BLOCK_LOG_SIZE + 2]; } sg; group--; diff --git a/fs/ext4/namei.c b/fs/ext4/namei.c index 104f8bf..eadba91 100644 --- a/fs/ext4/namei.c +++ b/fs/ext4/namei.c @@ -1941,7 +1941,7 @@ static int add_dirent_to_buf(handle_t *handle, struct ext4_filename *fname, * happen is that the times are slightly out of date * and/or different from the directory change time. */ - dir->i_mtime = dir->i_ctime = ext4_current_time(dir); + dir->i_mtime = dir->i_ctime = current_time(dir); ext4_update_dx_flag(dir); dir->i_version++; ext4_mark_inode_dirty(handle, dir); @@ -2987,7 +2987,7 @@ static int ext4_rmdir(struct inode *dir, struct dentry *dentry) * recovery. */ inode->i_size = 0; ext4_orphan_add(handle, inode); - inode->i_ctime = dir->i_ctime = dir->i_mtime = ext4_current_time(inode); + inode->i_ctime = dir->i_ctime = dir->i_mtime = current_time(inode); ext4_mark_inode_dirty(handle, inode); ext4_dec_count(handle, dir); ext4_update_dx_flag(dir); @@ -3050,13 +3050,13 @@ static int ext4_unlink(struct inode *dir, struct dentry *dentry) retval = ext4_delete_entry(handle, dir, de, bh); if (retval) goto end_unlink; - dir->i_ctime = dir->i_mtime = ext4_current_time(dir); + dir->i_ctime = dir->i_mtime = current_time(dir); ext4_update_dx_flag(dir); ext4_mark_inode_dirty(handle, dir); drop_nlink(inode); if (!inode->i_nlink) ext4_orphan_add(handle, inode); - inode->i_ctime = ext4_current_time(inode); + inode->i_ctime = current_time(inode); ext4_mark_inode_dirty(handle, inode); end_unlink: @@ -3254,7 +3254,7 @@ retry: if (IS_DIRSYNC(dir)) ext4_handle_sync(handle); - inode->i_ctime = ext4_current_time(inode); + inode->i_ctime = current_time(inode); ext4_inc_count(handle, inode); ihold(inode); @@ -3381,7 +3381,7 @@ static int ext4_setent(handle_t *handle, struct ext4_renament *ent, ent->de->file_type = file_type; ent->dir->i_version++; ent->dir->i_ctime = ent->dir->i_mtime = - ext4_current_time(ent->dir); + current_time(ent->dir); ext4_mark_inode_dirty(handle, ent->dir); BUFFER_TRACE(ent->bh, "call ext4_handle_dirty_metadata"); if (!ent->inlined) { @@ -3651,7 +3651,7 @@ static int ext4_rename(struct inode *old_dir, struct dentry *old_dentry, * Like most other Unix systems, set the ctime for inodes on a * rename. */ - old.inode->i_ctime = ext4_current_time(old.inode); + old.inode->i_ctime = current_time(old.inode); ext4_mark_inode_dirty(handle, old.inode); if (!whiteout) { @@ -3663,9 +3663,9 @@ static int ext4_rename(struct inode *old_dir, struct dentry *old_dentry, if (new.inode) { ext4_dec_count(handle, new.inode); - new.inode->i_ctime = ext4_current_time(new.inode); + new.inode->i_ctime = current_time(new.inode); } - old.dir->i_ctime = old.dir->i_mtime = ext4_current_time(old.dir); + old.dir->i_ctime = old.dir->i_mtime = current_time(old.dir); ext4_update_dx_flag(old.dir); if (old.dir_bh) { retval = ext4_rename_dir_finish(handle, &old, new.dir->i_ino); @@ -3723,6 +3723,7 @@ static int ext4_cross_rename(struct inode *old_dir, struct dentry *old_dentry, }; u8 new_file_type; int retval; + struct timespec ctime; if ((ext4_encrypted_inode(old_dir) || ext4_encrypted_inode(new_dir)) && @@ -3823,8 +3824,9 @@ static int ext4_cross_rename(struct inode *old_dir, struct dentry *old_dentry, * Like most other Unix systems, set the ctime for inodes on a * rename. */ - old.inode->i_ctime = ext4_current_time(old.inode); - new.inode->i_ctime = ext4_current_time(new.inode); + ctime = current_time(old.inode); + old.inode->i_ctime = ctime; + new.inode->i_ctime = ctime; ext4_mark_inode_dirty(handle, old.inode); ext4_mark_inode_dirty(handle, new.inode); diff --git a/fs/ext4/page-io.c b/fs/ext4/page-io.c index e0b3b54..e2332a6 100644 --- a/fs/ext4/page-io.c +++ b/fs/ext4/page-io.c @@ -470,7 +470,8 @@ int ext4_bio_write_page(struct ext4_io_submit *io, gfp_t gfp_flags = GFP_NOFS; retry_encrypt: - data_page = fscrypt_encrypt_page(inode, page, gfp_flags); + data_page = fscrypt_encrypt_page(inode, page, PAGE_SIZE, 0, + page->index, gfp_flags); if (IS_ERR(data_page)) { ret = PTR_ERR(data_page); if (ret == -ENOMEM && wbc->sync_mode == WB_SYNC_ALL) { diff --git a/fs/ext4/super.c b/fs/ext4/super.c index caa4147..dfc8309 100644 --- a/fs/ext4/super.c +++ b/fs/ext4/super.c @@ -863,7 +863,6 @@ static void ext4_put_super(struct super_block *sb) percpu_counter_destroy(&sbi->s_dirs_counter); percpu_counter_destroy(&sbi->s_dirtyclusters_counter); percpu_free_rwsem(&sbi->s_journal_flag_rwsem); - brelse(sbi->s_sbh); #ifdef CONFIG_QUOTA for (i = 0; i < EXT4_MAXQUOTAS; i++) kfree(sbi->s_qf_names[i]); @@ -895,6 +894,7 @@ static void ext4_put_super(struct super_block *sb) } if (sbi->s_mmp_tsk) kthread_stop(sbi->s_mmp_tsk); + brelse(sbi->s_sbh); sb->s_fs_info = NULL; /* * Now that we are completely done shutting down the @@ -1114,37 +1114,55 @@ static int ext4_prepare_context(struct inode *inode) static int ext4_set_context(struct inode *inode, const void *ctx, size_t len, void *fs_data) { - handle_t *handle; - int res, res2; + handle_t *handle = fs_data; + int res, res2, retries = 0; + + /* + * If a journal handle was specified, then the encryption context is + * being set on a new inode via inheritance and is part of a larger + * transaction to create the inode. Otherwise the encryption context is + * being set on an existing inode in its own transaction. Only in the + * latter case should the "retry on ENOSPC" logic be used. + */ - /* fs_data is null when internally used. */ - if (fs_data) { - res = ext4_xattr_set(inode, EXT4_XATTR_INDEX_ENCRYPTION, - EXT4_XATTR_NAME_ENCRYPTION_CONTEXT, ctx, - len, 0); + if (handle) { + res = ext4_xattr_set_handle(handle, inode, + EXT4_XATTR_INDEX_ENCRYPTION, + EXT4_XATTR_NAME_ENCRYPTION_CONTEXT, + ctx, len, 0); if (!res) { ext4_set_inode_flag(inode, EXT4_INODE_ENCRYPT); ext4_clear_inode_state(inode, EXT4_STATE_MAY_INLINE_DATA); + /* + * Update inode->i_flags - e.g. S_DAX may get disabled + */ + ext4_set_inode_flags(inode); } return res; } +retry: handle = ext4_journal_start(inode, EXT4_HT_MISC, ext4_jbd2_credits_xattr(inode)); if (IS_ERR(handle)) return PTR_ERR(handle); - res = ext4_xattr_set(inode, EXT4_XATTR_INDEX_ENCRYPTION, - EXT4_XATTR_NAME_ENCRYPTION_CONTEXT, ctx, - len, 0); + res = ext4_xattr_set_handle(handle, inode, EXT4_XATTR_INDEX_ENCRYPTION, + EXT4_XATTR_NAME_ENCRYPTION_CONTEXT, + ctx, len, 0); if (!res) { ext4_set_inode_flag(inode, EXT4_INODE_ENCRYPT); + /* Update inode->i_flags - e.g. S_DAX may get disabled */ + ext4_set_inode_flags(inode); res = ext4_mark_inode_dirty(handle, inode); if (res) EXT4_ERROR_INODE(inode, "Failed to mark inode dirty"); } res2 = ext4_journal_stop(handle); + + if (res == -ENOSPC && ext4_should_retry_alloc(inode->i_sb, &retries)) + goto retry; if (!res) res = res2; return res; @@ -1883,12 +1901,6 @@ static int parse_options(char *options, struct super_block *sb, return 0; } } - if (test_opt(sb, DATA_FLAGS) == EXT4_MOUNT_ORDERED_DATA && - test_opt(sb, JOURNAL_ASYNC_COMMIT)) { - ext4_msg(sb, KERN_ERR, "can't mount with journal_async_commit " - "in data=ordered mode"); - return 0; - } return 1; } @@ -2330,7 +2342,7 @@ static void ext4_orphan_cleanup(struct super_block *sb, struct ext4_super_block *es) { unsigned int s_flags = sb->s_flags; - int nr_orphans = 0, nr_truncates = 0; + int ret, nr_orphans = 0, nr_truncates = 0; #ifdef CONFIG_QUOTA int i; #endif @@ -2412,7 +2424,9 @@ static void ext4_orphan_cleanup(struct super_block *sb, inode->i_ino, inode->i_size); inode_lock(inode); truncate_inode_pages(inode->i_mapping, inode->i_size); - ext4_truncate(inode); + ret = ext4_truncate(inode); + if (ret) + ext4_std_error(inode->i_sb, ret); inode_unlock(inode); nr_truncates++; } else { @@ -3193,10 +3207,15 @@ static int count_overhead(struct super_block *sb, ext4_group_t grp, ext4_set_bit(s++, buf); count++; } - for (j = ext4_bg_num_gdb(sb, grp); j > 0; j--) { - ext4_set_bit(EXT4_B2C(sbi, s++), buf); - count++; + j = ext4_bg_num_gdb(sb, grp); + if (s + j > EXT4_BLOCKS_PER_GROUP(sb)) { + ext4_error(sb, "Invalid number of block group " + "descriptor blocks: %d", j); + j = EXT4_BLOCKS_PER_GROUP(sb) - s; } + count += j; + for (; j > 0; j--) + ext4_set_bit(EXT4_B2C(sbi, s++), buf); } if (!count) return 0; @@ -3301,7 +3320,7 @@ static int ext4_fill_super(struct super_block *sb, void *data, int silent) char *orig_data = kstrdup(data, GFP_KERNEL); struct buffer_head *bh; struct ext4_super_block *es = NULL; - struct ext4_sb_info *sbi; + struct ext4_sb_info *sbi = kzalloc(sizeof(*sbi), GFP_KERNEL); ext4_fsblk_t block; ext4_fsblk_t sb_block = get_sb_block(&data); ext4_fsblk_t logical_sb_block; @@ -3320,16 +3339,14 @@ static int ext4_fill_super(struct super_block *sb, void *data, int silent) unsigned int journal_ioprio = DEFAULT_JOURNAL_IOPRIO; ext4_group_t first_not_zeroed; - sbi = kzalloc(sizeof(*sbi), GFP_KERNEL); - if (!sbi) - goto out_free_orig; + if ((data && !orig_data) || !sbi) + goto out_free_base; sbi->s_blockgroup_lock = kzalloc(sizeof(struct blockgroup_lock), GFP_KERNEL); - if (!sbi->s_blockgroup_lock) { - kfree(sbi); - goto out_free_orig; - } + if (!sbi->s_blockgroup_lock) + goto out_free_base; + sb->s_fs_info = sbi; sbi->s_sb = sb; sbi->s_inode_readahead_blks = EXT4_DEF_INODE_READAHEAD_BLKS; @@ -3475,11 +3492,19 @@ static int ext4_fill_super(struct super_block *sb, void *data, int silent) */ sbi->s_li_wait_mult = EXT4_DEF_LI_WAIT_MULT; - if (!parse_options((char *) sbi->s_es->s_mount_opts, sb, - &journal_devnum, &journal_ioprio, 0)) { - ext4_msg(sb, KERN_WARNING, - "failed to parse options in superblock: %s", - sbi->s_es->s_mount_opts); + if (sbi->s_es->s_mount_opts[0]) { + char *s_mount_opts = kstrndup(sbi->s_es->s_mount_opts, + sizeof(sbi->s_es->s_mount_opts), + GFP_KERNEL); + if (!s_mount_opts) + goto failed_mount; + if (!parse_options(s_mount_opts, sb, &journal_devnum, + &journal_ioprio, 0)) { + ext4_msg(sb, KERN_WARNING, + "failed to parse options in superblock: %s", + s_mount_opts); + } + kfree(s_mount_opts); } sbi->s_def_mount_opt = sbi->s_mount_opt; if (!parse_options((char *) data, sb, &journal_devnum, @@ -3505,6 +3530,11 @@ static int ext4_fill_super(struct super_block *sb, void *data, int silent) "both data=journal and dax"); goto failed_mount; } + if (ext4_has_feature_encrypt(sb)) { + ext4_msg(sb, KERN_WARNING, + "encrypted files will use data=ordered " + "instead of data journaling mode"); + } if (test_opt(sb, DELALLOC)) clear_opt(sb, DELALLOC); } else { @@ -3660,12 +3690,16 @@ static int ext4_fill_super(struct super_block *sb, void *data, int silent) sbi->s_blocks_per_group = le32_to_cpu(es->s_blocks_per_group); sbi->s_inodes_per_group = le32_to_cpu(es->s_inodes_per_group); - if (EXT4_INODE_SIZE(sb) == 0 || EXT4_INODES_PER_GROUP(sb) == 0) - goto cantfind_ext4; sbi->s_inodes_per_block = blocksize / EXT4_INODE_SIZE(sb); if (sbi->s_inodes_per_block == 0) goto cantfind_ext4; + if (sbi->s_inodes_per_group < sbi->s_inodes_per_block || + sbi->s_inodes_per_group > blocksize * 8) { + ext4_msg(sb, KERN_ERR, "invalid inodes per group: %lu\n", + sbi->s_blocks_per_group); + goto failed_mount; + } sbi->s_itb_per_group = sbi->s_inodes_per_group / sbi->s_inodes_per_block; sbi->s_desc_per_block = blocksize / EXT4_DESC_SIZE(sb); @@ -3748,13 +3782,6 @@ static int ext4_fill_super(struct super_block *sb, void *data, int silent) } sbi->s_cluster_ratio = clustersize / blocksize; - if (sbi->s_inodes_per_group > blocksize * 8) { - ext4_msg(sb, KERN_ERR, - "#inodes per group too big: %lu", - sbi->s_inodes_per_group); - goto failed_mount; - } - /* Do we have standard group size of clustersize * 8 blocks ? */ if (sbi->s_blocks_per_group == clustersize << 3) set_opt2(sb, STD_GROUP_SIZE); @@ -3814,6 +3841,15 @@ static int ext4_fill_super(struct super_block *sb, void *data, int silent) (EXT4_MAX_BLOCK_FILE_PHYS / EXT4_BLOCKS_PER_GROUP(sb))); db_count = (sbi->s_groups_count + EXT4_DESC_PER_BLOCK(sb) - 1) / EXT4_DESC_PER_BLOCK(sb); + if (ext4_has_feature_meta_bg(sb)) { + if (le32_to_cpu(es->s_first_meta_bg) >= db_count) { + ext4_msg(sb, KERN_WARNING, + "first meta block group too large: %u " + "(group descriptor block count %u)", + le32_to_cpu(es->s_first_meta_bg), db_count); + goto failed_mount; + } + } sbi->s_group_desc = ext4_kvmalloc(db_count * sizeof(struct buffer_head *), GFP_KERNEL); @@ -3967,6 +4003,14 @@ static int ext4_fill_super(struct super_block *sb, void *data, int silent) default: break; } + + if (test_opt(sb, DATA_FLAGS) == EXT4_MOUNT_ORDERED_DATA && + test_opt(sb, JOURNAL_ASYNC_COMMIT)) { + ext4_msg(sb, KERN_ERR, "can't mount with " + "journal_async_commit in data=ordered mode"); + goto failed_mount_wq; + } + set_task_ioprio(sbi->s_journal->j_task, journal_ioprio); sbi->s_journal->j_commit_callback = ext4_journal_commit_callback; @@ -4160,7 +4204,9 @@ no_journal: if (___ratelimit(&ext4_mount_msg_ratelimit, "EXT4-fs mount")) ext4_msg(sb, KERN_INFO, "mounted filesystem with%s. " - "Opts: %s%s%s", descr, sbi->s_es->s_mount_opts, + "Opts: %.*s%s%s", descr, + (int) sizeof(sbi->s_es->s_mount_opts), + sbi->s_es->s_mount_opts, *sbi->s_es->s_mount_opts ? "; " : "", orig_data); if (es->s_error_count) @@ -4239,8 +4285,8 @@ failed_mount: out_fail: sb->s_fs_info = NULL; kfree(sbi->s_blockgroup_lock); +out_free_base: kfree(sbi); -out_free_orig: kfree(orig_data); return err ? err : ret; } @@ -4550,7 +4596,8 @@ static int ext4_commit_super(struct super_block *sb, int sync) &EXT4_SB(sb)->s_freeinodes_counter)); BUFFER_TRACE(sbh, "marking dirty"); ext4_superblock_csum_set(sb); - lock_buffer(sbh); + if (sync) + lock_buffer(sbh); if (buffer_write_io_error(sbh)) { /* * Oh, dear. A previous attempt to write the @@ -4566,8 +4613,8 @@ static int ext4_commit_super(struct super_block *sb, int sync) set_buffer_uptodate(sbh); } mark_buffer_dirty(sbh); - unlock_buffer(sbh); if (sync) { + unlock_buffer(sbh); error = __sync_dirty_buffer(sbh, test_opt(sb, BARRIER) ? REQ_FUA : REQ_SYNC); if (error) @@ -4857,6 +4904,13 @@ static int ext4_remount(struct super_block *sb, int *flags, char *data) err = -EINVAL; goto restore_opts; } + } else if (test_opt(sb, DATA_FLAGS) == EXT4_MOUNT_ORDERED_DATA) { + if (test_opt(sb, JOURNAL_ASYNC_COMMIT)) { + ext4_msg(sb, KERN_ERR, "can't mount with " + "journal_async_commit in data=ordered mode"); + err = -EINVAL; + goto restore_opts; + } } if ((sbi->s_mount_opt ^ old_opts.s_mount_opt) & EXT4_MOUNT_DAX) { @@ -5366,7 +5420,7 @@ static int ext4_quota_off(struct super_block *sb, int type) handle = ext4_journal_start(inode, EXT4_HT_QUOTA, 1); if (IS_ERR(handle)) goto out; - inode->i_mtime = inode->i_ctime = CURRENT_TIME; + inode->i_mtime = inode->i_ctime = current_time(inode); ext4_mark_inode_dirty(handle, inode); ext4_journal_stop(handle); diff --git a/fs/ext4/xattr.c b/fs/ext4/xattr.c index d77be9e..5a94fa52 100644 --- a/fs/ext4/xattr.c +++ b/fs/ext4/xattr.c @@ -185,6 +185,7 @@ ext4_xattr_check_names(struct ext4_xattr_entry *entry, void *end, { struct ext4_xattr_entry *e = entry; + /* Find the end of the names list */ while (!IS_LAST_ENTRY(e)) { struct ext4_xattr_entry *next = EXT4_XATTR_NEXT(e); if ((void *)next >= end) @@ -192,15 +193,29 @@ ext4_xattr_check_names(struct ext4_xattr_entry *entry, void *end, e = next; } + /* Check the values */ while (!IS_LAST_ENTRY(entry)) { if (entry->e_value_block != 0) return -EFSCORRUPTED; - if (entry->e_value_size != 0 && - (value_start + le16_to_cpu(entry->e_value_offs) < - (void *)e + sizeof(__u32) || - value_start + le16_to_cpu(entry->e_value_offs) + - le32_to_cpu(entry->e_value_size) > end)) - return -EFSCORRUPTED; + if (entry->e_value_size != 0) { + u16 offs = le16_to_cpu(entry->e_value_offs); + u32 size = le32_to_cpu(entry->e_value_size); + void *value; + + /* + * The value cannot overlap the names, and the value + * with padding cannot extend beyond 'end'. Check both + * the padded and unpadded sizes, since the size may + * overflow to 0 when adding padding. + */ + if (offs > end - value_start) + return -EFSCORRUPTED; + value = value_start + offs; + if (value < (void *)e + sizeof(u32) || + size > end - value || + EXT4_XATTR_SIZE(size) > end - value) + return -EFSCORRUPTED; + } entry = EXT4_XATTR_NEXT(entry); } @@ -231,13 +246,12 @@ static int __xattr_check_inode(struct inode *inode, struct ext4_xattr_ibody_header *header, void *end, const char *function, unsigned int line) { - struct ext4_xattr_entry *entry = IFIRST(header); int error = -EFSCORRUPTED; - if (((void *) header >= end) || + if (end - (void *)header < sizeof(*header) + sizeof(u32) || (header->h_magic != cpu_to_le32(EXT4_XATTR_MAGIC))) goto errout; - error = ext4_xattr_check_names(entry, end, entry); + error = ext4_xattr_check_names(IFIRST(header), end, IFIRST(header)); errout: if (error) __ext4_error_inode(inode, function, line, 0, @@ -1109,7 +1123,7 @@ int ext4_xattr_ibody_inline_set(handle_t *handle, struct inode *inode, return 0; } -static int ext4_xattr_ibody_set(handle_t *handle, struct inode *inode, +static int ext4_xattr_ibody_set(struct inode *inode, struct ext4_xattr_info *i, struct ext4_xattr_ibody_find *is) { @@ -1216,7 +1230,7 @@ ext4_xattr_set_handle(handle_t *handle, struct inode *inode, int name_index, } if (!value) { if (!is.s.not_found) - error = ext4_xattr_ibody_set(handle, inode, &i, &is); + error = ext4_xattr_ibody_set(inode, &i, &is); else if (!bs.s.not_found) error = ext4_xattr_block_set(handle, inode, &i, &bs); } else { @@ -1227,7 +1241,7 @@ ext4_xattr_set_handle(handle_t *handle, struct inode *inode, int name_index, if (!bs.s.not_found && ext4_xattr_value_same(&bs.s, &i)) goto cleanup; - error = ext4_xattr_ibody_set(handle, inode, &i, &is); + error = ext4_xattr_ibody_set(inode, &i, &is); if (!error && !bs.s.not_found) { i.value = NULL; error = ext4_xattr_block_set(handle, inode, &i, &bs); @@ -1242,14 +1256,13 @@ ext4_xattr_set_handle(handle_t *handle, struct inode *inode, int name_index, goto cleanup; if (!is.s.not_found) { i.value = NULL; - error = ext4_xattr_ibody_set(handle, inode, &i, - &is); + error = ext4_xattr_ibody_set(inode, &i, &is); } } } if (!error) { ext4_xattr_update_super_block(handle, inode->i_sb); - inode->i_ctime = ext4_current_time(inode); + inode->i_ctime = current_time(inode); if (!value) ext4_clear_inode_state(inode, EXT4_STATE_NO_EXPAND); error = ext4_mark_iloc_dirty(handle, inode, &is.iloc); @@ -1384,7 +1397,7 @@ static int ext4_xattr_move_to_block(handle_t *handle, struct inode *inode, goto out; /* Remove the chosen entry from the inode */ - error = ext4_xattr_ibody_set(handle, inode, &i, is); + error = ext4_xattr_ibody_set(inode, &i, is); if (error) goto out; diff --git a/fs/f2fs/acl.c b/fs/f2fs/acl.c index 6fe23af..8f48769 100644 --- a/fs/f2fs/acl.c +++ b/fs/f2fs/acl.c @@ -384,7 +384,7 @@ int f2fs_init_acl(struct inode *inode, struct inode *dir, struct page *ipage, if (error) return error; - f2fs_mark_inode_dirty_sync(inode); + f2fs_mark_inode_dirty_sync(inode, true); if (default_acl) { error = __f2fs_set_acl(inode, ACL_TYPE_DEFAULT, default_acl, diff --git a/fs/f2fs/checkpoint.c b/fs/f2fs/checkpoint.c index d935c06..f73ee95 100644 --- a/fs/f2fs/checkpoint.c +++ b/fs/f2fs/checkpoint.c @@ -228,7 +228,7 @@ void ra_meta_pages_cond(struct f2fs_sb_info *sbi, pgoff_t index) f2fs_put_page(page, 0); if (readahead) - ra_meta_pages(sbi, index, MAX_BIO_BLOCKS(sbi), META_POR, true); + ra_meta_pages(sbi, index, BIO_MAX_PAGES, META_POR, true); } static int f2fs_write_meta_page(struct page *page, @@ -770,7 +770,12 @@ int get_valid_checkpoint(struct f2fs_sb_info *sbi) /* Sanity checking of checkpoint */ if (sanity_check_ckpt(sbi)) - goto fail_no_cp; + goto free_fail_no_cp; + + if (cur_page == cp1) + sbi->cur_cp_pack = 1; + else + sbi->cur_cp_pack = 2; if (cp_blks <= 1) goto done; @@ -793,6 +798,9 @@ done: f2fs_put_page(cp2, 1); return 0; +free_fail_no_cp: + f2fs_put_page(cp1, 1); + f2fs_put_page(cp2, 1); fail_no_cp: kfree(sbi->ckpt); return -EINVAL; @@ -921,7 +929,11 @@ int f2fs_sync_inode_meta(struct f2fs_sb_info *sbi) inode = igrab(&fi->vfs_inode); spin_unlock(&sbi->inode_lock[DIRTY_META]); if (inode) { - update_inode_page(inode); + sync_inode_metadata(inode, 0); + + /* it's on eviction */ + if (is_inode_flag_set(inode, FI_DIRTY_INODE)) + update_inode_page(inode); iput(inode); } }; @@ -987,7 +999,7 @@ static void unblock_operations(struct f2fs_sb_info *sbi) { up_write(&sbi->node_write); - build_free_nids(sbi); + build_free_nids(sbi, false); f2fs_unlock_all(sbi); } @@ -998,7 +1010,7 @@ static void wait_on_all_pages_writeback(struct f2fs_sb_info *sbi) for (;;) { prepare_to_wait(&sbi->cp_wait, &wait, TASK_UNINTERRUPTIBLE); - if (!atomic_read(&sbi->nr_wb_bios)) + if (!get_pages(sbi, F2FS_WB_CP_DATA)) break; io_schedule_timeout(5*HZ); @@ -1123,7 +1135,7 @@ static int do_checkpoint(struct f2fs_sb_info *sbi, struct cp_control *cpc) le32_to_cpu(ckpt->checksum_offset))) = cpu_to_le32(crc32); - start_blk = __start_cp_addr(sbi); + start_blk = __start_cp_next_addr(sbi); /* need to wait for end_io results */ wait_on_all_pages_writeback(sbi); @@ -1184,9 +1196,9 @@ static int do_checkpoint(struct f2fs_sb_info *sbi, struct cp_control *cpc) if (unlikely(f2fs_cp_error(sbi))) return -EIO; - clear_prefree_segments(sbi, cpc); clear_sbi_flag(sbi, SBI_IS_DIRTY); clear_sbi_flag(sbi, SBI_NEED_CP); + __set_cp_next_pack(sbi); /* * redirty superblock if metadata like node page or inode cache is @@ -1261,8 +1273,12 @@ int write_checkpoint(struct f2fs_sb_info *sbi, struct cp_control *cpc) /* unlock all the fs_lock[] in do_checkpoint() */ err = do_checkpoint(sbi, cpc); - - f2fs_wait_all_discard_bio(sbi); + if (err) { + release_discard_addrs(sbi); + } else { + clear_prefree_segments(sbi, cpc); + f2fs_wait_all_discard_bio(sbi); + } unblock_operations(sbi); stat_inc_cp_count(sbi->stat_info); diff --git a/fs/f2fs/data.c b/fs/f2fs/data.c index 9e5561f..9ac2625 100644 --- a/fs/f2fs/data.c +++ b/fs/f2fs/data.c @@ -29,6 +29,26 @@ #include "trace.h" #include <trace/events/f2fs.h> +static bool __is_cp_guaranteed(struct page *page) +{ + struct address_space *mapping = page->mapping; + struct inode *inode; + struct f2fs_sb_info *sbi; + + if (!mapping) + return false; + + inode = mapping->host; + sbi = F2FS_I_SB(inode); + + if (inode->i_ino == F2FS_META_INO(sbi) || + inode->i_ino == F2FS_NODE_INO(sbi) || + S_ISDIR(inode->i_mode) || + is_cold_data(page)) + return true; + return false; +} + static void f2fs_read_end_io(struct bio *bio) { struct bio_vec *bvec; @@ -71,6 +91,7 @@ static void f2fs_write_end_io(struct bio *bio) bio_for_each_segment_all(bvec, bio, i) { struct page *page = bvec->bv_page; + enum count_type type = WB_DATA_TYPE(page); fscrypt_pullback_bio_page(&page, true); @@ -78,9 +99,11 @@ static void f2fs_write_end_io(struct bio *bio) mapping_set_error(page->mapping, -EIO); f2fs_stop_checkpoint(sbi, true); } + dec_page_count(sbi, type); + clear_cold_data(page); end_page_writeback(page); } - if (atomic_dec_and_test(&sbi->nr_wb_bios) && + if (!get_pages(sbi, F2FS_WB_CP_DATA) && wq_has_sleeper(&sbi->cp_wait)) wake_up(&sbi->cp_wait); @@ -88,6 +111,46 @@ static void f2fs_write_end_io(struct bio *bio) } /* + * Return true, if pre_bio's bdev is same as its target device. + */ +struct block_device *f2fs_target_device(struct f2fs_sb_info *sbi, + block_t blk_addr, struct bio *bio) +{ + struct block_device *bdev = sbi->sb->s_bdev; + int i; + + for (i = 0; i < sbi->s_ndevs; i++) { + if (FDEV(i).start_blk <= blk_addr && + FDEV(i).end_blk >= blk_addr) { + blk_addr -= FDEV(i).start_blk; + bdev = FDEV(i).bdev; + break; + } + } + if (bio) { + bio->bi_bdev = bdev; + bio->bi_iter.bi_sector = SECTOR_FROM_BLOCK(blk_addr); + } + return bdev; +} + +int f2fs_target_device_index(struct f2fs_sb_info *sbi, block_t blkaddr) +{ + int i; + + for (i = 0; i < sbi->s_ndevs; i++) + if (FDEV(i).start_blk <= blkaddr && FDEV(i).end_blk >= blkaddr) + return i; + return 0; +} + +static bool __same_bdev(struct f2fs_sb_info *sbi, + block_t blk_addr, struct bio *bio) +{ + return f2fs_target_device(sbi, blk_addr, NULL) == bio->bi_bdev; +} + +/* * Low-level block read/write IO operations. */ static struct bio *__bio_alloc(struct f2fs_sb_info *sbi, block_t blk_addr, @@ -97,8 +160,7 @@ static struct bio *__bio_alloc(struct f2fs_sb_info *sbi, block_t blk_addr, bio = f2fs_bio_alloc(npages); - bio->bi_bdev = sbi->sb->s_bdev; - bio->bi_iter.bi_sector = SECTOR_FROM_BLOCK(blk_addr); + f2fs_target_device(sbi, blk_addr, bio); bio->bi_end_io = is_read ? f2fs_read_end_io : f2fs_write_end_io; bio->bi_private = is_read ? NULL : sbi; @@ -109,8 +171,7 @@ static inline void __submit_bio(struct f2fs_sb_info *sbi, struct bio *bio, enum page_type type) { if (!is_read_io(bio_op(bio))) { - atomic_inc(&sbi->nr_wb_bios); - if (f2fs_sb_mounted_hmsmr(sbi->sb) && + if (f2fs_sb_mounted_blkzoned(sbi->sb) && current->plug && (type == DATA || type == NODE)) blk_finish_plug(current->plug); } @@ -268,22 +329,24 @@ void f2fs_submit_page_mbio(struct f2fs_io_info *fio) verify_block_addr(sbi, fio->old_blkaddr); verify_block_addr(sbi, fio->new_blkaddr); + bio_page = fio->encrypted_page ? fio->encrypted_page : fio->page; + + if (!is_read) + inc_page_count(sbi, WB_DATA_TYPE(bio_page)); + down_write(&io->io_rwsem); if (io->bio && (io->last_block_in_bio != fio->new_blkaddr - 1 || - (io->fio.op != fio->op || io->fio.op_flags != fio->op_flags))) + (io->fio.op != fio->op || io->fio.op_flags != fio->op_flags) || + !__same_bdev(sbi, fio->new_blkaddr, io->bio))) __submit_merged_bio(io); alloc_new: if (io->bio == NULL) { - int bio_blocks = MAX_BIO_BLOCKS(sbi); - io->bio = __bio_alloc(sbi, fio->new_blkaddr, - bio_blocks, is_read); + BIO_MAX_PAGES, is_read); io->fio = *fio; } - bio_page = fio->encrypted_page ? fio->encrypted_page : fio->page; - if (bio_add_page(io->bio, bio_page, PAGE_SIZE, 0) < PAGE_SIZE) { __submit_merged_bio(io); @@ -588,7 +651,6 @@ static int __allocate_data_block(struct dnode_of_data *dn) struct f2fs_sb_info *sbi = F2FS_I_SB(dn->inode); struct f2fs_summary sum; struct node_info ni; - int seg = CURSEG_WARM_DATA; pgoff_t fofs; blkcnt_t count = 1; @@ -606,11 +668,8 @@ alloc: get_node_info(sbi, dn->nid, &ni); set_summary(&sum, dn->nid, dn->ofs_in_node, ni.version); - if (dn->ofs_in_node == 0 && dn->inode_page == dn->node_page) - seg = CURSEG_DIRECT_IO; - allocate_data_block(sbi, NULL, dn->data_blkaddr, &dn->data_blkaddr, - &sum, seg); + &sum, CURSEG_WARM_DATA); set_data_blkaddr(dn); /* update i_size */ @@ -622,11 +681,18 @@ alloc: return 0; } -ssize_t f2fs_preallocate_blocks(struct kiocb *iocb, struct iov_iter *from) +static inline bool __force_buffered_io(struct inode *inode, int rw) +{ + return ((f2fs_encrypted_inode(inode) && S_ISREG(inode->i_mode)) || + (rw == WRITE && test_opt(F2FS_I_SB(inode), LFS)) || + F2FS_I_SB(inode)->s_ndevs); +} + +int f2fs_preallocate_blocks(struct kiocb *iocb, struct iov_iter *from) { struct inode *inode = file_inode(iocb->ki_filp); struct f2fs_map_blocks map; - ssize_t ret = 0; + int err = 0; map.m_lblk = F2FS_BLK_ALIGN(iocb->ki_pos); map.m_len = F2FS_BYTES_TO_BLK(iocb->ki_pos + iov_iter_count(from)); @@ -638,19 +704,22 @@ ssize_t f2fs_preallocate_blocks(struct kiocb *iocb, struct iov_iter *from) map.m_next_pgofs = NULL; if (iocb->ki_flags & IOCB_DIRECT) { - ret = f2fs_convert_inline_inode(inode); - if (ret) - return ret; - return f2fs_map_blocks(inode, &map, 1, F2FS_GET_BLOCK_PRE_DIO); + err = f2fs_convert_inline_inode(inode); + if (err) + return err; + return f2fs_map_blocks(inode, &map, 1, + __force_buffered_io(inode, WRITE) ? + F2FS_GET_BLOCK_PRE_AIO : + F2FS_GET_BLOCK_PRE_DIO); } if (iocb->ki_pos + iov_iter_count(from) > MAX_INLINE_DATA) { - ret = f2fs_convert_inline_inode(inode); - if (ret) - return ret; + err = f2fs_convert_inline_inode(inode); + if (err) + return err; } if (!f2fs_has_inline_data(inode)) return f2fs_map_blocks(inode, &map, 1, F2FS_GET_BLOCK_PRE_AIO); - return ret; + return err; } /* @@ -674,7 +743,6 @@ int f2fs_map_blocks(struct inode *inode, struct f2fs_map_blocks *map, unsigned int ofs_in_node, last_ofs_in_node; blkcnt_t prealloc; struct extent_info ei; - bool allocated = false; block_t blkaddr; if (!maxblocks) @@ -714,7 +782,7 @@ next_dnode: } prealloc = 0; - ofs_in_node = dn.ofs_in_node; + last_ofs_in_node = ofs_in_node = dn.ofs_in_node; end_offset = ADDRS_PER_PAGE(dn.node_page, inode); next_block: @@ -733,10 +801,8 @@ next_block: } } else { err = __allocate_data_block(&dn); - if (!err) { + if (!err) set_inode_flag(inode, FI_APPEND_WRITE); - allocated = true; - } } if (err) goto sync_out; @@ -791,7 +857,6 @@ skip: err = reserve_new_blocks(&dn, prealloc); if (err) goto sync_out; - allocated = dn.node_changed; map->m_len += dn.ofs_in_node - ofs_in_node; if (prealloc && dn.ofs_in_node != last_ofs_in_node + 1) { @@ -810,9 +875,8 @@ skip: if (create) { f2fs_unlock_op(sbi); - f2fs_balance_fs(sbi, allocated); + f2fs_balance_fs(sbi, dn.node_changed); } - allocated = false; goto next_dnode; sync_out: @@ -820,7 +884,7 @@ sync_out: unlock_out: if (create) { f2fs_unlock_op(sbi); - f2fs_balance_fs(sbi, allocated); + f2fs_balance_fs(sbi, dn.node_changed); } out: trace_f2fs_map_blocks(inode, map, err); @@ -832,19 +896,19 @@ static int __get_data_block(struct inode *inode, sector_t iblock, pgoff_t *next_pgofs) { struct f2fs_map_blocks map; - int ret; + int err; map.m_lblk = iblock; map.m_len = bh->b_size >> inode->i_blkbits; map.m_next_pgofs = next_pgofs; - ret = f2fs_map_blocks(inode, &map, create, flag); - if (!ret) { + err = f2fs_map_blocks(inode, &map, create, flag); + if (!err) { map_bh(bh, inode->i_sb, map.m_pblk); bh->b_state = (bh->b_state & ~F2FS_MAP_FLAGS) | map.m_flags; bh->b_size = map.m_len << inode->i_blkbits; } - return ret; + return err; } static int get_data_block(struct inode *inode, sector_t iblock, @@ -889,7 +953,6 @@ int f2fs_fiemap(struct inode *inode, struct fiemap_extent_info *fieinfo, struct buffer_head map_bh; sector_t start_blk, last_blk; pgoff_t next_pgofs; - loff_t isize; u64 logical = 0, phys = 0, size = 0; u32 flags = 0; int ret = 0; @@ -906,13 +969,6 @@ int f2fs_fiemap(struct inode *inode, struct fiemap_extent_info *fieinfo, inode_lock(inode); - isize = i_size_read(inode); - if (start >= isize) - goto out; - - if (start + len > isize) - len = isize - start; - if (logical_to_blk(inode, len) == 0) len = blk_to_logical(inode, 1); @@ -931,13 +987,11 @@ next: /* HOLE */ if (!buffer_mapped(&map_bh)) { start_blk = next_pgofs; - /* Go through holes util pass the EOF */ - if (blk_to_logical(inode, start_blk) < isize) + + if (blk_to_logical(inode, start_blk) < blk_to_logical(inode, + F2FS_I_SB(inode)->max_file_blocks)) goto prep_next; - /* Found a hole beyond isize means no more extents. - * Note that the premise is that filesystems don't - * punch holes beyond isize and keep size unchanged. - */ + flags |= FIEMAP_EXTENT_LAST; } @@ -980,7 +1034,6 @@ static struct bio *f2fs_grab_bio(struct inode *inode, block_t blkaddr, { struct f2fs_sb_info *sbi = F2FS_I_SB(inode); struct fscrypt_ctx *ctx = NULL; - struct block_device *bdev = sbi->sb->s_bdev; struct bio *bio; if (f2fs_encrypted_inode(inode) && S_ISREG(inode->i_mode)) { @@ -998,8 +1051,7 @@ static struct bio *f2fs_grab_bio(struct inode *inode, block_t blkaddr, fscrypt_release_ctx(ctx); return ERR_PTR(-ENOMEM); } - bio->bi_bdev = bdev; - bio->bi_iter.bi_sector = SECTOR_FROM_BLOCK(blkaddr); + f2fs_target_device(sbi, blkaddr, bio); bio->bi_end_io = f2fs_read_end_io; bio->bi_private = ctx; @@ -1094,7 +1146,8 @@ got_it: * This page will go to BIO. Do we need to send this * BIO off first? */ - if (bio && (last_block_in_bio != block_nr - 1)) { + if (bio && (last_block_in_bio != block_nr - 1 || + !__same_bdev(F2FS_I_SB(inode), block_nr, bio))) { submit_and_realloc: __submit_bio(F2FS_I_SB(inode), bio, DATA); bio = NULL; @@ -1193,7 +1246,9 @@ int do_write_data_page(struct f2fs_io_info *fio) fio->old_blkaddr); retry_encrypt: fio->encrypted_page = fscrypt_encrypt_page(inode, fio->page, - gfp_flags); + PAGE_SIZE, 0, + fio->page->index, + gfp_flags); if (IS_ERR(fio->encrypted_page)) { err = PTR_ERR(fio->encrypted_page); if (err == -ENOMEM) { @@ -1309,7 +1364,6 @@ done: if (err && err != -ENOENT) goto redirty_out; - clear_cold_data(page); out: inode_dec_dirty_pages(inode); if (err) @@ -1330,6 +1384,8 @@ out: redirty_out: redirty_page_for_writepage(wbc, page); + if (!err) + return AOP_WRITEPAGE_ACTIVATE; unlock_page(page); return err; } @@ -1425,6 +1481,15 @@ continue_unlock: ret = mapping->a_ops->writepage(page, wbc); if (unlikely(ret)) { + /* + * keep nr_to_write, since vfs uses this to + * get # of written pages. + */ + if (ret == AOP_WRITEPAGE_ACTIVATE) { + unlock_page(page); + ret = 0; + continue; + } done_index = page->index + 1; done = 1; break; @@ -1712,7 +1777,6 @@ static int f2fs_write_end(struct file *file, goto unlock_out; set_page_dirty(page); - clear_cold_data(page); if (pos + copied > i_size_read(inode)) f2fs_i_size_write(inode, pos + copied); @@ -1749,9 +1813,7 @@ static ssize_t f2fs_direct_IO(struct kiocb *iocb, struct iov_iter *iter) if (err) return err; - if (f2fs_encrypted_inode(inode) && S_ISREG(inode->i_mode)) - return 0; - if (test_opt(F2FS_I_SB(inode), LFS)) + if (__force_buffered_io(inode, rw)) return 0; trace_f2fs_direct_IO_enter(inode, offset, count, rw); @@ -1783,12 +1845,14 @@ void f2fs_invalidate_page(struct page *page, unsigned int offset, return; if (PageDirty(page)) { - if (inode->i_ino == F2FS_META_INO(sbi)) + if (inode->i_ino == F2FS_META_INO(sbi)) { dec_page_count(sbi, F2FS_DIRTY_META); - else if (inode->i_ino == F2FS_NODE_INO(sbi)) + } else if (inode->i_ino == F2FS_NODE_INO(sbi)) { dec_page_count(sbi, F2FS_DIRTY_NODES); - else + } else { inode_dec_dirty_pages(inode); + remove_dirty_inode(inode); + } } /* This is atomic written page, keep Private */ diff --git a/fs/f2fs/debug.c b/fs/f2fs/debug.c index fb245bd..fbd5184 100644 --- a/fs/f2fs/debug.c +++ b/fs/f2fs/debug.c @@ -50,7 +50,8 @@ static void update_general_status(struct f2fs_sb_info *sbi) si->ndirty_files = sbi->ndirty_inode[FILE_INODE]; si->ndirty_all = sbi->ndirty_inode[DIRTY_META]; si->inmem_pages = get_pages(sbi, F2FS_INMEM_PAGES); - si->wb_bios = atomic_read(&sbi->nr_wb_bios); + si->nr_wb_cp_data = get_pages(sbi, F2FS_WB_CP_DATA); + si->nr_wb_data = get_pages(sbi, F2FS_WB_DATA); si->total_count = (int)sbi->user_block_count / sbi->blocks_per_seg; si->rsvd_segs = reserved_segments(sbi); si->overp_segs = overprovision_segments(sbi); @@ -74,7 +75,8 @@ static void update_general_status(struct f2fs_sb_info *sbi) si->dirty_nats = NM_I(sbi)->dirty_nat_cnt; si->sits = MAIN_SEGS(sbi); si->dirty_sits = SIT_I(sbi)->dirty_sentries; - si->fnids = NM_I(sbi)->fcnt; + si->free_nids = NM_I(sbi)->nid_cnt[FREE_NID_LIST]; + si->alloc_nids = NM_I(sbi)->nid_cnt[ALLOC_NID_LIST]; si->bg_gc = sbi->bg_gc; si->util_free = (int)(free_user_blocks(sbi) >> sbi->log_blocks_per_seg) * 100 / (int)(sbi->user_block_count >> sbi->log_blocks_per_seg) @@ -194,7 +196,9 @@ get_cache: si->cache_mem += sizeof(struct flush_cmd_control); /* free nids */ - si->cache_mem += NM_I(sbi)->fcnt * sizeof(struct free_nid); + si->cache_mem += (NM_I(sbi)->nid_cnt[FREE_NID_LIST] + + NM_I(sbi)->nid_cnt[ALLOC_NID_LIST]) * + sizeof(struct free_nid); si->cache_mem += NM_I(sbi)->nat_cnt * sizeof(struct nat_entry); si->cache_mem += NM_I(sbi)->dirty_nat_cnt * sizeof(struct nat_entry_set); @@ -310,22 +314,22 @@ static int stat_show(struct seq_file *s, void *v) seq_printf(s, " - Inner Struct Count: tree: %d(%d), node: %d\n", si->ext_tree, si->zombie_tree, si->ext_node); seq_puts(s, "\nBalancing F2FS Async:\n"); - seq_printf(s, " - inmem: %4lld, wb_bios: %4d\n", - si->inmem_pages, si->wb_bios); - seq_printf(s, " - nodes: %4lld in %4d\n", + seq_printf(s, " - inmem: %4d, wb_cp_data: %4d, wb_data: %4d\n", + si->inmem_pages, si->nr_wb_cp_data, si->nr_wb_data); + seq_printf(s, " - nodes: %4d in %4d\n", si->ndirty_node, si->node_pages); - seq_printf(s, " - dents: %4lld in dirs:%4d (%4d)\n", + seq_printf(s, " - dents: %4d in dirs:%4d (%4d)\n", si->ndirty_dent, si->ndirty_dirs, si->ndirty_all); - seq_printf(s, " - datas: %4lld in files:%4d\n", + seq_printf(s, " - datas: %4d in files:%4d\n", si->ndirty_data, si->ndirty_files); - seq_printf(s, " - meta: %4lld in %4d\n", + seq_printf(s, " - meta: %4d in %4d\n", si->ndirty_meta, si->meta_pages); - seq_printf(s, " - imeta: %4lld\n", + seq_printf(s, " - imeta: %4d\n", si->ndirty_imeta); seq_printf(s, " - NATs: %9d/%9d\n - SITs: %9d/%9d\n", si->dirty_nats, si->nats, si->dirty_sits, si->sits); - seq_printf(s, " - free_nids: %9d\n", - si->fnids); + seq_printf(s, " - free_nids: %9d, alloc_nids: %9d\n", + si->free_nids, si->alloc_nids); seq_puts(s, "\nDistribution of User Blocks:"); seq_puts(s, " [ valid | invalid | free ]\n"); seq_puts(s, " ["); @@ -373,6 +377,7 @@ static int stat_open(struct inode *inode, struct file *file) } static const struct file_operations stat_fops = { + .owner = THIS_MODULE, .open = stat_open, .read = seq_read, .llseek = seq_lseek, diff --git a/fs/f2fs/dir.c b/fs/f2fs/dir.c index 369f451..827c5da 100644 --- a/fs/f2fs/dir.c +++ b/fs/f2fs/dir.c @@ -136,7 +136,7 @@ struct f2fs_dir_entry *find_target_dentry(struct fscrypt_name *fname, /* show encrypted name */ if (fname->hash) { - if (de->hash_code == fname->hash) + if (de->hash_code == cpu_to_le32(fname->hash)) goto found; } else if (de_name.len == name->len && de->hash_code == namehash && @@ -313,7 +313,7 @@ void f2fs_set_link(struct inode *dir, struct f2fs_dir_entry *de, set_page_dirty(page); dir->i_mtime = dir->i_ctime = current_time(dir); - f2fs_mark_inode_dirty_sync(dir); + f2fs_mark_inode_dirty_sync(dir, false); f2fs_put_page(page, 1); } @@ -466,7 +466,7 @@ void update_parent_metadata(struct inode *dir, struct inode *inode, clear_inode_flag(inode, FI_NEW_INODE); } dir->i_mtime = dir->i_ctime = current_time(dir); - f2fs_mark_inode_dirty_sync(dir); + f2fs_mark_inode_dirty_sync(dir, false); if (F2FS_I(dir)->i_current_depth != current_depth) f2fs_i_depth_write(dir, current_depth); @@ -731,7 +731,7 @@ void f2fs_delete_entry(struct f2fs_dir_entry *dentry, struct page *page, set_page_dirty(page); dir->i_ctime = dir->i_mtime = current_time(dir); - f2fs_mark_inode_dirty_sync(dir); + f2fs_mark_inode_dirty_sync(dir, false); if (inode) f2fs_drop_nlink(dir, inode); @@ -742,6 +742,7 @@ void f2fs_delete_entry(struct f2fs_dir_entry *dentry, struct page *page, ClearPagePrivate(page); ClearPageUptodate(page); inode_dec_dirty_pages(dir); + remove_dirty_inode(dir); } f2fs_put_page(page, 1); } @@ -784,7 +785,7 @@ bool f2fs_empty_dir(struct inode *dir) return true; } -bool f2fs_fill_dentries(struct dir_context *ctx, struct f2fs_dentry_ptr *d, +int f2fs_fill_dentries(struct dir_context *ctx, struct f2fs_dentry_ptr *d, unsigned int start_pos, struct fscrypt_str *fstr) { unsigned char d_type = DT_UNKNOWN; @@ -819,7 +820,7 @@ bool f2fs_fill_dentries(struct dir_context *ctx, struct f2fs_dentry_ptr *d, (u32)de->hash_code, 0, &de_name, fstr); if (err) - return true; + return err; de_name = *fstr; fstr->len = save_len; @@ -827,12 +828,12 @@ bool f2fs_fill_dentries(struct dir_context *ctx, struct f2fs_dentry_ptr *d, if (!dir_emit(ctx, de_name.name, de_name.len, le32_to_cpu(de->ino), d_type)) - return true; + return 1; bit_pos += GET_DENTRY_SLOTS(le16_to_cpu(de->name_len)); ctx->pos = start_pos + bit_pos; } - return false; + return 0; } static int f2fs_readdir(struct file *file, struct dir_context *ctx) @@ -871,17 +872,21 @@ static int f2fs_readdir(struct file *file, struct dir_context *ctx) dentry_page = get_lock_data_page(inode, n, false); if (IS_ERR(dentry_page)) { err = PTR_ERR(dentry_page); - if (err == -ENOENT) + if (err == -ENOENT) { + err = 0; continue; - else + } else { goto out; + } } dentry_blk = kmap(dentry_page); make_dentry_ptr(inode, &d, (void *)dentry_blk, 1); - if (f2fs_fill_dentries(ctx, &d, n * NR_DENTRY_IN_BLOCK, &fstr)) { + err = f2fs_fill_dentries(ctx, &d, + n * NR_DENTRY_IN_BLOCK, &fstr); + if (err) { kunmap(dentry_page); f2fs_put_page(dentry_page, 1); break; @@ -891,10 +896,9 @@ static int f2fs_readdir(struct file *file, struct dir_context *ctx) kunmap(dentry_page); f2fs_put_page(dentry_page, 1); } - err = 0; out: fscrypt_fname_free_buffer(&fstr); - return err; + return err < 0 ? err : 0; } static int f2fs_dir_open(struct inode *inode, struct file *filp) diff --git a/fs/f2fs/extent_cache.c b/fs/f2fs/extent_cache.c index 2b06d4f..4db44da 100644 --- a/fs/f2fs/extent_cache.c +++ b/fs/f2fs/extent_cache.c @@ -172,7 +172,7 @@ static void __drop_largest_extent(struct inode *inode, if (fofs < largest->fofs + largest->len && fofs + len > largest->fofs) { largest->len = 0; - f2fs_mark_inode_dirty_sync(inode); + f2fs_mark_inode_dirty_sync(inode, true); } } diff --git a/fs/f2fs/f2fs.h b/fs/f2fs/f2fs.h index 2cf4f7f..2da8c3a 100644 --- a/fs/f2fs/f2fs.h +++ b/fs/f2fs/f2fs.h @@ -103,7 +103,7 @@ struct f2fs_mount_info { }; #define F2FS_FEATURE_ENCRYPT 0x0001 -#define F2FS_FEATURE_HMSMR 0x0002 +#define F2FS_FEATURE_BLKZONED 0x0002 #define F2FS_HAS_FEATURE(sb, mask) \ ((F2FS_SB(sb)->raw_super->feature & cpu_to_le32(mask)) != 0) @@ -401,6 +401,7 @@ struct f2fs_map_blocks { #define FADVISE_LOST_PINO_BIT 0x02 #define FADVISE_ENCRYPT_BIT 0x04 #define FADVISE_ENC_NAME_BIT 0x08 +#define FADVISE_KEEP_SIZE_BIT 0x10 #define file_is_cold(inode) is_file(inode, FADVISE_COLD_BIT) #define file_wrong_pino(inode) is_file(inode, FADVISE_LOST_PINO_BIT) @@ -413,6 +414,8 @@ struct f2fs_map_blocks { #define file_clear_encrypt(inode) clear_file(inode, FADVISE_ENCRYPT_BIT) #define file_enc_name(inode) is_file(inode, FADVISE_ENC_NAME_BIT) #define file_set_enc_name(inode) set_file(inode, FADVISE_ENC_NAME_BIT) +#define file_keep_isize(inode) is_file(inode, FADVISE_KEEP_SIZE_BIT) +#define file_set_keep_isize(inode) set_file(inode, FADVISE_KEEP_SIZE_BIT) #define DEF_DIR_LEVEL 0 @@ -428,7 +431,7 @@ struct f2fs_inode_info { /* Use below internally in f2fs*/ unsigned long flags; /* use to pass per-file flags */ struct rw_semaphore i_sem; /* protect fi info */ - struct percpu_counter dirty_pages; /* # of dirty pages */ + atomic_t dirty_pages; /* # of dirty pages */ f2fs_hash_t chash; /* hash value of given file name */ unsigned int clevel; /* maximum level of given file name */ nid_t i_xattr_nid; /* node id that contains xattrs */ @@ -493,20 +496,26 @@ static inline bool __is_front_mergeable(struct extent_info *cur, return __is_extent_mergeable(cur, front); } -extern void f2fs_mark_inode_dirty_sync(struct inode *); +extern void f2fs_mark_inode_dirty_sync(struct inode *, bool); static inline void __try_update_largest_extent(struct inode *inode, struct extent_tree *et, struct extent_node *en) { if (en->ei.len > et->largest.len) { et->largest = en->ei; - f2fs_mark_inode_dirty_sync(inode); + f2fs_mark_inode_dirty_sync(inode, true); } } +enum nid_list { + FREE_NID_LIST, + ALLOC_NID_LIST, + MAX_NID_LIST, +}; + struct f2fs_nm_info { block_t nat_blkaddr; /* base disk address of NAT */ nid_t max_nid; /* maximum possible node ids */ - nid_t available_nids; /* maximum available node ids */ + nid_t available_nids; /* # of available node ids */ nid_t next_scan_nid; /* the next nid to be scanned */ unsigned int ram_thresh; /* control the memory footprint */ unsigned int ra_nid_pages; /* # of nid pages to be readaheaded */ @@ -522,9 +531,9 @@ struct f2fs_nm_info { /* free node ids management */ struct radix_tree_root free_nid_root;/* root of the free_nid cache */ - struct list_head free_nid_list; /* a list for free nids */ - spinlock_t free_nid_list_lock; /* protect free nid list */ - unsigned int fcnt; /* the number of free node id */ + struct list_head nid_list[MAX_NID_LIST];/* lists for free nids */ + unsigned int nid_cnt[MAX_NID_LIST]; /* the number of free node id */ + spinlock_t nid_list_lock; /* protect nid lists ops */ struct mutex build_lock; /* lock for build free nids */ /* for checkpoint */ @@ -585,7 +594,6 @@ enum { CURSEG_WARM_NODE, /* direct node blocks of normal files */ CURSEG_COLD_NODE, /* indirect node blocks */ NO_CHECK_TYPE, - CURSEG_DIRECT_IO, /* to use for the direct IO path */ }; struct flush_cmd { @@ -649,6 +657,7 @@ struct f2fs_sm_info { * f2fs monitors the number of several block types such as on-writeback, * dirty dentry blocks, dirty node blocks, and dirty meta blocks. */ +#define WB_DATA_TYPE(p) (__is_cp_guaranteed(p) ? F2FS_WB_CP_DATA : F2FS_WB_DATA) enum count_type { F2FS_DIRTY_DENTS, F2FS_DIRTY_DATA, @@ -656,6 +665,8 @@ enum count_type { F2FS_DIRTY_META, F2FS_INMEM_PAGES, F2FS_DIRTY_IMETA, + F2FS_WB_CP_DATA, + F2FS_WB_DATA, NR_COUNT_TYPE, }; @@ -704,6 +715,20 @@ struct f2fs_bio_info { struct rw_semaphore io_rwsem; /* blocking op for bio */ }; +#define FDEV(i) (sbi->devs[i]) +#define RDEV(i) (raw_super->devs[i]) +struct f2fs_dev_info { + struct block_device *bdev; + char path[MAX_PATH_LEN]; + unsigned int total_segments; + block_t start_blk; + block_t end_blk; +#ifdef CONFIG_BLK_DEV_ZONED + unsigned int nr_blkz; /* Total number of zones */ + u8 *blkz_type; /* Array of zones type */ +#endif +}; + enum inode_type { DIR_INODE, /* for dirty dir inode */ FILE_INODE, /* for dirty regular/symlink inode */ @@ -750,6 +775,12 @@ struct f2fs_sb_info { u8 key_prefix[F2FS_KEY_DESC_PREFIX_SIZE]; u8 key_prefix_size; #endif + +#ifdef CONFIG_BLK_DEV_ZONED + unsigned int blocks_per_blkz; /* F2FS blocks per zone */ + unsigned int log_blocks_per_blkz; /* log2 F2FS blocks per zone */ +#endif + /* for node-related operations */ struct f2fs_nm_info *nm_info; /* node manager */ struct inode *node_inode; /* cache node blocks */ @@ -764,6 +795,7 @@ struct f2fs_sb_info { /* for checkpoint */ struct f2fs_checkpoint *ckpt; /* raw checkpoint pointer */ + int cur_cp_pack; /* remain current cp pack */ spinlock_t cp_lock; /* for flag in ckpt */ struct inode *meta_inode; /* cache meta blocks */ struct mutex cp_mutex; /* checkpoint procedure lock */ @@ -815,10 +847,9 @@ struct f2fs_sb_info { block_t discard_blks; /* discard command candidats */ block_t last_valid_block_count; /* for recovery */ u32 s_next_generation; /* for NFS support */ - atomic_t nr_wb_bios; /* # of writeback bios */ /* # of pages, see count_type */ - struct percpu_counter nr_pages[NR_COUNT_TYPE]; + atomic_t nr_pages[NR_COUNT_TYPE]; /* # of allocated blocks */ struct percpu_counter alloc_valid_block_count; @@ -863,6 +894,8 @@ struct f2fs_sb_info { /* For shrinker support */ struct list_head s_list; + int s_ndevs; /* number of devices */ + struct f2fs_dev_info *devs; /* for device list */ struct mutex umount_mutex; unsigned int shrinker_run_no; @@ -1105,13 +1138,6 @@ static inline void clear_ckpt_flags(struct f2fs_sb_info *sbi, unsigned int f) spin_unlock(&sbi->cp_lock); } -static inline bool f2fs_discard_en(struct f2fs_sb_info *sbi) -{ - struct request_queue *q = bdev_get_queue(sbi->sb->s_bdev); - - return blk_queue_discard(q); -} - static inline void f2fs_lock_op(struct f2fs_sb_info *sbi) { down_read(&sbi->cp_rwsem); @@ -1232,9 +1258,10 @@ static inline void dec_valid_block_count(struct f2fs_sb_info *sbi, static inline void inc_page_count(struct f2fs_sb_info *sbi, int count_type) { - percpu_counter_inc(&sbi->nr_pages[count_type]); + atomic_inc(&sbi->nr_pages[count_type]); - if (count_type == F2FS_DIRTY_DATA || count_type == F2FS_INMEM_PAGES) + if (count_type == F2FS_DIRTY_DATA || count_type == F2FS_INMEM_PAGES || + count_type == F2FS_WB_CP_DATA || count_type == F2FS_WB_DATA) return; set_sbi_flag(sbi, SBI_IS_DIRTY); @@ -1242,14 +1269,14 @@ static inline void inc_page_count(struct f2fs_sb_info *sbi, int count_type) static inline void inode_inc_dirty_pages(struct inode *inode) { - percpu_counter_inc(&F2FS_I(inode)->dirty_pages); + atomic_inc(&F2FS_I(inode)->dirty_pages); inc_page_count(F2FS_I_SB(inode), S_ISDIR(inode->i_mode) ? F2FS_DIRTY_DENTS : F2FS_DIRTY_DATA); } static inline void dec_page_count(struct f2fs_sb_info *sbi, int count_type) { - percpu_counter_dec(&sbi->nr_pages[count_type]); + atomic_dec(&sbi->nr_pages[count_type]); } static inline void inode_dec_dirty_pages(struct inode *inode) @@ -1258,19 +1285,19 @@ static inline void inode_dec_dirty_pages(struct inode *inode) !S_ISLNK(inode->i_mode)) return; - percpu_counter_dec(&F2FS_I(inode)->dirty_pages); + atomic_dec(&F2FS_I(inode)->dirty_pages); dec_page_count(F2FS_I_SB(inode), S_ISDIR(inode->i_mode) ? F2FS_DIRTY_DENTS : F2FS_DIRTY_DATA); } static inline s64 get_pages(struct f2fs_sb_info *sbi, int count_type) { - return percpu_counter_sum_positive(&sbi->nr_pages[count_type]); + return atomic_read(&sbi->nr_pages[count_type]); } -static inline s64 get_dirty_pages(struct inode *inode) +static inline int get_dirty_pages(struct inode *inode) { - return percpu_counter_sum_positive(&F2FS_I(inode)->dirty_pages); + return atomic_read(&F2FS_I(inode)->dirty_pages); } static inline int get_blocktype_secs(struct f2fs_sb_info *sbi, int block_type) @@ -1329,22 +1356,27 @@ static inline void *__bitmap_ptr(struct f2fs_sb_info *sbi, int flag) static inline block_t __start_cp_addr(struct f2fs_sb_info *sbi) { - block_t start_addr; - struct f2fs_checkpoint *ckpt = F2FS_CKPT(sbi); - unsigned long long ckpt_version = cur_cp_version(ckpt); - - start_addr = le32_to_cpu(F2FS_RAW_SUPER(sbi)->cp_blkaddr); + block_t start_addr = le32_to_cpu(F2FS_RAW_SUPER(sbi)->cp_blkaddr); - /* - * odd numbered checkpoint should at cp segment 0 - * and even segment must be at cp segment 1 - */ - if (!(ckpt_version & 1)) + if (sbi->cur_cp_pack == 2) start_addr += sbi->blocks_per_seg; + return start_addr; +} + +static inline block_t __start_cp_next_addr(struct f2fs_sb_info *sbi) +{ + block_t start_addr = le32_to_cpu(F2FS_RAW_SUPER(sbi)->cp_blkaddr); + if (sbi->cur_cp_pack == 1) + start_addr += sbi->blocks_per_seg; return start_addr; } +static inline void __set_cp_next_pack(struct f2fs_sb_info *sbi) +{ + sbi->cur_cp_pack = (sbi->cur_cp_pack == 1) ? 2 : 1; +} + static inline block_t __start_sum_addr(struct f2fs_sb_info *sbi) { return le32_to_cpu(F2FS_CKPT(sbi)->cp_pack_start_sum); @@ -1621,7 +1653,7 @@ static inline void __mark_inode_dirty_flag(struct inode *inode, return; case FI_DATA_EXIST: case FI_INLINE_DOTS: - f2fs_mark_inode_dirty_sync(inode); + f2fs_mark_inode_dirty_sync(inode, true); } } @@ -1648,7 +1680,7 @@ static inline void set_acl_inode(struct inode *inode, umode_t mode) { F2FS_I(inode)->i_acl_mode = mode; set_inode_flag(inode, FI_ACL_MODE); - f2fs_mark_inode_dirty_sync(inode); + f2fs_mark_inode_dirty_sync(inode, false); } static inline void f2fs_i_links_write(struct inode *inode, bool inc) @@ -1657,7 +1689,7 @@ static inline void f2fs_i_links_write(struct inode *inode, bool inc) inc_nlink(inode); else drop_nlink(inode); - f2fs_mark_inode_dirty_sync(inode); + f2fs_mark_inode_dirty_sync(inode, true); } static inline void f2fs_i_blocks_write(struct inode *inode, @@ -1668,7 +1700,7 @@ static inline void f2fs_i_blocks_write(struct inode *inode, inode->i_blocks = add ? inode->i_blocks + diff : inode->i_blocks - diff; - f2fs_mark_inode_dirty_sync(inode); + f2fs_mark_inode_dirty_sync(inode, true); if (clean || recover) set_inode_flag(inode, FI_AUTO_RECOVER); } @@ -1682,34 +1714,27 @@ static inline void f2fs_i_size_write(struct inode *inode, loff_t i_size) return; i_size_write(inode, i_size); - f2fs_mark_inode_dirty_sync(inode); + f2fs_mark_inode_dirty_sync(inode, true); if (clean || recover) set_inode_flag(inode, FI_AUTO_RECOVER); } -static inline bool f2fs_skip_inode_update(struct inode *inode) -{ - if (!is_inode_flag_set(inode, FI_AUTO_RECOVER)) - return false; - return F2FS_I(inode)->last_disk_size == i_size_read(inode); -} - static inline void f2fs_i_depth_write(struct inode *inode, unsigned int depth) { F2FS_I(inode)->i_current_depth = depth; - f2fs_mark_inode_dirty_sync(inode); + f2fs_mark_inode_dirty_sync(inode, true); } static inline void f2fs_i_xnid_write(struct inode *inode, nid_t xnid) { F2FS_I(inode)->i_xattr_nid = xnid; - f2fs_mark_inode_dirty_sync(inode); + f2fs_mark_inode_dirty_sync(inode, true); } static inline void f2fs_i_pino_write(struct inode *inode, nid_t pino) { F2FS_I(inode)->i_pino = pino; - f2fs_mark_inode_dirty_sync(inode); + f2fs_mark_inode_dirty_sync(inode, true); } static inline void get_inline_info(struct inode *inode, struct f2fs_inode *ri) @@ -1837,13 +1862,31 @@ static inline int is_file(struct inode *inode, int type) static inline void set_file(struct inode *inode, int type) { F2FS_I(inode)->i_advise |= type; - f2fs_mark_inode_dirty_sync(inode); + f2fs_mark_inode_dirty_sync(inode, true); } static inline void clear_file(struct inode *inode, int type) { F2FS_I(inode)->i_advise &= ~type; - f2fs_mark_inode_dirty_sync(inode); + f2fs_mark_inode_dirty_sync(inode, true); +} + +static inline bool f2fs_skip_inode_update(struct inode *inode, int dsync) +{ + if (dsync) { + struct f2fs_sb_info *sbi = F2FS_I_SB(inode); + bool ret; + + spin_lock(&sbi->inode_lock[DIRTY_META]); + ret = list_empty(&F2FS_I(inode)->gdirty_list); + spin_unlock(&sbi->inode_lock[DIRTY_META]); + return ret; + } + if (!is_inode_flag_set(inode, FI_AUTO_RECOVER) || + file_keep_isize(inode) || + i_size_read(inode) & PAGE_MASK) + return false; + return F2FS_I(inode)->last_disk_size == i_size_read(inode); } static inline int f2fs_readonly(struct super_block *sb) @@ -1955,7 +1998,7 @@ void set_de_type(struct f2fs_dir_entry *, umode_t); unsigned char get_de_type(struct f2fs_dir_entry *); struct f2fs_dir_entry *find_target_dentry(struct fscrypt_name *, f2fs_hash_t, int *, struct f2fs_dentry_ptr *); -bool f2fs_fill_dentries(struct dir_context *, struct f2fs_dentry_ptr *, +int f2fs_fill_dentries(struct dir_context *, struct f2fs_dentry_ptr *, unsigned int, struct fscrypt_str *); void do_make_empty_dir(struct inode *, struct inode *, struct f2fs_dentry_ptr *); @@ -1995,7 +2038,7 @@ static inline int f2fs_add_link(struct dentry *dentry, struct inode *inode) /* * super.c */ -int f2fs_inode_dirtied(struct inode *); +int f2fs_inode_dirtied(struct inode *, bool); void f2fs_inode_synced(struct inode *); int f2fs_commit_super(struct f2fs_sb_info *, bool); int f2fs_sync_fs(struct super_block *, int); @@ -2034,7 +2077,7 @@ void move_node_page(struct page *, int); int fsync_node_pages(struct f2fs_sb_info *, struct inode *, struct writeback_control *, bool); int sync_node_pages(struct f2fs_sb_info *, struct writeback_control *); -void build_free_nids(struct f2fs_sb_info *); +void build_free_nids(struct f2fs_sb_info *, bool); bool alloc_nid(struct f2fs_sb_info *, nid_t *); void alloc_nid_done(struct f2fs_sb_info *, nid_t); void alloc_nid_failed(struct f2fs_sb_info *, nid_t); @@ -2060,7 +2103,7 @@ void f2fs_balance_fs(struct f2fs_sb_info *, bool); void f2fs_balance_fs_bg(struct f2fs_sb_info *); int f2fs_issue_flush(struct f2fs_sb_info *); int create_flush_cmd_control(struct f2fs_sb_info *); -void destroy_flush_cmd_control(struct f2fs_sb_info *); +void destroy_flush_cmd_control(struct f2fs_sb_info *, bool); void invalidate_blocks(struct f2fs_sb_info *, block_t); bool is_checkpointed_data(struct f2fs_sb_info *, block_t); void refresh_sit_entry(struct f2fs_sb_info *, block_t, block_t); @@ -2132,12 +2175,15 @@ void f2fs_submit_merged_bio_cond(struct f2fs_sb_info *, struct inode *, void f2fs_flush_merged_bios(struct f2fs_sb_info *); int f2fs_submit_page_bio(struct f2fs_io_info *); void f2fs_submit_page_mbio(struct f2fs_io_info *); +struct block_device *f2fs_target_device(struct f2fs_sb_info *, + block_t, struct bio *); +int f2fs_target_device_index(struct f2fs_sb_info *, block_t); void set_data_blkaddr(struct dnode_of_data *); void f2fs_update_data_blkaddr(struct dnode_of_data *, block_t); int reserve_new_blocks(struct dnode_of_data *, blkcnt_t); int reserve_new_block(struct dnode_of_data *); int f2fs_get_block(struct dnode_of_data *, pgoff_t); -ssize_t f2fs_preallocate_blocks(struct kiocb *, struct iov_iter *); +int f2fs_preallocate_blocks(struct kiocb *, struct iov_iter *); int f2fs_reserve_block(struct dnode_of_data *, pgoff_t); struct page *get_read_data_page(struct inode *, pgoff_t, int, bool); struct page *find_data_page(struct inode *, pgoff_t); @@ -2160,7 +2206,7 @@ int f2fs_migrate_page(struct address_space *, struct page *, struct page *, int start_gc_thread(struct f2fs_sb_info *); void stop_gc_thread(struct f2fs_sb_info *); block_t start_bidx_of_node(unsigned int, struct inode *); -int f2fs_gc(struct f2fs_sb_info *, bool); +int f2fs_gc(struct f2fs_sb_info *, bool, bool); void build_gc_manager(struct f2fs_sb_info *); /* @@ -2181,12 +2227,12 @@ struct f2fs_stat_info { unsigned long long hit_largest, hit_cached, hit_rbtree; unsigned long long hit_total, total_ext; int ext_tree, zombie_tree, ext_node; - s64 ndirty_node, ndirty_dent, ndirty_meta, ndirty_data, ndirty_imeta; - s64 inmem_pages; + int ndirty_node, ndirty_dent, ndirty_meta, ndirty_data, ndirty_imeta; + int inmem_pages; unsigned int ndirty_dirs, ndirty_files, ndirty_all; - int nats, dirty_nats, sits, dirty_sits, fnids; + int nats, dirty_nats, sits, dirty_sits, free_nids, alloc_nids; int total_count, utilization; - int bg_gc, wb_bios; + int bg_gc, nr_wb_cp_data, nr_wb_data; int inline_xattr, inline_inode, inline_dir, orphans; unsigned int valid_count, valid_node_count, valid_inode_count, discard_blks; unsigned int bimodal, avg_vblocks; @@ -2412,9 +2458,30 @@ static inline int f2fs_sb_has_crypto(struct super_block *sb) return F2FS_HAS_FEATURE(sb, F2FS_FEATURE_ENCRYPT); } -static inline int f2fs_sb_mounted_hmsmr(struct super_block *sb) +static inline int f2fs_sb_mounted_blkzoned(struct super_block *sb) +{ + return F2FS_HAS_FEATURE(sb, F2FS_FEATURE_BLKZONED); +} + +#ifdef CONFIG_BLK_DEV_ZONED +static inline int get_blkz_type(struct f2fs_sb_info *sbi, + struct block_device *bdev, block_t blkaddr) +{ + unsigned int zno = blkaddr >> sbi->log_blocks_per_blkz; + int i; + + for (i = 0; i < sbi->s_ndevs; i++) + if (FDEV(i).bdev == bdev) + return FDEV(i).blkz_type[zno]; + return -EINVAL; +} +#endif + +static inline bool f2fs_discard_en(struct f2fs_sb_info *sbi) { - return F2FS_HAS_FEATURE(sb, F2FS_FEATURE_HMSMR); + struct request_queue *q = bdev_get_queue(sbi->sb->s_bdev); + + return blk_queue_discard(q) || f2fs_sb_mounted_blkzoned(sbi->sb); } static inline void set_opt_mode(struct f2fs_sb_info *sbi, unsigned int mt) @@ -2453,8 +2520,8 @@ static inline bool f2fs_may_encrypt(struct inode *inode) #define fscrypt_pullback_bio_page fscrypt_notsupp_pullback_bio_page #define fscrypt_restore_control_page fscrypt_notsupp_restore_control_page #define fscrypt_zeroout_range fscrypt_notsupp_zeroout_range -#define fscrypt_process_policy fscrypt_notsupp_process_policy -#define fscrypt_get_policy fscrypt_notsupp_get_policy +#define fscrypt_ioctl_set_policy fscrypt_notsupp_ioctl_set_policy +#define fscrypt_ioctl_get_policy fscrypt_notsupp_ioctl_get_policy #define fscrypt_has_permitted_context fscrypt_notsupp_has_permitted_context #define fscrypt_inherit_context fscrypt_notsupp_inherit_context #define fscrypt_get_encryption_info fscrypt_notsupp_get_encryption_info diff --git a/fs/f2fs/file.c b/fs/f2fs/file.c index c786507..49f10dc 100644 --- a/fs/f2fs/file.c +++ b/fs/f2fs/file.c @@ -94,8 +94,6 @@ mapped: if (f2fs_encrypted_inode(inode) && S_ISREG(inode->i_mode)) f2fs_wait_on_encrypted_page_writeback(sbi, dn.data_blkaddr); - /* if gced page is attached, don't write to cold segment */ - clear_cold_data(page); out: sb_end_pagefault(inode->i_sb); f2fs_update_time(sbi, REQ_TIME); @@ -210,7 +208,7 @@ static int f2fs_do_sync_file(struct file *file, loff_t start, loff_t end, } /* if the inode is dirty, let's recover all the time */ - if (!datasync && !f2fs_skip_inode_update(inode)) { + if (!f2fs_skip_inode_update(inode, datasync)) { f2fs_write_inode(inode, NULL); goto go_write; } @@ -264,7 +262,7 @@ sync_nodes: } if (need_inode_block_update(sbi, ino)) { - f2fs_mark_inode_dirty_sync(inode); + f2fs_mark_inode_dirty_sync(inode, true); f2fs_write_inode(inode, NULL); goto sync_nodes; } @@ -632,7 +630,7 @@ int f2fs_truncate(struct inode *inode) return err; inode->i_mtime = inode->i_ctime = current_time(inode); - f2fs_mark_inode_dirty_sync(inode); + f2fs_mark_inode_dirty_sync(inode, false); return 0; } @@ -679,6 +677,7 @@ int f2fs_setattr(struct dentry *dentry, struct iattr *attr) { struct inode *inode = d_inode(dentry); int err; + bool size_changed = false; err = setattr_prepare(dentry, attr); if (err) @@ -694,7 +693,6 @@ int f2fs_setattr(struct dentry *dentry, struct iattr *attr) err = f2fs_truncate(inode); if (err) return err; - f2fs_balance_fs(F2FS_I_SB(inode), true); } else { /* * do not trim all blocks after i_size if target size is @@ -710,6 +708,8 @@ int f2fs_setattr(struct dentry *dentry, struct iattr *attr) } inode->i_mtime = inode->i_ctime = current_time(inode); } + + size_changed = true; } __setattr_copy(inode, attr); @@ -722,7 +722,12 @@ int f2fs_setattr(struct dentry *dentry, struct iattr *attr) } } - f2fs_mark_inode_dirty_sync(inode); + /* file size may changed here */ + f2fs_mark_inode_dirty_sync(inode, size_changed); + + /* inode change will produce dirty node pages flushed by checkpoint */ + f2fs_balance_fs(F2FS_I_SB(inode), true); + return err; } @@ -967,7 +972,7 @@ static int __clone_blkaddrs(struct inode *src_inode, struct inode *dst_inode, new_size = (dst + i) << PAGE_SHIFT; if (dst_inode->i_size < new_size) f2fs_i_size_write(dst_inode, new_size); - } while ((do_replace[i] || blkaddr[i] == NULL_ADDR) && --ilen); + } while (--ilen && (do_replace[i] || blkaddr[i] == NULL_ADDR)); f2fs_put_dnode(&dn); } else { @@ -1218,6 +1223,9 @@ static int f2fs_zero_range(struct inode *inode, loff_t offset, loff_t len, ret = f2fs_do_zero_range(&dn, index, end); f2fs_put_dnode(&dn); f2fs_unlock_op(sbi); + + f2fs_balance_fs(sbi, dn.node_changed); + if (ret) goto out; @@ -1313,15 +1321,15 @@ static int expand_inode_data(struct inode *inode, loff_t offset, pgoff_t pg_end; loff_t new_size = i_size_read(inode); loff_t off_end; - int ret; + int err; - ret = inode_newsize_ok(inode, (len + offset)); - if (ret) - return ret; + err = inode_newsize_ok(inode, (len + offset)); + if (err) + return err; - ret = f2fs_convert_inline_inode(inode); - if (ret) - return ret; + err = f2fs_convert_inline_inode(inode); + if (err) + return err; f2fs_balance_fs(sbi, true); @@ -1333,12 +1341,12 @@ static int expand_inode_data(struct inode *inode, loff_t offset, if (off_end) map.m_len++; - ret = f2fs_map_blocks(inode, &map, 1, F2FS_GET_BLOCK_PRE_AIO); - if (ret) { + err = f2fs_map_blocks(inode, &map, 1, F2FS_GET_BLOCK_PRE_AIO); + if (err) { pgoff_t last_off; if (!map.m_len) - return ret; + return err; last_off = map.m_lblk + map.m_len - 1; @@ -1352,7 +1360,7 @@ static int expand_inode_data(struct inode *inode, loff_t offset, if (!(mode & FALLOC_FL_KEEP_SIZE) && i_size_read(inode) < new_size) f2fs_i_size_write(inode, new_size); - return ret; + return err; } static long f2fs_fallocate(struct file *file, int mode, @@ -1393,7 +1401,9 @@ static long f2fs_fallocate(struct file *file, int mode, if (!ret) { inode->i_mtime = inode->i_ctime = current_time(inode); - f2fs_mark_inode_dirty_sync(inode); + f2fs_mark_inode_dirty_sync(inode, false); + if (mode & FALLOC_FL_KEEP_SIZE) + file_set_keep_isize(inode); f2fs_update_time(F2FS_I_SB(inode), REQ_TIME); } @@ -1526,7 +1536,7 @@ static int f2fs_ioc_start_atomic_write(struct file *filp) goto out; f2fs_msg(F2FS_I_SB(inode)->sb, KERN_WARNING, - "Unexpected flush for atomic writes: ino=%lu, npages=%lld", + "Unexpected flush for atomic writes: ino=%lu, npages=%u", inode->i_ino, get_dirty_pages(inode)); ret = filemap_write_and_wait_range(inode->i_mapping, 0, LLONG_MAX); if (ret) @@ -1752,31 +1762,16 @@ static bool uuid_is_nonzero(__u8 u[16]) static int f2fs_ioc_set_encryption_policy(struct file *filp, unsigned long arg) { - struct fscrypt_policy policy; struct inode *inode = file_inode(filp); - if (copy_from_user(&policy, (struct fscrypt_policy __user *)arg, - sizeof(policy))) - return -EFAULT; - f2fs_update_time(F2FS_I_SB(inode), REQ_TIME); - return fscrypt_process_policy(filp, &policy); + return fscrypt_ioctl_set_policy(filp, (const void __user *)arg); } static int f2fs_ioc_get_encryption_policy(struct file *filp, unsigned long arg) { - struct fscrypt_policy policy; - struct inode *inode = file_inode(filp); - int err; - - err = fscrypt_get_policy(inode, &policy); - if (err) - return err; - - if (copy_to_user((struct fscrypt_policy __user *)arg, &policy, sizeof(policy))) - return -EFAULT; - return 0; + return fscrypt_ioctl_get_policy(filp, (void __user *)arg); } static int f2fs_ioc_get_encryption_pwsalt(struct file *filp, unsigned long arg) @@ -1842,7 +1837,7 @@ static int f2fs_ioc_gc(struct file *filp, unsigned long arg) mutex_lock(&sbi->gc_mutex); } - ret = f2fs_gc(sbi, sync); + ret = f2fs_gc(sbi, sync, true); out: mnt_drop_write_file(filp); return ret; @@ -2256,12 +2251,15 @@ static ssize_t f2fs_file_write_iter(struct kiocb *iocb, struct iov_iter *from) inode_lock(inode); ret = generic_write_checks(iocb, from); if (ret > 0) { - ret = f2fs_preallocate_blocks(iocb, from); - if (!ret) { - blk_start_plug(&plug); - ret = __generic_file_write_iter(iocb, from); - blk_finish_plug(&plug); + int err = f2fs_preallocate_blocks(iocb, from); + + if (err) { + inode_unlock(inode); + return err; } + blk_start_plug(&plug); + ret = __generic_file_write_iter(iocb, from); + blk_finish_plug(&plug); } inode_unlock(inode); diff --git a/fs/f2fs/gc.c b/fs/f2fs/gc.c index fcca12b..88bfc3d 100644 --- a/fs/f2fs/gc.c +++ b/fs/f2fs/gc.c @@ -82,7 +82,7 @@ static int gc_thread_func(void *data) stat_inc_bggc_count(sbi); /* if return value is not zero, no victim was selected */ - if (f2fs_gc(sbi, test_opt(sbi, FORCE_FG_GC))) + if (f2fs_gc(sbi, test_opt(sbi, FORCE_FG_GC), true)) wait_ms = gc_th->no_gc_sleep_time; trace_f2fs_background_gc(sbi->sb, wait_ms, @@ -544,7 +544,8 @@ static bool is_alive(struct f2fs_sb_info *sbi, struct f2fs_summary *sum, return true; } -static void move_encrypted_block(struct inode *inode, block_t bidx) +static void move_encrypted_block(struct inode *inode, block_t bidx, + unsigned int segno, int off) { struct f2fs_io_info fio = { .sbi = F2FS_I_SB(inode), @@ -565,6 +566,9 @@ static void move_encrypted_block(struct inode *inode, block_t bidx) if (!page) return; + if (!check_valid_map(F2FS_I_SB(inode), segno, off)) + goto out; + set_new_dnode(&dn, inode, NULL, NULL, 0); err = get_dnode_of_data(&dn, bidx, LOOKUP_NODE); if (err) @@ -645,7 +649,8 @@ out: f2fs_put_page(page, 1); } -static void move_data_page(struct inode *inode, block_t bidx, int gc_type) +static void move_data_page(struct inode *inode, block_t bidx, int gc_type, + unsigned int segno, int off) { struct page *page; @@ -653,6 +658,9 @@ static void move_data_page(struct inode *inode, block_t bidx, int gc_type) if (IS_ERR(page)) return; + if (!check_valid_map(F2FS_I_SB(inode), segno, off)) + goto out; + if (gc_type == BG_GC) { if (PageWriteback(page)) goto out; @@ -673,8 +681,10 @@ static void move_data_page(struct inode *inode, block_t bidx, int gc_type) retry: set_page_dirty(page); f2fs_wait_on_page_writeback(page, DATA, true); - if (clear_page_dirty_for_io(page)) + if (clear_page_dirty_for_io(page)) { inode_dec_dirty_pages(inode); + remove_dirty_inode(inode); + } set_cold_data(page); @@ -683,8 +693,6 @@ retry: congestion_wait(BLK_RW_ASYNC, HZ/50); goto retry; } - - clear_cold_data(page); } out: f2fs_put_page(page, 1); @@ -794,9 +802,9 @@ next_step: start_bidx = start_bidx_of_node(nofs, inode) + ofs_in_node; if (f2fs_encrypted_inode(inode) && S_ISREG(inode->i_mode)) - move_encrypted_block(inode, start_bidx); + move_encrypted_block(inode, start_bidx, segno, off); else - move_data_page(inode, start_bidx, gc_type); + move_data_page(inode, start_bidx, gc_type, segno, off); if (locked) { up_write(&fi->dio_rwsem[WRITE]); @@ -899,7 +907,7 @@ next: return sec_freed; } -int f2fs_gc(struct f2fs_sb_info *sbi, bool sync) +int f2fs_gc(struct f2fs_sb_info *sbi, bool sync, bool background) { unsigned int segno; int gc_type = sync ? FG_GC : BG_GC; @@ -940,6 +948,9 @@ gc_more: if (ret) goto stop; } + } else if (gc_type == BG_GC && !background) { + /* f2fs_balance_fs doesn't need to do BG_GC in critical path. */ + goto stop; } if (segno == NULL_SEGNO && !__get_victim(sbi, &segno, gc_type)) diff --git a/fs/f2fs/inline.c b/fs/f2fs/inline.c index 2e7f54c..e32a9e5 100644 --- a/fs/f2fs/inline.c +++ b/fs/f2fs/inline.c @@ -137,8 +137,10 @@ int f2fs_convert_inline_page(struct dnode_of_data *dn, struct page *page) fio.old_blkaddr = dn->data_blkaddr; write_data_page(dn, &fio); f2fs_wait_on_page_writeback(page, DATA, true); - if (dirty) + if (dirty) { inode_dec_dirty_pages(dn->inode); + remove_dirty_inode(dn->inode); + } /* this converted inline_data should be recovered. */ set_inode_flag(dn->inode, FI_APPEND_WRITE); @@ -419,7 +421,7 @@ static int f2fs_add_inline_entries(struct inode *dir, } new_name.name = d.filename[bit_pos]; - new_name.len = de->name_len; + new_name.len = le16_to_cpu(de->name_len); ino = le32_to_cpu(de->ino); fake_mode = get_de_type(de) << S_SHIFT; @@ -573,7 +575,7 @@ void f2fs_delete_inline_entry(struct f2fs_dir_entry *dentry, struct page *page, f2fs_put_page(page, 1); dir->i_ctime = dir->i_mtime = current_time(dir); - f2fs_mark_inode_dirty_sync(dir); + f2fs_mark_inode_dirty_sync(dir, false); if (inode) f2fs_drop_nlink(dir, inode); @@ -610,6 +612,7 @@ int f2fs_read_inline_dir(struct file *file, struct dir_context *ctx, struct f2fs_inline_dentry *inline_dentry = NULL; struct page *ipage = NULL; struct f2fs_dentry_ptr d; + int err; if (ctx->pos == NR_INLINE_DENTRY) return 0; @@ -622,11 +625,12 @@ int f2fs_read_inline_dir(struct file *file, struct dir_context *ctx, make_dentry_ptr(inode, &d, (void *)inline_dentry, 2); - if (!f2fs_fill_dentries(ctx, &d, 0, fstr)) + err = f2fs_fill_dentries(ctx, &d, 0, fstr); + if (!err) ctx->pos = NR_INLINE_DENTRY; f2fs_put_page(ipage, 1); - return 0; + return err < 0 ? err : 0; } int f2fs_inline_data_fiemap(struct inode *inode, diff --git a/fs/f2fs/inode.c b/fs/f2fs/inode.c index d736989..af06bda 100644 --- a/fs/f2fs/inode.c +++ b/fs/f2fs/inode.c @@ -19,10 +19,11 @@ #include <trace/events/f2fs.h> -void f2fs_mark_inode_dirty_sync(struct inode *inode) +void f2fs_mark_inode_dirty_sync(struct inode *inode, bool sync) { - if (f2fs_inode_dirtied(inode)) + if (f2fs_inode_dirtied(inode, sync)) return; + mark_inode_dirty_sync(inode); } @@ -43,7 +44,7 @@ void f2fs_set_inode_flags(struct inode *inode) new_fl |= S_DIRSYNC; inode_set_flags(inode, new_fl, S_SYNC|S_APPEND|S_IMMUTABLE|S_NOATIME|S_DIRSYNC); - f2fs_mark_inode_dirty_sync(inode); + f2fs_mark_inode_dirty_sync(inode, false); } static void __get_inode_rdev(struct inode *inode, struct f2fs_inode *ri) @@ -252,6 +253,7 @@ retry: int update_inode(struct inode *inode, struct page *node_page) { struct f2fs_inode *ri; + struct extent_tree *et = F2FS_I(inode)->extent_tree; f2fs_inode_synced(inode); @@ -267,11 +269,13 @@ int update_inode(struct inode *inode, struct page *node_page) ri->i_size = cpu_to_le64(i_size_read(inode)); ri->i_blocks = cpu_to_le64(inode->i_blocks); - if (F2FS_I(inode)->extent_tree) - set_raw_extent(&F2FS_I(inode)->extent_tree->largest, - &ri->i_ext); - else + if (et) { + read_lock(&et->lock); + set_raw_extent(&et->largest, &ri->i_ext); + read_unlock(&et->lock); + } else { memset(&ri->i_ext, 0, sizeof(ri->i_ext)); + } set_raw_inline(inode, ri); ri->i_atime = cpu_to_le64(inode->i_atime.tv_sec); @@ -335,7 +339,7 @@ int f2fs_write_inode(struct inode *inode, struct writeback_control *wbc) * We need to balance fs here to prevent from producing dirty node pages * during the urgent cleaning time when runing out of free sections. */ - if (update_inode_page(inode)) + if (update_inode_page(inode) && wbc && wbc->nr_to_write) f2fs_balance_fs(sbi, true); return 0; } @@ -373,6 +377,9 @@ void f2fs_evict_inode(struct inode *inode) goto no_delete; #endif + remove_ino_entry(sbi, inode->i_ino, APPEND_INO); + remove_ino_entry(sbi, inode->i_ino, UPDATE_INO); + sb_start_intwrite(inode->i_sb); set_inode_flag(inode, FI_NO_ALLOC); i_size_write(inode, 0); @@ -384,6 +391,8 @@ retry: f2fs_lock_op(sbi); err = remove_inode_page(inode); f2fs_unlock_op(sbi); + if (err == -ENOENT) + err = 0; } /* give more chances, if ENOMEM case */ @@ -403,10 +412,12 @@ no_delete: invalidate_mapping_pages(NODE_MAPPING(sbi), inode->i_ino, inode->i_ino); if (xnid) invalidate_mapping_pages(NODE_MAPPING(sbi), xnid, xnid); - if (is_inode_flag_set(inode, FI_APPEND_WRITE)) - add_ino_entry(sbi, inode->i_ino, APPEND_INO); - if (is_inode_flag_set(inode, FI_UPDATE_WRITE)) - add_ino_entry(sbi, inode->i_ino, UPDATE_INO); + if (inode->i_nlink) { + if (is_inode_flag_set(inode, FI_APPEND_WRITE)) + add_ino_entry(sbi, inode->i_ino, APPEND_INO); + if (is_inode_flag_set(inode, FI_UPDATE_WRITE)) + add_ino_entry(sbi, inode->i_ino, UPDATE_INO); + } if (is_inode_flag_set(inode, FI_FREE_NID)) { alloc_nid_failed(sbi, inode->i_ino); clear_inode_flag(inode, FI_FREE_NID); @@ -424,6 +435,18 @@ void handle_failed_inode(struct inode *inode) struct f2fs_sb_info *sbi = F2FS_I_SB(inode); struct node_info ni; + /* + * clear nlink of inode in order to release resource of inode + * immediately. + */ + clear_nlink(inode); + + /* + * we must call this to avoid inode being remained as dirty, resulting + * in a panic when flushing dirty inodes in gdirty_list. + */ + update_inode_page(inode); + /* don't make bad inode, since it becomes a regular file. */ unlock_new_inode(inode); diff --git a/fs/f2fs/namei.c b/fs/f2fs/namei.c index 489fa0d..db33b56 100644 --- a/fs/f2fs/namei.c +++ b/fs/f2fs/namei.c @@ -778,7 +778,7 @@ static int f2fs_rename(struct inode *old_dir, struct dentry *old_dentry, up_write(&F2FS_I(old_inode)->i_sem); old_inode->i_ctime = current_time(old_inode); - f2fs_mark_inode_dirty_sync(old_inode); + f2fs_mark_inode_dirty_sync(old_inode, false); f2fs_delete_entry(old_entry, old_page, old_dir, NULL); @@ -938,7 +938,7 @@ static int f2fs_cross_rename(struct inode *old_dir, struct dentry *old_dentry, f2fs_i_links_write(old_dir, old_nlink > 0); up_write(&F2FS_I(old_dir)->i_sem); } - f2fs_mark_inode_dirty_sync(old_dir); + f2fs_mark_inode_dirty_sync(old_dir, false); /* update directory entry info of new dir inode */ f2fs_set_link(new_dir, new_entry, new_page, old_inode); @@ -953,7 +953,7 @@ static int f2fs_cross_rename(struct inode *old_dir, struct dentry *old_dentry, f2fs_i_links_write(new_dir, new_nlink > 0); up_write(&F2FS_I(new_dir)->i_sem); } - f2fs_mark_inode_dirty_sync(new_dir); + f2fs_mark_inode_dirty_sync(new_dir, false); f2fs_unlock_op(sbi); diff --git a/fs/f2fs/node.c b/fs/f2fs/node.c index d1e29de..b9078fd 100644 --- a/fs/f2fs/node.c +++ b/fs/f2fs/node.c @@ -45,8 +45,8 @@ bool available_free_memory(struct f2fs_sb_info *sbi, int type) * give 25%, 25%, 50%, 50%, 50% memory for each components respectively */ if (type == FREE_NIDS) { - mem_size = (nm_i->fcnt * sizeof(struct free_nid)) >> - PAGE_SHIFT; + mem_size = (nm_i->nid_cnt[FREE_NID_LIST] * + sizeof(struct free_nid)) >> PAGE_SHIFT; res = mem_size < ((avail_ram * nm_i->ram_thresh / 100) >> 2); } else if (type == NAT_ENTRIES) { mem_size = (nm_i->nat_cnt * sizeof(struct nat_entry)) >> @@ -270,8 +270,9 @@ static void cache_nat_entry(struct f2fs_sb_info *sbi, nid_t nid, e = grab_nat_entry(nm_i, nid); node_info_from_raw_nat(&e->ni, ne); } else { - f2fs_bug_on(sbi, nat_get_ino(e) != ne->ino || - nat_get_blkaddr(e) != ne->block_addr || + f2fs_bug_on(sbi, nat_get_ino(e) != le32_to_cpu(ne->ino) || + nat_get_blkaddr(e) != + le32_to_cpu(ne->block_addr) || nat_get_version(e) != ne->version); } } @@ -1204,6 +1205,7 @@ static void flush_inline_data(struct f2fs_sb_info *sbi, nid_t ino) ret = f2fs_write_inline_data(inode, page); inode_dec_dirty_pages(inode); + remove_dirty_inode(inode); if (ret) set_page_dirty(page); page_out: @@ -1338,7 +1340,8 @@ retry: if (unlikely(f2fs_cp_error(sbi))) { f2fs_put_page(last_page, 0); pagevec_release(&pvec); - return -EIO; + ret = -EIO; + goto out; } if (!IS_DNODE(page) || !is_cold_node(page)) @@ -1407,11 +1410,12 @@ continue_unlock: "Retry to write fsync mark: ino=%u, idx=%lx", ino, last_page->index); lock_page(last_page); + f2fs_wait_on_page_writeback(last_page, NODE, true); set_page_dirty(last_page); unlock_page(last_page); goto retry; } - +out: if (nwritten) f2fs_submit_merged_bio_cond(sbi, NULL, NULL, ino, NODE, WRITE); return ret ? -EIO: 0; @@ -1692,11 +1696,35 @@ static struct free_nid *__lookup_free_nid_list(struct f2fs_nm_info *nm_i, return radix_tree_lookup(&nm_i->free_nid_root, n); } -static void __del_from_free_nid_list(struct f2fs_nm_info *nm_i, - struct free_nid *i) +static int __insert_nid_to_list(struct f2fs_sb_info *sbi, + struct free_nid *i, enum nid_list list, bool new) { + struct f2fs_nm_info *nm_i = NM_I(sbi); + + if (new) { + int err = radix_tree_insert(&nm_i->free_nid_root, i->nid, i); + if (err) + return err; + } + + f2fs_bug_on(sbi, list == FREE_NID_LIST ? i->state != NID_NEW : + i->state != NID_ALLOC); + nm_i->nid_cnt[list]++; + list_add_tail(&i->list, &nm_i->nid_list[list]); + return 0; +} + +static void __remove_nid_from_list(struct f2fs_sb_info *sbi, + struct free_nid *i, enum nid_list list, bool reuse) +{ + struct f2fs_nm_info *nm_i = NM_I(sbi); + + f2fs_bug_on(sbi, list == FREE_NID_LIST ? i->state != NID_NEW : + i->state != NID_ALLOC); + nm_i->nid_cnt[list]--; list_del(&i->list); - radix_tree_delete(&nm_i->free_nid_root, i->nid); + if (!reuse) + radix_tree_delete(&nm_i->free_nid_root, i->nid); } static int add_free_nid(struct f2fs_sb_info *sbi, nid_t nid, bool build) @@ -1704,9 +1732,7 @@ static int add_free_nid(struct f2fs_sb_info *sbi, nid_t nid, bool build) struct f2fs_nm_info *nm_i = NM_I(sbi); struct free_nid *i; struct nat_entry *ne; - - if (!available_free_memory(sbi, FREE_NIDS)) - return -1; + int err; /* 0 nid should not be used */ if (unlikely(nid == 0)) @@ -1729,33 +1755,30 @@ static int add_free_nid(struct f2fs_sb_info *sbi, nid_t nid, bool build) return 0; } - spin_lock(&nm_i->free_nid_list_lock); - if (radix_tree_insert(&nm_i->free_nid_root, i->nid, i)) { - spin_unlock(&nm_i->free_nid_list_lock); - radix_tree_preload_end(); + spin_lock(&nm_i->nid_list_lock); + err = __insert_nid_to_list(sbi, i, FREE_NID_LIST, true); + spin_unlock(&nm_i->nid_list_lock); + radix_tree_preload_end(); + if (err) { kmem_cache_free(free_nid_slab, i); return 0; } - list_add_tail(&i->list, &nm_i->free_nid_list); - nm_i->fcnt++; - spin_unlock(&nm_i->free_nid_list_lock); - radix_tree_preload_end(); return 1; } -static void remove_free_nid(struct f2fs_nm_info *nm_i, nid_t nid) +static void remove_free_nid(struct f2fs_sb_info *sbi, nid_t nid) { + struct f2fs_nm_info *nm_i = NM_I(sbi); struct free_nid *i; bool need_free = false; - spin_lock(&nm_i->free_nid_list_lock); + spin_lock(&nm_i->nid_list_lock); i = __lookup_free_nid_list(nm_i, nid); if (i && i->state == NID_NEW) { - __del_from_free_nid_list(nm_i, i); - nm_i->fcnt--; + __remove_nid_from_list(sbi, i, FREE_NID_LIST, false); need_free = true; } - spin_unlock(&nm_i->free_nid_list_lock); + spin_unlock(&nm_i->nid_list_lock); if (need_free) kmem_cache_free(free_nid_slab, i); @@ -1778,14 +1801,12 @@ static void scan_nat_page(struct f2fs_sb_info *sbi, blk_addr = le32_to_cpu(nat_blk->entries[i].block_addr); f2fs_bug_on(sbi, blk_addr == NEW_ADDR); - if (blk_addr == NULL_ADDR) { - if (add_free_nid(sbi, start_nid, true) < 0) - break; - } + if (blk_addr == NULL_ADDR) + add_free_nid(sbi, start_nid, true); } } -void build_free_nids(struct f2fs_sb_info *sbi) +static void __build_free_nids(struct f2fs_sb_info *sbi, bool sync) { struct f2fs_nm_info *nm_i = NM_I(sbi); struct curseg_info *curseg = CURSEG_I(sbi, CURSEG_HOT_DATA); @@ -1794,7 +1815,10 @@ void build_free_nids(struct f2fs_sb_info *sbi) nid_t nid = nm_i->next_scan_nid; /* Enough entries */ - if (nm_i->fcnt >= NAT_ENTRY_PER_BLOCK) + if (nm_i->nid_cnt[FREE_NID_LIST] >= NAT_ENTRY_PER_BLOCK) + return; + + if (!sync && !available_free_memory(sbi, FREE_NIDS)) return; /* readahead nat pages to be scanned */ @@ -1830,7 +1854,7 @@ void build_free_nids(struct f2fs_sb_info *sbi) if (addr == NULL_ADDR) add_free_nid(sbi, nid, true); else - remove_free_nid(nm_i, nid); + remove_free_nid(sbi, nid); } up_read(&curseg->journal_rwsem); up_read(&nm_i->nat_tree_lock); @@ -1839,6 +1863,13 @@ void build_free_nids(struct f2fs_sb_info *sbi) nm_i->ra_nid_pages, META_NAT, false); } +void build_free_nids(struct f2fs_sb_info *sbi, bool sync) +{ + mutex_lock(&NM_I(sbi)->build_lock); + __build_free_nids(sbi, sync); + mutex_unlock(&NM_I(sbi)->build_lock); +} + /* * If this function returns success, caller can obtain a new nid * from second parameter of this function. @@ -1853,31 +1884,31 @@ retry: if (time_to_inject(sbi, FAULT_ALLOC_NID)) return false; #endif - if (unlikely(sbi->total_valid_node_count + 1 > nm_i->available_nids)) - return false; + spin_lock(&nm_i->nid_list_lock); - spin_lock(&nm_i->free_nid_list_lock); + if (unlikely(nm_i->available_nids == 0)) { + spin_unlock(&nm_i->nid_list_lock); + return false; + } /* We should not use stale free nids created by build_free_nids */ - if (nm_i->fcnt && !on_build_free_nids(nm_i)) { - f2fs_bug_on(sbi, list_empty(&nm_i->free_nid_list)); - list_for_each_entry(i, &nm_i->free_nid_list, list) - if (i->state == NID_NEW) - break; - - f2fs_bug_on(sbi, i->state != NID_NEW); + if (nm_i->nid_cnt[FREE_NID_LIST] && !on_build_free_nids(nm_i)) { + f2fs_bug_on(sbi, list_empty(&nm_i->nid_list[FREE_NID_LIST])); + i = list_first_entry(&nm_i->nid_list[FREE_NID_LIST], + struct free_nid, list); *nid = i->nid; + + __remove_nid_from_list(sbi, i, FREE_NID_LIST, true); i->state = NID_ALLOC; - nm_i->fcnt--; - spin_unlock(&nm_i->free_nid_list_lock); + __insert_nid_to_list(sbi, i, ALLOC_NID_LIST, false); + nm_i->available_nids--; + spin_unlock(&nm_i->nid_list_lock); return true; } - spin_unlock(&nm_i->free_nid_list_lock); + spin_unlock(&nm_i->nid_list_lock); /* Let's scan nat pages and its caches to get free nids */ - mutex_lock(&nm_i->build_lock); - build_free_nids(sbi); - mutex_unlock(&nm_i->build_lock); + build_free_nids(sbi, true); goto retry; } @@ -1889,11 +1920,11 @@ void alloc_nid_done(struct f2fs_sb_info *sbi, nid_t nid) struct f2fs_nm_info *nm_i = NM_I(sbi); struct free_nid *i; - spin_lock(&nm_i->free_nid_list_lock); + spin_lock(&nm_i->nid_list_lock); i = __lookup_free_nid_list(nm_i, nid); - f2fs_bug_on(sbi, !i || i->state != NID_ALLOC); - __del_from_free_nid_list(nm_i, i); - spin_unlock(&nm_i->free_nid_list_lock); + f2fs_bug_on(sbi, !i); + __remove_nid_from_list(sbi, i, ALLOC_NID_LIST, false); + spin_unlock(&nm_i->nid_list_lock); kmem_cache_free(free_nid_slab, i); } @@ -1910,17 +1941,22 @@ void alloc_nid_failed(struct f2fs_sb_info *sbi, nid_t nid) if (!nid) return; - spin_lock(&nm_i->free_nid_list_lock); + spin_lock(&nm_i->nid_list_lock); i = __lookup_free_nid_list(nm_i, nid); - f2fs_bug_on(sbi, !i || i->state != NID_ALLOC); + f2fs_bug_on(sbi, !i); + if (!available_free_memory(sbi, FREE_NIDS)) { - __del_from_free_nid_list(nm_i, i); + __remove_nid_from_list(sbi, i, ALLOC_NID_LIST, false); need_free = true; } else { + __remove_nid_from_list(sbi, i, ALLOC_NID_LIST, true); i->state = NID_NEW; - nm_i->fcnt++; + __insert_nid_to_list(sbi, i, FREE_NID_LIST, false); } - spin_unlock(&nm_i->free_nid_list_lock); + + nm_i->available_nids++; + + spin_unlock(&nm_i->nid_list_lock); if (need_free) kmem_cache_free(free_nid_slab, i); @@ -1932,24 +1968,24 @@ int try_to_free_nids(struct f2fs_sb_info *sbi, int nr_shrink) struct free_nid *i, *next; int nr = nr_shrink; - if (nm_i->fcnt <= MAX_FREE_NIDS) + if (nm_i->nid_cnt[FREE_NID_LIST] <= MAX_FREE_NIDS) return 0; if (!mutex_trylock(&nm_i->build_lock)) return 0; - spin_lock(&nm_i->free_nid_list_lock); - list_for_each_entry_safe(i, next, &nm_i->free_nid_list, list) { - if (nr_shrink <= 0 || nm_i->fcnt <= MAX_FREE_NIDS) + spin_lock(&nm_i->nid_list_lock); + list_for_each_entry_safe(i, next, &nm_i->nid_list[FREE_NID_LIST], + list) { + if (nr_shrink <= 0 || + nm_i->nid_cnt[FREE_NID_LIST] <= MAX_FREE_NIDS) break; - if (i->state == NID_ALLOC) - continue; - __del_from_free_nid_list(nm_i, i); + + __remove_nid_from_list(sbi, i, FREE_NID_LIST, false); kmem_cache_free(free_nid_slab, i); - nm_i->fcnt--; nr_shrink--; } - spin_unlock(&nm_i->free_nid_list_lock); + spin_unlock(&nm_i->nid_list_lock); mutex_unlock(&nm_i->build_lock); return nr - nr_shrink; @@ -2005,7 +2041,7 @@ recover_xnid: if (unlikely(!inc_valid_node_count(sbi, inode))) f2fs_bug_on(sbi, 1); - remove_free_nid(NM_I(sbi), new_xnid); + remove_free_nid(sbi, new_xnid); get_node_info(sbi, new_xnid, &ni); ni.ino = inode->i_ino; set_node_addr(sbi, &ni, NEW_ADDR, false); @@ -2035,7 +2071,7 @@ retry: } /* Should not use this inode from free nid list */ - remove_free_nid(NM_I(sbi), ino); + remove_free_nid(sbi, ino); if (!PageUptodate(ipage)) SetPageUptodate(ipage); @@ -2069,7 +2105,6 @@ int restore_node_summary(struct f2fs_sb_info *sbi, struct f2fs_node *rn; struct f2fs_summary *sum_entry; block_t addr; - int bio_blocks = MAX_BIO_BLOCKS(sbi); int i, idx, last_offset, nrpages; /* scan the node segment */ @@ -2078,7 +2113,7 @@ int restore_node_summary(struct f2fs_sb_info *sbi, sum_entry = &sum->entries[0]; for (i = 0; i < last_offset; i += nrpages, addr += nrpages) { - nrpages = min(last_offset - i, bio_blocks); + nrpages = min(last_offset - i, BIO_MAX_PAGES); /* readahead node pages */ ra_meta_pages(sbi, addr, nrpages, META_POR, true); @@ -2120,6 +2155,19 @@ static void remove_nats_in_journal(struct f2fs_sb_info *sbi) ne = grab_nat_entry(nm_i, nid); node_info_from_raw_nat(&ne->ni, &raw_ne); } + + /* + * if a free nat in journal has not been used after last + * checkpoint, we should remove it from available nids, + * since later we will add it again. + */ + if (!get_nat_flag(ne, IS_DIRTY) && + le32_to_cpu(raw_ne.block_addr) == NULL_ADDR) { + spin_lock(&nm_i->nid_list_lock); + nm_i->available_nids--; + spin_unlock(&nm_i->nid_list_lock); + } + __set_nat_cache_dirty(nm_i, ne); } update_nats_in_cursum(journal, -i); @@ -2192,8 +2240,12 @@ static void __flush_nat_entry_set(struct f2fs_sb_info *sbi, raw_nat_from_node_info(raw_ne, &ne->ni); nat_reset_flag(ne); __clear_nat_cache_dirty(NM_I(sbi), ne); - if (nat_get_blkaddr(ne) == NULL_ADDR) + if (nat_get_blkaddr(ne) == NULL_ADDR) { add_free_nid(sbi, nid, false); + spin_lock(&NM_I(sbi)->nid_list_lock); + NM_I(sbi)->available_nids++; + spin_unlock(&NM_I(sbi)->nid_list_lock); + } } if (to_journal) @@ -2268,21 +2320,24 @@ static int init_node_manager(struct f2fs_sb_info *sbi) nm_i->max_nid = NAT_ENTRY_PER_BLOCK * nat_blocks; /* not used nids: 0, node, meta, (and root counted as valid node) */ - nm_i->available_nids = nm_i->max_nid - F2FS_RESERVED_NODE_NUM; - nm_i->fcnt = 0; + nm_i->available_nids = nm_i->max_nid - sbi->total_valid_node_count - + F2FS_RESERVED_NODE_NUM; + nm_i->nid_cnt[FREE_NID_LIST] = 0; + nm_i->nid_cnt[ALLOC_NID_LIST] = 0; nm_i->nat_cnt = 0; nm_i->ram_thresh = DEF_RAM_THRESHOLD; nm_i->ra_nid_pages = DEF_RA_NID_PAGES; nm_i->dirty_nats_ratio = DEF_DIRTY_NAT_RATIO_THRESHOLD; INIT_RADIX_TREE(&nm_i->free_nid_root, GFP_ATOMIC); - INIT_LIST_HEAD(&nm_i->free_nid_list); + INIT_LIST_HEAD(&nm_i->nid_list[FREE_NID_LIST]); + INIT_LIST_HEAD(&nm_i->nid_list[ALLOC_NID_LIST]); INIT_RADIX_TREE(&nm_i->nat_root, GFP_NOIO); INIT_RADIX_TREE(&nm_i->nat_set_root, GFP_NOIO); INIT_LIST_HEAD(&nm_i->nat_entries); mutex_init(&nm_i->build_lock); - spin_lock_init(&nm_i->free_nid_list_lock); + spin_lock_init(&nm_i->nid_list_lock); init_rwsem(&nm_i->nat_tree_lock); nm_i->next_scan_nid = le32_to_cpu(sbi->ckpt->next_free_nid); @@ -2310,7 +2365,7 @@ int build_node_manager(struct f2fs_sb_info *sbi) if (err) return err; - build_free_nids(sbi); + build_free_nids(sbi, true); return 0; } @@ -2327,17 +2382,18 @@ void destroy_node_manager(struct f2fs_sb_info *sbi) return; /* destroy free nid list */ - spin_lock(&nm_i->free_nid_list_lock); - list_for_each_entry_safe(i, next_i, &nm_i->free_nid_list, list) { - f2fs_bug_on(sbi, i->state == NID_ALLOC); - __del_from_free_nid_list(nm_i, i); - nm_i->fcnt--; - spin_unlock(&nm_i->free_nid_list_lock); + spin_lock(&nm_i->nid_list_lock); + list_for_each_entry_safe(i, next_i, &nm_i->nid_list[FREE_NID_LIST], + list) { + __remove_nid_from_list(sbi, i, FREE_NID_LIST, false); + spin_unlock(&nm_i->nid_list_lock); kmem_cache_free(free_nid_slab, i); - spin_lock(&nm_i->free_nid_list_lock); + spin_lock(&nm_i->nid_list_lock); } - f2fs_bug_on(sbi, nm_i->fcnt); - spin_unlock(&nm_i->free_nid_list_lock); + f2fs_bug_on(sbi, nm_i->nid_cnt[FREE_NID_LIST]); + f2fs_bug_on(sbi, nm_i->nid_cnt[ALLOC_NID_LIST]); + f2fs_bug_on(sbi, !list_empty(&nm_i->nid_list[ALLOC_NID_LIST])); + spin_unlock(&nm_i->nid_list_lock); /* destroy nat cache */ down_write(&nm_i->nat_tree_lock); diff --git a/fs/f2fs/node.h b/fs/f2fs/node.h index 868bec6..e7997e2 100644 --- a/fs/f2fs/node.h +++ b/fs/f2fs/node.h @@ -169,14 +169,15 @@ static inline void next_free_nid(struct f2fs_sb_info *sbi, nid_t *nid) struct f2fs_nm_info *nm_i = NM_I(sbi); struct free_nid *fnid; - spin_lock(&nm_i->free_nid_list_lock); - if (nm_i->fcnt <= 0) { - spin_unlock(&nm_i->free_nid_list_lock); + spin_lock(&nm_i->nid_list_lock); + if (nm_i->nid_cnt[FREE_NID_LIST] <= 0) { + spin_unlock(&nm_i->nid_list_lock); return; } - fnid = list_entry(nm_i->free_nid_list.next, struct free_nid, list); + fnid = list_entry(nm_i->nid_list[FREE_NID_LIST].next, + struct free_nid, list); *nid = fnid->nid; - spin_unlock(&nm_i->free_nid_list_lock); + spin_unlock(&nm_i->nid_list_lock); } /* @@ -313,7 +314,7 @@ static inline bool is_recoverable_dnode(struct page *page) ((unsigned char *)ckpt + crc_offset))); cp_ver |= (crc << 32); } - return cpu_to_le64(cp_ver) == cpver_of_node(page); + return cp_ver == cpver_of_node(page); } /* diff --git a/fs/f2fs/recovery.c b/fs/f2fs/recovery.c index 2fc84a9..981a958 100644 --- a/fs/f2fs/recovery.c +++ b/fs/f2fs/recovery.c @@ -180,13 +180,15 @@ static void recover_inode(struct inode *inode, struct page *page) inode->i_mode = le16_to_cpu(raw->i_mode); f2fs_i_size_write(inode, le64_to_cpu(raw->i_size)); - inode->i_atime.tv_sec = le64_to_cpu(raw->i_mtime); + inode->i_atime.tv_sec = le64_to_cpu(raw->i_atime); inode->i_ctime.tv_sec = le64_to_cpu(raw->i_ctime); inode->i_mtime.tv_sec = le64_to_cpu(raw->i_mtime); - inode->i_atime.tv_nsec = le32_to_cpu(raw->i_mtime_nsec); + inode->i_atime.tv_nsec = le32_to_cpu(raw->i_atime_nsec); inode->i_ctime.tv_nsec = le32_to_cpu(raw->i_ctime_nsec); inode->i_mtime.tv_nsec = le32_to_cpu(raw->i_mtime_nsec); + F2FS_I(inode)->i_advise = raw->i_advise; + if (file_enc_name(inode)) name = "<encrypted>"; else @@ -196,32 +198,6 @@ static void recover_inode(struct inode *inode, struct page *page) ino_of_node(page), name); } -static bool is_same_inode(struct inode *inode, struct page *ipage) -{ - struct f2fs_inode *ri = F2FS_INODE(ipage); - struct timespec disk; - - if (!IS_INODE(ipage)) - return true; - - disk.tv_sec = le64_to_cpu(ri->i_ctime); - disk.tv_nsec = le32_to_cpu(ri->i_ctime_nsec); - if (timespec_compare(&inode->i_ctime, &disk) > 0) - return false; - - disk.tv_sec = le64_to_cpu(ri->i_atime); - disk.tv_nsec = le32_to_cpu(ri->i_atime_nsec); - if (timespec_compare(&inode->i_atime, &disk) > 0) - return false; - - disk.tv_sec = le64_to_cpu(ri->i_mtime); - disk.tv_nsec = le32_to_cpu(ri->i_mtime_nsec); - if (timespec_compare(&inode->i_mtime, &disk) > 0) - return false; - - return true; -} - static int find_fsync_dnodes(struct f2fs_sb_info *sbi, struct list_head *head) { struct curseg_info *curseg; @@ -248,10 +224,7 @@ static int find_fsync_dnodes(struct f2fs_sb_info *sbi, struct list_head *head) goto next; entry = get_fsync_inode(head, ino_of_node(page)); - if (entry) { - if (!is_same_inode(entry->inode, page)) - goto next; - } else { + if (!entry) { if (IS_INODE(page) && is_dent_dnode(page)) { err = recover_inode_page(sbi, page); if (err) @@ -454,7 +427,8 @@ retry_dn: continue; } - if ((start + 1) << PAGE_SHIFT > i_size_read(inode)) + if (!file_keep_isize(inode) && + (i_size_read(inode) <= (start << PAGE_SHIFT))) f2fs_i_size_write(inode, (start + 1) << PAGE_SHIFT); /* @@ -507,8 +481,10 @@ err: f2fs_put_dnode(&dn); out: f2fs_msg(sbi->sb, KERN_NOTICE, - "recover_data: ino = %lx, recovered = %d blocks, err = %d", - inode->i_ino, recovered, err); + "recover_data: ino = %lx (i_size: %s) recovered = %d, err = %d", + inode->i_ino, + file_keep_isize(inode) ? "keep" : "recover", + recovered, err); return err; } diff --git a/fs/f2fs/segment.c b/fs/f2fs/segment.c index f1b4a17..0738f48 100644 --- a/fs/f2fs/segment.c +++ b/fs/f2fs/segment.c @@ -274,8 +274,10 @@ static int __commit_inmem_pages(struct inode *inode, set_page_dirty(page); f2fs_wait_on_page_writeback(page, DATA, true); - if (clear_page_dirty_for_io(page)) + if (clear_page_dirty_for_io(page)) { inode_dec_dirty_pages(inode); + remove_dirty_inode(inode); + } fio.page = page; err = do_write_data_page(&fio); @@ -287,7 +289,6 @@ static int __commit_inmem_pages(struct inode *inode, /* record old blkaddr for revoking */ cur->old_addr = fio.old_blkaddr; - clear_cold_data(page); submit_bio = true; } unlock_page(page); @@ -363,7 +364,7 @@ void f2fs_balance_fs(struct f2fs_sb_info *sbi, bool need) */ if (has_not_enough_free_secs(sbi, 0, 0)) { mutex_lock(&sbi->gc_mutex); - f2fs_gc(sbi, false); + f2fs_gc(sbi, false, false); } } @@ -380,14 +381,17 @@ void f2fs_balance_fs_bg(struct f2fs_sb_info *sbi) if (!available_free_memory(sbi, FREE_NIDS)) try_to_free_nids(sbi, MAX_FREE_NIDS); else - build_free_nids(sbi); + build_free_nids(sbi, false); + + if (!is_idle(sbi)) + return; /* checkpoint is the only way to shrink partial cached entries */ if (!available_free_memory(sbi, NAT_ENTRIES) || !available_free_memory(sbi, INO_ENTRIES) || excess_prefree_segs(sbi) || excess_dirty_nats(sbi) || - (is_idle(sbi) && f2fs_time_over(sbi, CP_TIME))) { + f2fs_time_over(sbi, CP_TIME)) { if (test_opt(sbi, DATA_FLUSH)) { struct blk_plug plug; @@ -400,6 +404,33 @@ void f2fs_balance_fs_bg(struct f2fs_sb_info *sbi) } } +static int __submit_flush_wait(struct block_device *bdev) +{ + struct bio *bio = f2fs_bio_alloc(0); + int ret; + + bio->bi_opf = REQ_OP_WRITE | REQ_PREFLUSH; + bio->bi_bdev = bdev; + ret = submit_bio_wait(bio); + bio_put(bio); + return ret; +} + +static int submit_flush_wait(struct f2fs_sb_info *sbi) +{ + int ret = __submit_flush_wait(sbi->sb->s_bdev); + int i; + + if (sbi->s_ndevs && !ret) { + for (i = 1; i < sbi->s_ndevs; i++) { + ret = __submit_flush_wait(FDEV(i).bdev); + if (ret) + break; + } + } + return ret; +} + static int issue_flush_thread(void *data) { struct f2fs_sb_info *sbi = data; @@ -410,25 +441,18 @@ repeat: return 0; if (!llist_empty(&fcc->issue_list)) { - struct bio *bio; struct flush_cmd *cmd, *next; int ret; - bio = f2fs_bio_alloc(0); - fcc->dispatch_list = llist_del_all(&fcc->issue_list); fcc->dispatch_list = llist_reverse_order(fcc->dispatch_list); - bio->bi_bdev = sbi->sb->s_bdev; - bio->bi_opf = REQ_OP_WRITE | REQ_PREFLUSH; - ret = submit_bio_wait(bio); - + ret = submit_flush_wait(sbi); llist_for_each_entry_safe(cmd, next, fcc->dispatch_list, llnode) { cmd->ret = ret; complete(&cmd->wait); } - bio_put(bio); fcc->dispatch_list = NULL; } @@ -449,15 +473,11 @@ int f2fs_issue_flush(struct f2fs_sb_info *sbi) return 0; if (!test_opt(sbi, FLUSH_MERGE) || !atomic_read(&fcc->submit_flush)) { - struct bio *bio = f2fs_bio_alloc(0); int ret; atomic_inc(&fcc->submit_flush); - bio->bi_bdev = sbi->sb->s_bdev; - bio->bi_opf = REQ_OP_WRITE | REQ_PREFLUSH; - ret = submit_bio_wait(bio); + ret = submit_flush_wait(sbi); atomic_dec(&fcc->submit_flush); - bio_put(bio); return ret; } @@ -469,8 +489,13 @@ int f2fs_issue_flush(struct f2fs_sb_info *sbi) if (!fcc->dispatch_list) wake_up(&fcc->flush_wait_queue); - wait_for_completion(&cmd.wait); - atomic_dec(&fcc->submit_flush); + if (fcc->f2fs_issue_flush) { + wait_for_completion(&cmd.wait); + atomic_dec(&fcc->submit_flush); + } else { + llist_del_all(&fcc->issue_list); + atomic_set(&fcc->submit_flush, 0); + } return cmd.ret; } @@ -481,6 +506,11 @@ int create_flush_cmd_control(struct f2fs_sb_info *sbi) struct flush_cmd_control *fcc; int err = 0; + if (SM_I(sbi)->cmd_control_info) { + fcc = SM_I(sbi)->cmd_control_info; + goto init_thread; + } + fcc = kzalloc(sizeof(struct flush_cmd_control), GFP_KERNEL); if (!fcc) return -ENOMEM; @@ -488,6 +518,7 @@ int create_flush_cmd_control(struct f2fs_sb_info *sbi) init_waitqueue_head(&fcc->flush_wait_queue); init_llist_head(&fcc->issue_list); SM_I(sbi)->cmd_control_info = fcc; +init_thread: fcc->f2fs_issue_flush = kthread_run(issue_flush_thread, sbi, "f2fs_flush-%u:%u", MAJOR(dev), MINOR(dev)); if (IS_ERR(fcc->f2fs_issue_flush)) { @@ -500,14 +531,20 @@ int create_flush_cmd_control(struct f2fs_sb_info *sbi) return err; } -void destroy_flush_cmd_control(struct f2fs_sb_info *sbi) +void destroy_flush_cmd_control(struct f2fs_sb_info *sbi, bool free) { struct flush_cmd_control *fcc = SM_I(sbi)->cmd_control_info; - if (fcc && fcc->f2fs_issue_flush) - kthread_stop(fcc->f2fs_issue_flush); - kfree(fcc); - SM_I(sbi)->cmd_control_info = NULL; + if (fcc && fcc->f2fs_issue_flush) { + struct task_struct *flush_thread = fcc->f2fs_issue_flush; + + fcc->f2fs_issue_flush = NULL; + kthread_stop(flush_thread); + } + if (free) { + kfree(fcc); + SM_I(sbi)->cmd_control_info = NULL; + } } static void __locate_dirty_segment(struct f2fs_sb_info *sbi, unsigned int segno, @@ -633,15 +670,23 @@ static void f2fs_submit_bio_wait_endio(struct bio *bio) } /* this function is copied from blkdev_issue_discard from block/blk-lib.c */ -int __f2fs_issue_discard_async(struct f2fs_sb_info *sbi, sector_t sector, - sector_t nr_sects, gfp_t gfp_mask, unsigned long flags) +static int __f2fs_issue_discard_async(struct f2fs_sb_info *sbi, + struct block_device *bdev, block_t blkstart, block_t blklen) { - struct block_device *bdev = sbi->sb->s_bdev; struct bio *bio = NULL; int err; - err = __blkdev_issue_discard(bdev, sector, nr_sects, gfp_mask, flags, - &bio); + trace_f2fs_issue_discard(sbi->sb, blkstart, blklen); + + if (sbi->s_ndevs) { + int devi = f2fs_target_device_index(sbi, blkstart); + + blkstart -= FDEV(devi).start_blk; + } + err = __blkdev_issue_discard(bdev, + SECTOR_FROM_BLOCK(blkstart), + SECTOR_FROM_BLOCK(blklen), + GFP_NOFS, 0, &bio); if (!err && bio) { struct bio_entry *be = __add_bio_entry(sbi, bio); @@ -654,24 +699,101 @@ int __f2fs_issue_discard_async(struct f2fs_sb_info *sbi, sector_t sector, return err; } +#ifdef CONFIG_BLK_DEV_ZONED +static int __f2fs_issue_discard_zone(struct f2fs_sb_info *sbi, + struct block_device *bdev, block_t blkstart, block_t blklen) +{ + sector_t nr_sects = SECTOR_FROM_BLOCK(blklen); + sector_t sector; + int devi = 0; + + if (sbi->s_ndevs) { + devi = f2fs_target_device_index(sbi, blkstart); + blkstart -= FDEV(devi).start_blk; + } + sector = SECTOR_FROM_BLOCK(blkstart); + + if (sector & (bdev_zone_size(bdev) - 1) || + nr_sects != bdev_zone_size(bdev)) { + f2fs_msg(sbi->sb, KERN_INFO, + "(%d) %s: Unaligned discard attempted (block %x + %x)", + devi, sbi->s_ndevs ? FDEV(devi).path: "", + blkstart, blklen); + return -EIO; + } + + /* + * We need to know the type of the zone: for conventional zones, + * use regular discard if the drive supports it. For sequential + * zones, reset the zone write pointer. + */ + switch (get_blkz_type(sbi, bdev, blkstart)) { + + case BLK_ZONE_TYPE_CONVENTIONAL: + if (!blk_queue_discard(bdev_get_queue(bdev))) + return 0; + return __f2fs_issue_discard_async(sbi, bdev, blkstart, blklen); + case BLK_ZONE_TYPE_SEQWRITE_REQ: + case BLK_ZONE_TYPE_SEQWRITE_PREF: + trace_f2fs_issue_reset_zone(sbi->sb, blkstart); + return blkdev_reset_zones(bdev, sector, + nr_sects, GFP_NOFS); + default: + /* Unknown zone type: broken device ? */ + return -EIO; + } +} +#endif + +static int __issue_discard_async(struct f2fs_sb_info *sbi, + struct block_device *bdev, block_t blkstart, block_t blklen) +{ +#ifdef CONFIG_BLK_DEV_ZONED + if (f2fs_sb_mounted_blkzoned(sbi->sb) && + bdev_zoned_model(bdev) != BLK_ZONED_NONE) + return __f2fs_issue_discard_zone(sbi, bdev, blkstart, blklen); +#endif + return __f2fs_issue_discard_async(sbi, bdev, blkstart, blklen); +} + static int f2fs_issue_discard(struct f2fs_sb_info *sbi, block_t blkstart, block_t blklen) { - sector_t start = SECTOR_FROM_BLOCK(blkstart); - sector_t len = SECTOR_FROM_BLOCK(blklen); + sector_t start = blkstart, len = 0; + struct block_device *bdev; struct seg_entry *se; unsigned int offset; block_t i; + int err = 0; + + bdev = f2fs_target_device(sbi, blkstart, NULL); + + for (i = blkstart; i < blkstart + blklen; i++, len++) { + if (i != start) { + struct block_device *bdev2 = + f2fs_target_device(sbi, i, NULL); + + if (bdev2 != bdev) { + err = __issue_discard_async(sbi, bdev, + start, len); + if (err) + return err; + bdev = bdev2; + start = i; + len = 0; + } + } - for (i = blkstart; i < blkstart + blklen; i++) { se = get_seg_entry(sbi, GET_SEGNO(sbi, i)); offset = GET_BLKOFF_FROM_SEG0(sbi, i); if (!f2fs_test_and_set_bit(offset, se->discard_map)) sbi->discard_blks--; } - trace_f2fs_issue_discard(sbi->sb, blkstart, blklen); - return __f2fs_issue_discard_async(sbi, start, len, GFP_NOFS, 0); + + if (len) + err = __issue_discard_async(sbi, bdev, start, len); + return err; } static void __add_discard_entry(struct f2fs_sb_info *sbi, @@ -1296,25 +1418,21 @@ static void allocate_segment_by_default(struct f2fs_sb_info *sbi, stat_inc_seg_type(sbi, curseg); } -static void __allocate_new_segments(struct f2fs_sb_info *sbi, int type) -{ - struct curseg_info *curseg = CURSEG_I(sbi, type); - unsigned int old_segno; - - old_segno = curseg->segno; - SIT_I(sbi)->s_ops->allocate_segment(sbi, type, true); - locate_dirty_segment(sbi, old_segno); -} - void allocate_new_segments(struct f2fs_sb_info *sbi) { + struct curseg_info *curseg; + unsigned int old_segno; int i; if (test_opt(sbi, LFS)) return; - for (i = CURSEG_HOT_DATA; i <= CURSEG_COLD_DATA; i++) - __allocate_new_segments(sbi, i); + for (i = CURSEG_HOT_DATA; i <= CURSEG_COLD_DATA; i++) { + curseg = CURSEG_I(sbi, i); + old_segno = curseg->segno; + SIT_I(sbi)->s_ops->allocate_segment(sbi, i, true); + locate_dirty_segment(sbi, old_segno); + } } static const struct segment_allocation default_salloc_ops = { @@ -1448,21 +1566,11 @@ void allocate_data_block(struct f2fs_sb_info *sbi, struct page *page, struct f2fs_summary *sum, int type) { struct sit_info *sit_i = SIT_I(sbi); - struct curseg_info *curseg; - bool direct_io = (type == CURSEG_DIRECT_IO); - - type = direct_io ? CURSEG_WARM_DATA : type; - - curseg = CURSEG_I(sbi, type); + struct curseg_info *curseg = CURSEG_I(sbi, type); mutex_lock(&curseg->curseg_mutex); mutex_lock(&sit_i->sentry_lock); - /* direct_io'ed data is aligned to the segment for better performance */ - if (direct_io && curseg->next_blkoff && - !has_not_enough_free_secs(sbi, 0, 0)) - __allocate_new_segments(sbi, type); - *new_blkaddr = NEXT_FREE_BLKADDR(sbi, curseg); /* @@ -2166,7 +2274,6 @@ out: static int build_sit_info(struct f2fs_sb_info *sbi) { struct f2fs_super_block *raw_super = F2FS_RAW_SUPER(sbi); - struct f2fs_checkpoint *ckpt = F2FS_CKPT(sbi); struct sit_info *sit_i; unsigned int sit_segs, start; char *src_bitmap, *dst_bitmap; @@ -2233,7 +2340,7 @@ static int build_sit_info(struct f2fs_sb_info *sbi) sit_i->sit_base_addr = le32_to_cpu(raw_super->sit_blkaddr); sit_i->sit_blocks = sit_segs << sbi->log_blocks_per_seg; - sit_i->written_valid_blocks = le64_to_cpu(ckpt->valid_block_count); + sit_i->written_valid_blocks = 0; sit_i->sit_bitmap = dst_bitmap; sit_i->bitmap_size = bitmap_size; sit_i->dirty_sentries = 0; @@ -2315,10 +2422,10 @@ static void build_sit_entries(struct f2fs_sb_info *sbi) int sit_blk_cnt = SIT_BLK_CNT(sbi); unsigned int i, start, end; unsigned int readed, start_blk = 0; - int nrpages = MAX_BIO_BLOCKS(sbi) * 8; do { - readed = ra_meta_pages(sbi, start_blk, nrpages, META_SIT, true); + readed = ra_meta_pages(sbi, start_blk, BIO_MAX_PAGES, + META_SIT, true); start = start_blk * sit_i->sents_per_block; end = (start_blk + readed) * sit_i->sents_per_block; @@ -2387,6 +2494,9 @@ static void init_free_segmap(struct f2fs_sb_info *sbi) struct seg_entry *sentry = get_seg_entry(sbi, start); if (!sentry->valid_blocks) __set_free(sbi, start); + else + SIT_I(sbi)->written_valid_blocks += + sentry->valid_blocks; } /* set use the current segments */ @@ -2645,7 +2755,7 @@ void destroy_segment_manager(struct f2fs_sb_info *sbi) if (!sm_info) return; - destroy_flush_cmd_control(sbi); + destroy_flush_cmd_control(sbi, true); destroy_dirty_segmap(sbi); destroy_curseg(sbi); destroy_free_segmap(sbi); diff --git a/fs/f2fs/segment.h b/fs/f2fs/segment.h index fecb856..9d44ce8 100644 --- a/fs/f2fs/segment.h +++ b/fs/f2fs/segment.h @@ -18,6 +18,8 @@ #define DEF_RECLAIM_PREFREE_SEGMENTS 5 /* 5% over total segments */ #define DEF_MAX_RECLAIM_PREFREE_SEGMENTS 4096 /* 8GB in maximum */ +#define F2FS_MIN_SEGMENTS 9 /* SB + 2 (CP + SIT + NAT) + SSA + MAIN */ + /* L: Logical segment # in volume, R: Relative segment # in main area */ #define GET_L2R_SEGNO(free_i, segno) (segno - free_i->start_segno) #define GET_R2L_SEGNO(free_i, segno) (segno + free_i->start_segno) @@ -102,8 +104,6 @@ (((sector_t)blk_addr) << F2FS_LOG_SECTORS_PER_BLOCK) #define SECTOR_TO_BLOCK(sectors) \ (sectors >> F2FS_LOG_SECTORS_PER_BLOCK) -#define MAX_BIO_BLOCKS(sbi) \ - ((int)min((int)max_hw_blocks(sbi), BIO_MAX_PAGES)) /* * indicate a block allocation direction: RIGHT and LEFT. @@ -471,11 +471,12 @@ static inline bool need_SSR(struct f2fs_sb_info *sbi) { int node_secs = get_blocktype_secs(sbi, F2FS_DIRTY_NODES); int dent_secs = get_blocktype_secs(sbi, F2FS_DIRTY_DENTS); + int imeta_secs = get_blocktype_secs(sbi, F2FS_DIRTY_IMETA); if (test_opt(sbi, LFS)) return false; - return free_sections(sbi) <= (node_secs + 2 * dent_secs + + return free_sections(sbi) <= (node_secs + 2 * dent_secs + imeta_secs + reserved_sections(sbi) + 1); } @@ -484,14 +485,14 @@ static inline bool has_not_enough_free_secs(struct f2fs_sb_info *sbi, { int node_secs = get_blocktype_secs(sbi, F2FS_DIRTY_NODES); int dent_secs = get_blocktype_secs(sbi, F2FS_DIRTY_DENTS); - - node_secs += get_blocktype_secs(sbi, F2FS_DIRTY_IMETA); + int imeta_secs = get_blocktype_secs(sbi, F2FS_DIRTY_IMETA); if (unlikely(is_sbi_flag_set(sbi, SBI_POR_DOING))) return false; return (free_sections(sbi) + freed) <= - (node_secs + 2 * dent_secs + reserved_sections(sbi) + needed); + (node_secs + 2 * dent_secs + imeta_secs + + reserved_sections(sbi) + needed); } static inline bool excess_prefree_segs(struct f2fs_sb_info *sbi) @@ -695,13 +696,6 @@ static inline bool sec_usage_check(struct f2fs_sb_info *sbi, unsigned int secno) return false; } -static inline unsigned int max_hw_blocks(struct f2fs_sb_info *sbi) -{ - struct block_device *bdev = sbi->sb->s_bdev; - struct request_queue *q = bdev_get_queue(bdev); - return SECTOR_TO_BLOCK(queue_max_sectors(q)); -} - /* * It is very important to gather dirty pages and write at once, so that we can * submit a big bio without interfering other data writes. @@ -719,7 +713,7 @@ static inline int nr_pages_to_skip(struct f2fs_sb_info *sbi, int type) else if (type == NODE) return 8 * sbi->blocks_per_seg; else if (type == META) - return 8 * MAX_BIO_BLOCKS(sbi); + return 8 * BIO_MAX_PAGES; else return 0; } @@ -736,11 +730,9 @@ static inline long nr_pages_to_write(struct f2fs_sb_info *sbi, int type, return 0; nr_to_write = wbc->nr_to_write; - + desired = BIO_MAX_PAGES; if (type == NODE) - desired = 2 * max_hw_blocks(sbi); - else - desired = MAX_BIO_BLOCKS(sbi); + desired <<= 1; wbc->nr_to_write = desired; return desired - nr_to_write; diff --git a/fs/f2fs/shrinker.c b/fs/f2fs/shrinker.c index 46c9154..5c60fc2 100644 --- a/fs/f2fs/shrinker.c +++ b/fs/f2fs/shrinker.c @@ -21,14 +21,16 @@ static unsigned int shrinker_run_no; static unsigned long __count_nat_entries(struct f2fs_sb_info *sbi) { - return NM_I(sbi)->nat_cnt - NM_I(sbi)->dirty_nat_cnt; + long count = NM_I(sbi)->nat_cnt - NM_I(sbi)->dirty_nat_cnt; + + return count > 0 ? count : 0; } static unsigned long __count_free_nids(struct f2fs_sb_info *sbi) { - if (NM_I(sbi)->fcnt > MAX_FREE_NIDS) - return NM_I(sbi)->fcnt - MAX_FREE_NIDS; - return 0; + long count = NM_I(sbi)->nid_cnt[FREE_NID_LIST] - MAX_FREE_NIDS; + + return count > 0 ? count : 0; } static unsigned long __count_extent_cache(struct f2fs_sb_info *sbi) diff --git a/fs/f2fs/super.c b/fs/f2fs/super.c index 2cac6bb..702638e 100644 --- a/fs/f2fs/super.c +++ b/fs/f2fs/super.c @@ -412,14 +412,20 @@ static int parse_options(struct super_block *sb, char *options) q = bdev_get_queue(sb->s_bdev); if (blk_queue_discard(q)) { set_opt(sbi, DISCARD); - } else { + } else if (!f2fs_sb_mounted_blkzoned(sb)) { f2fs_msg(sb, KERN_WARNING, "mounting with \"discard\" option, but " "the device does not support discard"); } break; case Opt_nodiscard: + if (f2fs_sb_mounted_blkzoned(sb)) { + f2fs_msg(sb, KERN_WARNING, + "discard is required for zoned block devices"); + return -EINVAL; + } clear_opt(sbi, DISCARD); + break; case Opt_noheap: set_opt(sbi, NOHEAP); break; @@ -512,6 +518,13 @@ static int parse_options(struct super_block *sb, char *options) return -ENOMEM; if (strlen(name) == 8 && !strncmp(name, "adaptive", 8)) { + if (f2fs_sb_mounted_blkzoned(sb)) { + f2fs_msg(sb, KERN_WARNING, + "adaptive mode is not allowed with " + "zoned block device feature"); + kfree(name); + return -EINVAL; + } set_opt_mode(sbi, F2FS_MOUNT_ADAPTIVE); } else if (strlen(name) == 3 && !strncmp(name, "lfs", 3)) { @@ -558,13 +571,9 @@ static struct inode *f2fs_alloc_inode(struct super_block *sb) init_once((void *) fi); - if (percpu_counter_init(&fi->dirty_pages, 0, GFP_NOFS)) { - kmem_cache_free(f2fs_inode_cachep, fi); - return NULL; - } - /* Initialize f2fs-specific inode info */ fi->vfs_inode.i_version = 1; + atomic_set(&fi->dirty_pages, 0); fi->i_current_depth = 1; fi->i_advise = 0; init_rwsem(&fi->i_sem); @@ -620,24 +629,25 @@ static int f2fs_drop_inode(struct inode *inode) return generic_drop_inode(inode); } -int f2fs_inode_dirtied(struct inode *inode) +int f2fs_inode_dirtied(struct inode *inode, bool sync) { struct f2fs_sb_info *sbi = F2FS_I_SB(inode); + int ret = 0; spin_lock(&sbi->inode_lock[DIRTY_META]); if (is_inode_flag_set(inode, FI_DIRTY_INODE)) { - spin_unlock(&sbi->inode_lock[DIRTY_META]); - return 1; + ret = 1; + } else { + set_inode_flag(inode, FI_DIRTY_INODE); + stat_inc_dirty_inode(sbi, DIRTY_META); } - - set_inode_flag(inode, FI_DIRTY_INODE); - list_add_tail(&F2FS_I(inode)->gdirty_list, + if (sync && list_empty(&F2FS_I(inode)->gdirty_list)) { + list_add_tail(&F2FS_I(inode)->gdirty_list, &sbi->inode_list[DIRTY_META]); - inc_page_count(sbi, F2FS_DIRTY_IMETA); - stat_inc_dirty_inode(sbi, DIRTY_META); + inc_page_count(sbi, F2FS_DIRTY_IMETA); + } spin_unlock(&sbi->inode_lock[DIRTY_META]); - - return 0; + return ret; } void f2fs_inode_synced(struct inode *inode) @@ -649,10 +659,12 @@ void f2fs_inode_synced(struct inode *inode) spin_unlock(&sbi->inode_lock[DIRTY_META]); return; } - list_del_init(&F2FS_I(inode)->gdirty_list); + if (!list_empty(&F2FS_I(inode)->gdirty_list)) { + list_del_init(&F2FS_I(inode)->gdirty_list); + dec_page_count(sbi, F2FS_DIRTY_IMETA); + } clear_inode_flag(inode, FI_DIRTY_INODE); clear_inode_flag(inode, FI_AUTO_RECOVER); - dec_page_count(sbi, F2FS_DIRTY_IMETA); stat_dec_dirty_inode(F2FS_I_SB(inode), DIRTY_META); spin_unlock(&sbi->inode_lock[DIRTY_META]); } @@ -676,7 +688,7 @@ static void f2fs_dirty_inode(struct inode *inode, int flags) if (is_inode_flag_set(inode, FI_AUTO_RECOVER)) clear_inode_flag(inode, FI_AUTO_RECOVER); - f2fs_inode_dirtied(inode); + f2fs_inode_dirtied(inode, false); } static void f2fs_i_callback(struct rcu_head *head) @@ -687,20 +699,28 @@ static void f2fs_i_callback(struct rcu_head *head) static void f2fs_destroy_inode(struct inode *inode) { - percpu_counter_destroy(&F2FS_I(inode)->dirty_pages); call_rcu(&inode->i_rcu, f2fs_i_callback); } static void destroy_percpu_info(struct f2fs_sb_info *sbi) { - int i; - - for (i = 0; i < NR_COUNT_TYPE; i++) - percpu_counter_destroy(&sbi->nr_pages[i]); percpu_counter_destroy(&sbi->alloc_valid_block_count); percpu_counter_destroy(&sbi->total_valid_inode_count); } +static void destroy_device_list(struct f2fs_sb_info *sbi) +{ + int i; + + for (i = 0; i < sbi->s_ndevs; i++) { + blkdev_put(FDEV(i).bdev, FMODE_EXCL); +#ifdef CONFIG_BLK_DEV_ZONED + kfree(FDEV(i).blkz_type); +#endif + } + kfree(sbi->devs); +} + static void f2fs_put_super(struct super_block *sb) { struct f2fs_sb_info *sbi = F2FS_SB(sb); @@ -738,7 +758,6 @@ static void f2fs_put_super(struct super_block *sb) * In addition, EIO will skip do checkpoint, we need this as well. */ release_ino_entry(sbi, true); - release_discard_addrs(sbi); f2fs_leave_shrinker(sbi); mutex_unlock(&sbi->umount_mutex); @@ -762,6 +781,8 @@ static void f2fs_put_super(struct super_block *sb) crypto_free_shash(sbi->s_chksum_driver); kfree(sbi->raw_super); + destroy_device_list(sbi); + destroy_percpu_info(sbi); kfree(sbi); } @@ -789,13 +810,17 @@ int f2fs_sync_fs(struct super_block *sb, int sync) static int f2fs_freeze(struct super_block *sb) { - int err; - if (f2fs_readonly(sb)) return 0; - err = f2fs_sync_fs(sb, 1); - return err; + /* IO error happened before */ + if (unlikely(f2fs_cp_error(F2FS_SB(sb)))) + return -EIO; + + /* must be clean, since sync_filesystem() was already called */ + if (is_sbi_flag_set(F2FS_SB(sb), SBI_IS_DIRTY)) + return -EINVAL; + return 0; } static int f2fs_unfreeze(struct super_block *sb) @@ -822,7 +847,8 @@ static int f2fs_statfs(struct dentry *dentry, struct kstatfs *buf) buf->f_bavail = user_block_count - valid_user_blocks(sbi); buf->f_files = sbi->total_node_count - F2FS_RESERVED_NODE_NUM; - buf->f_ffree = buf->f_files - valid_inode_count(sbi); + buf->f_ffree = min(buf->f_files - valid_node_count(sbi), + buf->f_bavail); buf->f_namelen = F2FS_NAME_LEN; buf->f_fsid.val[0] = (u32)id; @@ -974,7 +1000,7 @@ static void default_options(struct f2fs_sb_info *sbi) set_opt(sbi, EXTENT_CACHE); sbi->sb->s_flags |= MS_LAZYTIME; set_opt(sbi, FLUSH_MERGE); - if (f2fs_sb_mounted_hmsmr(sbi->sb)) { + if (f2fs_sb_mounted_blkzoned(sbi->sb)) { set_opt_mode(sbi, F2FS_MOUNT_LFS); set_opt(sbi, DISCARD); } else { @@ -1076,8 +1102,9 @@ static int f2fs_remount(struct super_block *sb, int *flags, char *data) * or if flush_merge is not passed in mount option. */ if ((*flags & MS_RDONLY) || !test_opt(sbi, FLUSH_MERGE)) { - destroy_flush_cmd_control(sbi); - } else if (!SM_I(sbi)->cmd_control_info) { + clear_opt(sbi, FLUSH_MERGE); + destroy_flush_cmd_control(sbi, false); + } else { err = create_flush_cmd_control(sbi); if (err) goto restore_gc; @@ -1426,6 +1453,7 @@ int sanity_check_ckpt(struct f2fs_sb_info *sbi) unsigned int total, fsmeta; struct f2fs_super_block *raw_super = F2FS_RAW_SUPER(sbi); struct f2fs_checkpoint *ckpt = F2FS_CKPT(sbi); + unsigned int ovp_segments, reserved_segments; total = le32_to_cpu(raw_super->segment_count); fsmeta = le32_to_cpu(raw_super->segment_count_ckpt); @@ -1437,6 +1465,16 @@ int sanity_check_ckpt(struct f2fs_sb_info *sbi) if (unlikely(fsmeta >= total)) return 1; + ovp_segments = le32_to_cpu(ckpt->overprov_segment_count); + reserved_segments = le32_to_cpu(ckpt->rsvd_segment_count); + + if (unlikely(fsmeta < F2FS_MIN_SEGMENTS || + ovp_segments == 0 || reserved_segments == 0)) { + f2fs_msg(sbi->sb, KERN_ERR, + "Wrong layout: check mkfs.f2fs version"); + return 1; + } + if (unlikely(f2fs_cp_error(sbi))) { f2fs_msg(sbi->sb, KERN_ERR, "A bug case: need to run fsck"); return 1; @@ -1447,6 +1485,7 @@ int sanity_check_ckpt(struct f2fs_sb_info *sbi) static void init_sb_info(struct f2fs_sb_info *sbi) { struct f2fs_super_block *raw_super = sbi->raw_super; + int i; sbi->log_sectors_per_block = le32_to_cpu(raw_super->log_sectors_per_block); @@ -1471,6 +1510,9 @@ static void init_sb_info(struct f2fs_sb_info *sbi) sbi->interval_time[REQ_TIME] = DEF_IDLE_INTERVAL; clear_sbi_flag(sbi, SBI_NEED_FSCK); + for (i = 0; i < NR_COUNT_TYPE; i++) + atomic_set(&sbi->nr_pages[i], 0); + INIT_LIST_HEAD(&sbi->s_list); mutex_init(&sbi->umount_mutex); mutex_init(&sbi->wio_mutex[NODE]); @@ -1486,13 +1528,7 @@ static void init_sb_info(struct f2fs_sb_info *sbi) static int init_percpu_info(struct f2fs_sb_info *sbi) { - int i, err; - - for (i = 0; i < NR_COUNT_TYPE; i++) { - err = percpu_counter_init(&sbi->nr_pages[i], 0, GFP_KERNEL); - if (err) - return err; - } + int err; err = percpu_counter_init(&sbi->alloc_valid_block_count, 0, GFP_KERNEL); if (err) @@ -1502,6 +1538,71 @@ static int init_percpu_info(struct f2fs_sb_info *sbi) GFP_KERNEL); } +#ifdef CONFIG_BLK_DEV_ZONED +static int init_blkz_info(struct f2fs_sb_info *sbi, int devi) +{ + struct block_device *bdev = FDEV(devi).bdev; + sector_t nr_sectors = bdev->bd_part->nr_sects; + sector_t sector = 0; + struct blk_zone *zones; + unsigned int i, nr_zones; + unsigned int n = 0; + int err = -EIO; + + if (!f2fs_sb_mounted_blkzoned(sbi->sb)) + return 0; + + if (sbi->blocks_per_blkz && sbi->blocks_per_blkz != + SECTOR_TO_BLOCK(bdev_zone_size(bdev))) + return -EINVAL; + sbi->blocks_per_blkz = SECTOR_TO_BLOCK(bdev_zone_size(bdev)); + if (sbi->log_blocks_per_blkz && sbi->log_blocks_per_blkz != + __ilog2_u32(sbi->blocks_per_blkz)) + return -EINVAL; + sbi->log_blocks_per_blkz = __ilog2_u32(sbi->blocks_per_blkz); + FDEV(devi).nr_blkz = SECTOR_TO_BLOCK(nr_sectors) >> + sbi->log_blocks_per_blkz; + if (nr_sectors & (bdev_zone_size(bdev) - 1)) + FDEV(devi).nr_blkz++; + + FDEV(devi).blkz_type = kmalloc(FDEV(devi).nr_blkz, GFP_KERNEL); + if (!FDEV(devi).blkz_type) + return -ENOMEM; + +#define F2FS_REPORT_NR_ZONES 4096 + + zones = kcalloc(F2FS_REPORT_NR_ZONES, sizeof(struct blk_zone), + GFP_KERNEL); + if (!zones) + return -ENOMEM; + + /* Get block zones type */ + while (zones && sector < nr_sectors) { + + nr_zones = F2FS_REPORT_NR_ZONES; + err = blkdev_report_zones(bdev, sector, + zones, &nr_zones, + GFP_KERNEL); + if (err) + break; + if (!nr_zones) { + err = -EIO; + break; + } + + for (i = 0; i < nr_zones; i++) { + FDEV(devi).blkz_type[n] = zones[i].type; + sector += zones[i].len; + n++; + } + } + + kfree(zones); + + return err; +} +#endif + /* * Read f2fs raw super block. * Because we have two copies of super block, so read both of them @@ -1594,6 +1695,77 @@ int f2fs_commit_super(struct f2fs_sb_info *sbi, bool recover) return err; } +static int f2fs_scan_devices(struct f2fs_sb_info *sbi) +{ + struct f2fs_super_block *raw_super = F2FS_RAW_SUPER(sbi); + int i; + + for (i = 0; i < MAX_DEVICES; i++) { + if (!RDEV(i).path[0]) + return 0; + + if (i == 0) { + sbi->devs = kzalloc(sizeof(struct f2fs_dev_info) * + MAX_DEVICES, GFP_KERNEL); + if (!sbi->devs) + return -ENOMEM; + } + + memcpy(FDEV(i).path, RDEV(i).path, MAX_PATH_LEN); + FDEV(i).total_segments = le32_to_cpu(RDEV(i).total_segments); + if (i == 0) { + FDEV(i).start_blk = 0; + FDEV(i).end_blk = FDEV(i).start_blk + + (FDEV(i).total_segments << + sbi->log_blocks_per_seg) - 1 + + le32_to_cpu(raw_super->segment0_blkaddr); + } else { + FDEV(i).start_blk = FDEV(i - 1).end_blk + 1; + FDEV(i).end_blk = FDEV(i).start_blk + + (FDEV(i).total_segments << + sbi->log_blocks_per_seg) - 1; + } + + FDEV(i).bdev = blkdev_get_by_path(FDEV(i).path, + sbi->sb->s_mode, sbi->sb->s_type); + if (IS_ERR(FDEV(i).bdev)) + return PTR_ERR(FDEV(i).bdev); + + /* to release errored devices */ + sbi->s_ndevs = i + 1; + +#ifdef CONFIG_BLK_DEV_ZONED + if (bdev_zoned_model(FDEV(i).bdev) == BLK_ZONED_HM && + !f2fs_sb_mounted_blkzoned(sbi->sb)) { + f2fs_msg(sbi->sb, KERN_ERR, + "Zoned block device feature not enabled\n"); + return -EINVAL; + } + if (bdev_zoned_model(FDEV(i).bdev) != BLK_ZONED_NONE) { + if (init_blkz_info(sbi, i)) { + f2fs_msg(sbi->sb, KERN_ERR, + "Failed to initialize F2FS blkzone information"); + return -EINVAL; + } + f2fs_msg(sbi->sb, KERN_INFO, + "Mount Device [%2d]: %20s, %8u, %8x - %8x (zone: %s)", + i, FDEV(i).path, + FDEV(i).total_segments, + FDEV(i).start_blk, FDEV(i).end_blk, + bdev_zoned_model(FDEV(i).bdev) == BLK_ZONED_HA ? + "Host-aware" : "Host-managed"); + continue; + } +#endif + f2fs_msg(sbi->sb, KERN_INFO, + "Mount Device [%2d]: %20s, %8u, %8x - %8x", + i, FDEV(i).path, + FDEV(i).total_segments, + FDEV(i).start_blk, FDEV(i).end_blk); + } + return 0; +} + static int f2fs_fill_super(struct super_block *sb, void *data, int silent) { struct f2fs_sb_info *sbi; @@ -1641,6 +1813,18 @@ try_onemore: sb->s_fs_info = sbi; sbi->raw_super = raw_super; + /* + * The BLKZONED feature indicates that the drive was formatted with + * zone alignment optimization. This is optional for host-aware + * devices, but mandatory for host-managed zoned block devices. + */ +#ifndef CONFIG_BLK_DEV_ZONED + if (f2fs_sb_mounted_blkzoned(sb)) { + f2fs_msg(sb, KERN_ERR, + "Zoned block device support is not enabled\n"); + goto free_sb_buf; + } +#endif default_options(sbi); /* parse mount options */ options = kstrdup((const char *)data, GFP_KERNEL); @@ -1710,6 +1894,13 @@ try_onemore: goto free_meta_inode; } + /* Initialize device list */ + err = f2fs_scan_devices(sbi); + if (err) { + f2fs_msg(sb, KERN_ERR, "Failed to find devices"); + goto free_devices; + } + sbi->total_valid_node_count = le32_to_cpu(sbi->ckpt->valid_node_count); percpu_counter_set(&sbi->total_valid_inode_count, @@ -1893,12 +2084,21 @@ free_node_inode: mutex_lock(&sbi->umount_mutex); release_ino_entry(sbi, true); f2fs_leave_shrinker(sbi); + /* + * Some dirty meta pages can be produced by recover_orphan_inodes() + * failed by EIO. Then, iput(node_inode) can trigger balance_fs_bg() + * followed by write_checkpoint() through f2fs_write_node_pages(), which + * falls into an infinite loop in sync_meta_pages(). + */ + truncate_inode_pages_final(META_MAPPING(sbi)); iput(sbi->node_inode); mutex_unlock(&sbi->umount_mutex); free_nm: destroy_node_manager(sbi); free_sm: destroy_segment_manager(sbi); +free_devices: + destroy_device_list(sbi); kfree(sbi->ckpt); free_meta_inode: make_bad_inode(sbi->meta_inode); @@ -2044,3 +2244,4 @@ module_exit(exit_f2fs_fs) MODULE_AUTHOR("Samsung Electronics's Praesto Team"); MODULE_DESCRIPTION("Flash Friendly File System"); MODULE_LICENSE("GPL"); + diff --git a/fs/f2fs/xattr.c b/fs/f2fs/xattr.c index 3e1c028..c47ce2f 100644 --- a/fs/f2fs/xattr.c +++ b/fs/f2fs/xattr.c @@ -106,7 +106,7 @@ static int f2fs_xattr_advise_set(const struct xattr_handler *handler, return -EINVAL; F2FS_I(inode)->i_advise |= *(char *)value; - f2fs_mark_inode_dirty_sync(inode); + f2fs_mark_inode_dirty_sync(inode, true); return 0; } @@ -554,7 +554,7 @@ static int __f2fs_setxattr(struct inode *inode, int index, if (index == F2FS_XATTR_INDEX_ENCRYPTION && !strcmp(name, F2FS_XATTR_NAME_ENCRYPTION_CONTEXT)) f2fs_set_encrypted_inode(inode); - f2fs_mark_inode_dirty_sync(inode); + f2fs_mark_inode_dirty_sync(inode, true); if (!error && S_ISDIR(inode->i_mode)) set_sbi_flag(F2FS_I_SB(inode), SBI_NEED_CP); exit: @@ -467,8 +467,9 @@ int iomap_page_mkwrite(struct vm_area_struct *vma, struct vm_fault *vmf, offset = page_offset(page); while (length > 0) { - ret = iomap_apply(inode, offset, length, IOMAP_WRITE, - ops, page, iomap_page_mkwrite_actor); + ret = iomap_apply(inode, offset, length, + IOMAP_WRITE | IOMAP_FAULT, ops, page, + iomap_page_mkwrite_actor); if (unlikely(ret <= 0)) goto out_unlock; offset += ret; diff --git a/fs/jfs/ioctl.c b/fs/jfs/ioctl.c index 8653cac..b6fd1ff 100644 --- a/fs/jfs/ioctl.c +++ b/fs/jfs/ioctl.c @@ -121,7 +121,7 @@ long jfs_ioctl(struct file *filp, unsigned int cmd, unsigned long arg) jfs_set_inode_flags(inode); inode_unlock(inode); - inode->i_ctime = CURRENT_TIME_SEC; + inode->i_ctime = current_time(inode); mark_inode_dirty(inode); setflags_out: mnt_drop_write_file(filp); diff --git a/fs/mbcache.c b/fs/mbcache.c index c5bd19f..b19be429d 100644 --- a/fs/mbcache.c +++ b/fs/mbcache.c @@ -29,7 +29,7 @@ struct mb_cache { /* log2 of hash table size */ int c_bucket_bits; /* Maximum entries in cache to avoid degrading hash too much */ - int c_max_entries; + unsigned long c_max_entries; /* Protects c_list, c_entry_count */ spinlock_t c_list_lock; struct list_head c_list; @@ -43,7 +43,7 @@ struct mb_cache { static struct kmem_cache *mb_entry_cache; static unsigned long mb_cache_shrink(struct mb_cache *cache, - unsigned int nr_to_scan); + unsigned long nr_to_scan); static inline struct hlist_bl_head *mb_cache_entry_head(struct mb_cache *cache, u32 key) @@ -155,12 +155,12 @@ out: } /* - * mb_cache_entry_find_first - find the first entry in cache with given key + * mb_cache_entry_find_first - find the first reusable entry with the given key * @cache: cache where we should search * @key: key to look for * - * Search in @cache for entry with key @key. Grabs reference to the first - * entry found and returns the entry. + * Search in @cache for a reusable entry with key @key. Grabs reference to the + * first reusable entry found and returns the entry. */ struct mb_cache_entry *mb_cache_entry_find_first(struct mb_cache *cache, u32 key) @@ -170,14 +170,14 @@ struct mb_cache_entry *mb_cache_entry_find_first(struct mb_cache *cache, EXPORT_SYMBOL(mb_cache_entry_find_first); /* - * mb_cache_entry_find_next - find next entry in cache with the same + * mb_cache_entry_find_next - find next reusable entry with the same key * @cache: cache where we should search * @entry: entry to start search from * - * Finds next entry in the hash chain which has the same key as @entry. - * If @entry is unhashed (which can happen when deletion of entry races - * with the search), finds the first entry in the hash chain. The function - * drops reference to @entry and returns with a reference to the found entry. + * Finds next reusable entry in the hash chain which has the same key as @entry. + * If @entry is unhashed (which can happen when deletion of entry races with the + * search), finds the first reusable entry in the hash chain. The function drops + * reference to @entry and returns with a reference to the found entry. */ struct mb_cache_entry *mb_cache_entry_find_next(struct mb_cache *cache, struct mb_cache_entry *entry) @@ -274,11 +274,11 @@ static unsigned long mb_cache_count(struct shrinker *shrink, /* Shrink number of entries in cache */ static unsigned long mb_cache_shrink(struct mb_cache *cache, - unsigned int nr_to_scan) + unsigned long nr_to_scan) { struct mb_cache_entry *entry; struct hlist_bl_head *head; - unsigned int shrunk = 0; + unsigned long shrunk = 0; spin_lock(&cache->c_list_lock); while (nr_to_scan-- && !list_empty(&cache->c_list)) { @@ -286,7 +286,7 @@ static unsigned long mb_cache_shrink(struct mb_cache *cache, struct mb_cache_entry, e_list); if (entry->e_referenced) { entry->e_referenced = 0; - list_move_tail(&cache->c_list, &entry->e_list); + list_move_tail(&entry->e_list, &cache->c_list); continue; } list_del_init(&entry->e_list); @@ -316,10 +316,9 @@ static unsigned long mb_cache_shrink(struct mb_cache *cache, static unsigned long mb_cache_scan(struct shrinker *shrink, struct shrink_control *sc) { - int nr_to_scan = sc->nr_to_scan; struct mb_cache *cache = container_of(shrink, struct mb_cache, c_shrink); - return mb_cache_shrink(cache, nr_to_scan); + return mb_cache_shrink(cache, sc->nr_to_scan); } /* We shrink 1/X of the cache when we have too many entries in it */ @@ -341,11 +340,8 @@ static void mb_cache_shrink_worker(struct work_struct *work) struct mb_cache *mb_cache_create(int bucket_bits) { struct mb_cache *cache; - int bucket_count = 1 << bucket_bits; - int i; - - if (!try_module_get(THIS_MODULE)) - return NULL; + unsigned long bucket_count = 1UL << bucket_bits; + unsigned long i; cache = kzalloc(sizeof(struct mb_cache), GFP_KERNEL); if (!cache) @@ -377,7 +373,6 @@ struct mb_cache *mb_cache_create(int bucket_bits) return cache; err_out: - module_put(THIS_MODULE); return NULL; } EXPORT_SYMBOL(mb_cache_create); @@ -411,7 +406,6 @@ void mb_cache_destroy(struct mb_cache *cache) } kfree(cache->c_hash); kfree(cache); - module_put(THIS_MODULE); } EXPORT_SYMBOL(mb_cache_destroy); @@ -420,7 +414,8 @@ static int __init mbcache_init(void) mb_entry_cache = kmem_cache_create("mbcache", sizeof(struct mb_cache_entry), 0, SLAB_RECLAIM_ACCOUNT|SLAB_MEM_SPREAD, NULL); - BUG_ON(!mb_entry_cache); + if (!mb_entry_cache) + return -ENOMEM; return 0; } diff --git a/fs/proc/generic.c b/fs/proc/generic.c index 5f2dc20..7eb3cef 100644 --- a/fs/proc/generic.c +++ b/fs/proc/generic.c @@ -479,6 +479,7 @@ struct proc_dir_entry *proc_create_mount_point(const char *name) } return ent; } +EXPORT_SYMBOL(proc_create_mount_point); struct proc_dir_entry *proc_create_data(const char *name, umode_t mode, struct proc_dir_entry *parent, diff --git a/fs/proc/internal.h b/fs/proc/internal.h index bbba5d2..109876a2 100644 --- a/fs/proc/internal.h +++ b/fs/proc/internal.h @@ -195,7 +195,6 @@ static inline bool is_empty_pde(const struct proc_dir_entry *pde) { return S_ISDIR(pde->mode) && !pde->proc_iops; } -struct proc_dir_entry *proc_create_mount_point(const char *name); /* * inode.c diff --git a/fs/xfs/xfs_aops.c b/fs/xfs/xfs_aops.c index 6be5204..38755ca 100644 --- a/fs/xfs/xfs_aops.c +++ b/fs/xfs/xfs_aops.c @@ -1297,8 +1297,7 @@ __xfs_get_blocks( sector_t iblock, struct buffer_head *bh_result, int create, - bool direct, - bool dax_fault) + bool direct) { struct xfs_inode *ip = XFS_I(inode); struct xfs_mount *mp = ip->i_mount; @@ -1419,13 +1418,8 @@ __xfs_get_blocks( if (ISUNWRITTEN(&imap)) set_buffer_unwritten(bh_result); /* direct IO needs special help */ - if (create) { - if (dax_fault) - ASSERT(!ISUNWRITTEN(&imap)); - else - xfs_map_direct(inode, bh_result, &imap, offset, - is_cow); - } + if (create) + xfs_map_direct(inode, bh_result, &imap, offset, is_cow); } /* @@ -1465,7 +1459,7 @@ xfs_get_blocks( struct buffer_head *bh_result, int create) { - return __xfs_get_blocks(inode, iblock, bh_result, create, false, false); + return __xfs_get_blocks(inode, iblock, bh_result, create, false); } int @@ -1475,17 +1469,7 @@ xfs_get_blocks_direct( struct buffer_head *bh_result, int create) { - return __xfs_get_blocks(inode, iblock, bh_result, create, true, false); -} - -int -xfs_get_blocks_dax_fault( - struct inode *inode, - sector_t iblock, - struct buffer_head *bh_result, - int create) -{ - return __xfs_get_blocks(inode, iblock, bh_result, create, true, true); + return __xfs_get_blocks(inode, iblock, bh_result, create, true); } /* diff --git a/fs/xfs/xfs_aops.h b/fs/xfs/xfs_aops.h index b3c6634..34dc00d 100644 --- a/fs/xfs/xfs_aops.h +++ b/fs/xfs/xfs_aops.h @@ -59,9 +59,6 @@ int xfs_get_blocks(struct inode *inode, sector_t offset, struct buffer_head *map_bh, int create); int xfs_get_blocks_direct(struct inode *inode, sector_t offset, struct buffer_head *map_bh, int create); -int xfs_get_blocks_dax_fault(struct inode *inode, sector_t offset, - struct buffer_head *map_bh, int create); - int xfs_end_io_direct_write(struct kiocb *iocb, loff_t offset, ssize_t size, void *private); int xfs_setfilesize(struct xfs_inode *ip, xfs_off_t offset, size_t size); diff --git a/fs/xfs/xfs_file.c b/fs/xfs/xfs_file.c index 6e4f7f9..d818c16 100644 --- a/fs/xfs/xfs_file.c +++ b/fs/xfs/xfs_file.c @@ -318,7 +318,7 @@ xfs_file_dax_read( return 0; /* skip atime */ xfs_rw_ilock(ip, XFS_IOLOCK_SHARED); - ret = iomap_dax_rw(iocb, to, &xfs_iomap_ops); + ret = dax_iomap_rw(iocb, to, &xfs_iomap_ops); xfs_rw_iunlock(ip, XFS_IOLOCK_SHARED); file_accessed(iocb->ki_filp); @@ -653,7 +653,7 @@ xfs_file_dax_write( trace_xfs_file_dax_write(ip, count, pos); - ret = iomap_dax_rw(iocb, from, &xfs_iomap_ops); + ret = dax_iomap_rw(iocb, from, &xfs_iomap_ops); if (ret > 0 && iocb->ki_pos > i_size_read(inode)) { i_size_write(inode, iocb->ki_pos); error = xfs_setfilesize(ip, pos, ret); @@ -1474,7 +1474,7 @@ xfs_filemap_page_mkwrite( xfs_ilock(XFS_I(inode), XFS_MMAPLOCK_SHARED); if (IS_DAX(inode)) { - ret = iomap_dax_fault(vma, vmf, &xfs_iomap_ops); + ret = dax_iomap_fault(vma, vmf, &xfs_iomap_ops); } else { ret = iomap_page_mkwrite(vma, vmf, &xfs_iomap_ops); ret = block_page_mkwrite_return(ret); @@ -1508,7 +1508,7 @@ xfs_filemap_fault( * changes to xfs_get_blocks_direct() to map unwritten extent * ioend for conversion on read-only mappings. */ - ret = iomap_dax_fault(vma, vmf, &xfs_iomap_ops); + ret = dax_iomap_fault(vma, vmf, &xfs_iomap_ops); } else ret = filemap_fault(vma, vmf); xfs_iunlock(XFS_I(inode), XFS_MMAPLOCK_SHARED); @@ -1545,7 +1545,7 @@ xfs_filemap_pmd_fault( } xfs_ilock(XFS_I(inode), XFS_MMAPLOCK_SHARED); - ret = dax_pmd_fault(vma, addr, pmd, flags, xfs_get_blocks_dax_fault); + ret = dax_iomap_pmd_fault(vma, addr, pmd, flags, &xfs_iomap_ops); xfs_iunlock(XFS_I(inode), XFS_MMAPLOCK_SHARED); if (flags & FAULT_FLAG_WRITE) diff --git a/include/linux/ahci-remap.h b/include/linux/ahci-remap.h new file mode 100644 index 0000000..62be3a4 --- /dev/null +++ b/include/linux/ahci-remap.h @@ -0,0 +1,28 @@ +#ifndef _LINUX_AHCI_REMAP_H +#define _LINUX_AHCI_REMAP_H + +#include <linux/sizes.h> + +#define AHCI_VSCAP 0xa4 +#define AHCI_REMAP_CAP 0x800 + +/* device class code */ +#define AHCI_REMAP_N_DCC 0x880 + +/* remap-device base relative to ahci-bar */ +#define AHCI_REMAP_N_OFFSET SZ_16K +#define AHCI_REMAP_N_SIZE SZ_16K + +#define AHCI_MAX_REMAP 3 + +static inline unsigned int ahci_remap_dcc(int i) +{ + return AHCI_REMAP_N_DCC + i * 0x80; +} + +static inline unsigned int ahci_remap_base(int i) +{ + return AHCI_REMAP_N_OFFSET + i * AHCI_REMAP_N_SIZE; +} + +#endif /* _LINUX_AHCI_REMAP_H */ diff --git a/include/linux/ata.h b/include/linux/ata.h index fdb1803..af6859b 100644 --- a/include/linux/ata.h +++ b/include/linux/ata.h @@ -348,6 +348,7 @@ enum { ATA_LOG_DEVSLP_DETO = 0x01, ATA_LOG_DEVSLP_VALID = 0x07, ATA_LOG_DEVSLP_VALID_MASK = 0x80, + ATA_LOG_NCQ_PRIO_OFFSET = 0x09, /* NCQ send and receive log */ ATA_LOG_NCQ_SEND_RECV_SUBCMDS_OFFSET = 0x00, @@ -940,6 +941,11 @@ static inline bool ata_id_has_ncq_non_data(const u16 *id) return id[ATA_ID_SATA_CAPABILITY_2] & BIT(5); } +static inline bool ata_id_has_ncq_prio(const u16 *id) +{ + return id[ATA_ID_SATA_CAPABILITY] & BIT(12); +} + static inline bool ata_id_has_trim(const u16 *id) { if (ata_id_major_version(id) >= 7 && diff --git a/include/linux/blk-mq.h b/include/linux/blk-mq.h index 87e404a..4a2ab5d9 100644 --- a/include/linux/blk-mq.h +++ b/include/linux/blk-mq.h @@ -242,6 +242,7 @@ void blk_mq_unfreeze_queue(struct request_queue *q); void blk_mq_freeze_queue_start(struct request_queue *q); int blk_mq_reinit_tagset(struct blk_mq_tag_set *set); +int blk_mq_map_queues(struct blk_mq_tag_set *set); void blk_mq_update_nr_hw_queues(struct blk_mq_tag_set *set, int nr_hw_queues); /* diff --git a/include/linux/blkdev.h b/include/linux/blkdev.h index c539376..286b2a2 100644 --- a/include/linux/blkdev.h +++ b/include/linux/blkdev.h @@ -1059,6 +1059,20 @@ static inline unsigned int blk_rq_count_bios(struct request *rq) } /* + * blk_rq_set_prio - associate a request with prio from ioc + * @rq: request of interest + * @ioc: target iocontext + * + * Assocate request prio with ioc prio so request based drivers + * can leverage priority information. + */ +static inline void blk_rq_set_prio(struct request *rq, struct io_context *ioc) +{ + if (ioc) + rq->ioprio = ioc->ioprio; +} + +/* * Request issue related functions. */ extern struct request *blk_peek_request(struct request_queue *q); diff --git a/include/linux/bsg-lib.h b/include/linux/bsg-lib.h index a226652..657a718 100644 --- a/include/linux/bsg-lib.h +++ b/include/linux/bsg-lib.h @@ -40,6 +40,8 @@ struct bsg_job { struct device *dev; struct request *req; + struct kref kref; + /* Transport/driver specific request/reply structs */ void *request; void *reply; @@ -67,5 +69,7 @@ void bsg_job_done(struct bsg_job *job, int result, int bsg_setup_queue(struct device *dev, struct request_queue *q, char *name, bsg_job_fn *job_fn, int dd_job_size); void bsg_request_fn(struct request_queue *q); +void bsg_job_put(struct bsg_job *job); +int __must_check bsg_job_get(struct bsg_job *job); #endif diff --git a/include/linux/configfs.h b/include/linux/configfs.h index d9d6a9d..9a30b92 100644 --- a/include/linux/configfs.h +++ b/include/linux/configfs.h @@ -228,7 +228,7 @@ static struct configfs_bin_attribute _pfx##attr_##_name = { \ struct configfs_item_operations { void (*release)(struct config_item *); int (*allow_link)(struct config_item *src, struct config_item *target); - int (*drop_link)(struct config_item *src, struct config_item *target); + void (*drop_link)(struct config_item *src, struct config_item *target); }; struct configfs_group_operations { diff --git a/include/linux/dax.h b/include/linux/dax.h index add6c4b..0afade8 100644 --- a/include/linux/dax.h +++ b/include/linux/dax.h @@ -8,21 +8,41 @@ struct iomap_ops; -/* We use lowest available exceptional entry bit for locking */ +/* + * We use lowest available bit in exceptional entry for locking, one bit for + * the entry size (PMD) and two more to tell us if the entry is a huge zero + * page (HZP) or an empty entry that is just used for locking. In total four + * special bits. + * + * If the PMD bit isn't set the entry has size PAGE_SIZE, and if the HZP and + * EMPTY bits aren't set the entry is a normal DAX entry with a filesystem + * block allocation. + */ +#define RADIX_DAX_SHIFT (RADIX_TREE_EXCEPTIONAL_SHIFT + 4) #define RADIX_DAX_ENTRY_LOCK (1 << RADIX_TREE_EXCEPTIONAL_SHIFT) +#define RADIX_DAX_PMD (1 << (RADIX_TREE_EXCEPTIONAL_SHIFT + 1)) +#define RADIX_DAX_HZP (1 << (RADIX_TREE_EXCEPTIONAL_SHIFT + 2)) +#define RADIX_DAX_EMPTY (1 << (RADIX_TREE_EXCEPTIONAL_SHIFT + 3)) -ssize_t iomap_dax_rw(struct kiocb *iocb, struct iov_iter *iter, +static inline unsigned long dax_radix_sector(void *entry) +{ + return (unsigned long)entry >> RADIX_DAX_SHIFT; +} + +static inline void *dax_radix_locked_entry(sector_t sector, unsigned long flags) +{ + return (void *)(RADIX_TREE_EXCEPTIONAL_ENTRY | flags | + ((unsigned long)sector << RADIX_DAX_SHIFT) | + RADIX_DAX_ENTRY_LOCK); +} + +ssize_t dax_iomap_rw(struct kiocb *iocb, struct iov_iter *iter, struct iomap_ops *ops); -ssize_t dax_do_io(struct kiocb *, struct inode *, struct iov_iter *, - get_block_t, dio_iodone_t, int flags); -int dax_zero_page_range(struct inode *, loff_t from, unsigned len, get_block_t); -int dax_truncate_page(struct inode *, loff_t from, get_block_t); -int iomap_dax_fault(struct vm_area_struct *vma, struct vm_fault *vmf, +int dax_iomap_fault(struct vm_area_struct *vma, struct vm_fault *vmf, struct iomap_ops *ops); -int dax_fault(struct vm_area_struct *, struct vm_fault *, get_block_t); int dax_delete_mapping_entry(struct address_space *mapping, pgoff_t index); void dax_wake_mapping_entry_waiter(struct address_space *mapping, - pgoff_t index, bool wake_all); + pgoff_t index, void *entry, bool wake_all); #ifdef CONFIG_FS_DAX struct page *read_dax_sector(struct block_device *bdev, sector_t n); @@ -48,18 +68,28 @@ static inline int __dax_zero_page_range(struct block_device *bdev, } #endif -#if defined(CONFIG_TRANSPARENT_HUGEPAGE) -int dax_pmd_fault(struct vm_area_struct *, unsigned long addr, pmd_t *, - unsigned int flags, get_block_t); +#ifdef CONFIG_FS_DAX_PMD +static inline unsigned int dax_radix_order(void *entry) +{ + if ((unsigned long)entry & RADIX_DAX_PMD) + return PMD_SHIFT - PAGE_SHIFT; + return 0; +} +int dax_iomap_pmd_fault(struct vm_area_struct *vma, unsigned long address, + pmd_t *pmd, unsigned int flags, struct iomap_ops *ops); #else -static inline int dax_pmd_fault(struct vm_area_struct *vma, unsigned long addr, - pmd_t *pmd, unsigned int flags, get_block_t gb) +static inline unsigned int dax_radix_order(void *entry) +{ + return 0; +} +static inline int dax_iomap_pmd_fault(struct vm_area_struct *vma, + unsigned long address, pmd_t *pmd, unsigned int flags, + struct iomap_ops *ops) { return VM_FAULT_FALLBACK; } #endif int dax_pfn_mkwrite(struct vm_area_struct *, struct vm_fault *); -#define dax_mkwrite(vma, vmf, gb) dax_fault(vma, vmf, gb) static inline bool vma_is_dax(struct vm_area_struct *vma) { diff --git a/include/linux/f2fs_fs.h b/include/linux/f2fs_fs.h index 422630b..cea41a1 100644 --- a/include/linux/f2fs_fs.h +++ b/include/linux/f2fs_fs.h @@ -52,10 +52,17 @@ #define VERSION_LEN 256 #define MAX_VOLUME_NAME 512 +#define MAX_PATH_LEN 64 +#define MAX_DEVICES 8 /* * For superblock */ +struct f2fs_device { + __u8 path[MAX_PATH_LEN]; + __le32 total_segments; +} __packed; + struct f2fs_super_block { __le32 magic; /* Magic Number */ __le16 major_ver; /* Major Version */ @@ -94,7 +101,8 @@ struct f2fs_super_block { __le32 feature; /* defined features */ __u8 encryption_level; /* versioning level for encryption */ __u8 encrypt_pw_salt[16]; /* Salt used for string2key algorithm */ - __u8 reserved[871]; /* valid reserved region */ + struct f2fs_device devs[MAX_DEVICES]; /* device list */ + __u8 reserved[327]; /* valid reserved region */ } __packed; /* diff --git a/include/linux/fscrypto.h b/include/linux/fscrypto.h index ff8b11b..c074b67 100644 --- a/include/linux/fscrypto.h +++ b/include/linux/fscrypto.h @@ -18,73 +18,9 @@ #include <crypto/skcipher.h> #include <uapi/linux/fs.h> -#define FS_KEY_DERIVATION_NONCE_SIZE 16 -#define FS_ENCRYPTION_CONTEXT_FORMAT_V1 1 - -#define FS_POLICY_FLAGS_PAD_4 0x00 -#define FS_POLICY_FLAGS_PAD_8 0x01 -#define FS_POLICY_FLAGS_PAD_16 0x02 -#define FS_POLICY_FLAGS_PAD_32 0x03 -#define FS_POLICY_FLAGS_PAD_MASK 0x03 -#define FS_POLICY_FLAGS_VALID 0x03 - -/* Encryption algorithms */ -#define FS_ENCRYPTION_MODE_INVALID 0 -#define FS_ENCRYPTION_MODE_AES_256_XTS 1 -#define FS_ENCRYPTION_MODE_AES_256_GCM 2 -#define FS_ENCRYPTION_MODE_AES_256_CBC 3 -#define FS_ENCRYPTION_MODE_AES_256_CTS 4 - -/** - * Encryption context for inode - * - * Protector format: - * 1 byte: Protector format (1 = this version) - * 1 byte: File contents encryption mode - * 1 byte: File names encryption mode - * 1 byte: Flags - * 8 bytes: Master Key descriptor - * 16 bytes: Encryption Key derivation nonce - */ -struct fscrypt_context { - u8 format; - u8 contents_encryption_mode; - u8 filenames_encryption_mode; - u8 flags; - u8 master_key_descriptor[FS_KEY_DESCRIPTOR_SIZE]; - u8 nonce[FS_KEY_DERIVATION_NONCE_SIZE]; -} __packed; - -/* Encryption parameters */ -#define FS_XTS_TWEAK_SIZE 16 -#define FS_AES_128_ECB_KEY_SIZE 16 -#define FS_AES_256_GCM_KEY_SIZE 32 -#define FS_AES_256_CBC_KEY_SIZE 32 -#define FS_AES_256_CTS_KEY_SIZE 32 -#define FS_AES_256_XTS_KEY_SIZE 64 -#define FS_MAX_KEY_SIZE 64 - -#define FS_KEY_DESC_PREFIX "fscrypt:" -#define FS_KEY_DESC_PREFIX_SIZE 8 - -/* This is passed in from userspace into the kernel keyring */ -struct fscrypt_key { - u32 mode; - u8 raw[FS_MAX_KEY_SIZE]; - u32 size; -} __packed; - -struct fscrypt_info { - u8 ci_data_mode; - u8 ci_filename_mode; - u8 ci_flags; - struct crypto_skcipher *ci_ctfm; - struct key *ci_keyring_key; - u8 ci_master_key[FS_KEY_DESCRIPTOR_SIZE]; -}; +#define FS_CRYPTO_BLOCK_SIZE 16 -#define FS_CTX_REQUIRES_FREE_ENCRYPT_FL 0x00000001 -#define FS_WRITE_PATH_FL 0x00000002 +struct fscrypt_info; struct fscrypt_ctx { union { @@ -102,19 +38,6 @@ struct fscrypt_ctx { u8 mode; /* Encryption mode for tfm */ }; -struct fscrypt_completion_result { - struct completion completion; - int res; -}; - -#define DECLARE_FS_COMPLETION_RESULT(ecr) \ - struct fscrypt_completion_result ecr = { \ - COMPLETION_INITIALIZER((ecr).completion), 0 } - -#define FS_FNAME_NUM_SCATTER_ENTRIES 4 -#define FS_CRYPTO_BLOCK_SIZE 16 -#define FS_FNAME_CRYPTO_DIGEST_SIZE 32 - /** * For encrypted symlinks, the ciphertext length is stored at the beginning * of the string in little-endian format. @@ -154,9 +77,15 @@ struct fscrypt_name { #define fname_len(p) ((p)->disk_name.len) /* + * fscrypt superblock flags + */ +#define FS_CFLG_OWN_PAGES (1U << 1) + +/* * crypto opertions for filesystems */ struct fscrypt_operations { + unsigned int flags; int (*get_context)(struct inode *, void *, size_t); int (*key_prefix)(struct inode *, u8 **); int (*prepare_context)(struct inode *); @@ -206,7 +135,7 @@ static inline struct page *fscrypt_control_page(struct page *page) #endif } -static inline int fscrypt_has_encryption_key(struct inode *inode) +static inline int fscrypt_has_encryption_key(const struct inode *inode) { #if IS_ENABLED(CONFIG_FS_ENCRYPTION) return (inode->i_crypt_info != NULL); @@ -238,25 +167,25 @@ static inline void fscrypt_set_d_op(struct dentry *dentry) #if IS_ENABLED(CONFIG_FS_ENCRYPTION) /* crypto.c */ extern struct kmem_cache *fscrypt_info_cachep; -int fscrypt_initialize(void); - -extern struct fscrypt_ctx *fscrypt_get_ctx(struct inode *, gfp_t); +extern struct fscrypt_ctx *fscrypt_get_ctx(const struct inode *, gfp_t); extern void fscrypt_release_ctx(struct fscrypt_ctx *); -extern struct page *fscrypt_encrypt_page(struct inode *, struct page *, gfp_t); -extern int fscrypt_decrypt_page(struct page *); +extern struct page *fscrypt_encrypt_page(const struct inode *, struct page *, + unsigned int, unsigned int, + u64, gfp_t); +extern int fscrypt_decrypt_page(const struct inode *, struct page *, unsigned int, + unsigned int, u64); extern void fscrypt_decrypt_bio_pages(struct fscrypt_ctx *, struct bio *); extern void fscrypt_pullback_bio_page(struct page **, bool); extern void fscrypt_restore_control_page(struct page *); -extern int fscrypt_zeroout_range(struct inode *, pgoff_t, sector_t, +extern int fscrypt_zeroout_range(const struct inode *, pgoff_t, sector_t, unsigned int); /* policy.c */ -extern int fscrypt_process_policy(struct file *, const struct fscrypt_policy *); -extern int fscrypt_get_policy(struct inode *, struct fscrypt_policy *); +extern int fscrypt_ioctl_set_policy(struct file *, const void __user *); +extern int fscrypt_ioctl_get_policy(struct file *, void __user *); extern int fscrypt_has_permitted_context(struct inode *, struct inode *); extern int fscrypt_inherit_context(struct inode *, struct inode *, void *, bool); /* keyinfo.c */ -extern int get_crypt_info(struct inode *); extern int fscrypt_get_encryption_info(struct inode *); extern void fscrypt_put_encryption_info(struct inode *, struct fscrypt_info *); @@ -264,8 +193,8 @@ extern void fscrypt_put_encryption_info(struct inode *, struct fscrypt_info *); extern int fscrypt_setup_filename(struct inode *, const struct qstr *, int lookup, struct fscrypt_name *); extern void fscrypt_free_filename(struct fscrypt_name *); -extern u32 fscrypt_fname_encrypted_size(struct inode *, u32); -extern int fscrypt_fname_alloc_buffer(struct inode *, u32, +extern u32 fscrypt_fname_encrypted_size(const struct inode *, u32); +extern int fscrypt_fname_alloc_buffer(const struct inode *, u32, struct fscrypt_str *); extern void fscrypt_fname_free_buffer(struct fscrypt_str *); extern int fscrypt_fname_disk_to_usr(struct inode *, u32, u32, @@ -275,7 +204,7 @@ extern int fscrypt_fname_usr_to_disk(struct inode *, const struct qstr *, #endif /* crypto.c */ -static inline struct fscrypt_ctx *fscrypt_notsupp_get_ctx(struct inode *i, +static inline struct fscrypt_ctx *fscrypt_notsupp_get_ctx(const struct inode *i, gfp_t f) { return ERR_PTR(-EOPNOTSUPP); @@ -286,13 +215,18 @@ static inline void fscrypt_notsupp_release_ctx(struct fscrypt_ctx *c) return; } -static inline struct page *fscrypt_notsupp_encrypt_page(struct inode *i, - struct page *p, gfp_t f) +static inline struct page *fscrypt_notsupp_encrypt_page(const struct inode *i, + struct page *p, + unsigned int len, + unsigned int offs, + u64 lblk_num, gfp_t f) { return ERR_PTR(-EOPNOTSUPP); } -static inline int fscrypt_notsupp_decrypt_page(struct page *p) +static inline int fscrypt_notsupp_decrypt_page(const struct inode *i, struct page *p, + unsigned int len, unsigned int offs, + u64 lblk_num) { return -EOPNOTSUPP; } @@ -313,21 +247,21 @@ static inline void fscrypt_notsupp_restore_control_page(struct page *p) return; } -static inline int fscrypt_notsupp_zeroout_range(struct inode *i, pgoff_t p, +static inline int fscrypt_notsupp_zeroout_range(const struct inode *i, pgoff_t p, sector_t s, unsigned int f) { return -EOPNOTSUPP; } /* policy.c */ -static inline int fscrypt_notsupp_process_policy(struct file *f, - const struct fscrypt_policy *p) +static inline int fscrypt_notsupp_ioctl_set_policy(struct file *f, + const void __user *arg) { return -EOPNOTSUPP; } -static inline int fscrypt_notsupp_get_policy(struct inode *i, - struct fscrypt_policy *p) +static inline int fscrypt_notsupp_ioctl_get_policy(struct file *f, + void __user *arg) { return -EOPNOTSUPP; } diff --git a/include/linux/hwmon.h b/include/linux/hwmon.h index 9d2f8bd..78d59db 100644 --- a/include/linux/hwmon.h +++ b/include/linux/hwmon.h @@ -298,8 +298,8 @@ enum hwmon_pwm_attributes { * Channel number * The function returns the file permissions. * If the return value is 0, no attribute will be created. - * @read: Read callback. Optional. If not provided, attributes - * will not be readable. + * @read: Read callback for data attributes. Mandatory if readable + * data attributes are present. * Parameters are: * @dev: Pointer to hardware monitoring device * @type: Sensor type @@ -308,8 +308,19 @@ enum hwmon_pwm_attributes { * Channel number * @val: Pointer to returned value * The function returns 0 on success or a negative error number. - * @write: Write callback. Optional. If not provided, attributes - * will not be writable. + * @read_string: + * Read callback for string attributes. Mandatory if string + * attributes are present. + * Parameters are: + * @dev: Pointer to hardware monitoring device + * @type: Sensor type + * @attr: Sensor attribute + * @channel: + * Channel number + * @str: Pointer to returned string + * The function returns 0 on success or a negative error number. + * @write: Write callback for data attributes. Mandatory if writeable + * data attributes are present. * Parameters are: * @dev: Pointer to hardware monitoring device * @type: Sensor type @@ -324,6 +335,8 @@ struct hwmon_ops { u32 attr, int channel); int (*read)(struct device *dev, enum hwmon_sensor_types type, u32 attr, int channel, long *val); + int (*read_string)(struct device *dev, enum hwmon_sensor_types type, + u32 attr, int channel, char **str); int (*write)(struct device *dev, enum hwmon_sensor_types type, u32 attr, int channel, long val); }; @@ -349,7 +362,9 @@ struct hwmon_chip_info { const struct hwmon_channel_info **info; }; +/* hwmon_device_register() is deprecated */ struct device *hwmon_device_register(struct device *dev); + struct device * hwmon_device_register_with_groups(struct device *dev, const char *name, void *drvdata, @@ -362,12 +377,12 @@ struct device * hwmon_device_register_with_info(struct device *dev, const char *name, void *drvdata, const struct hwmon_chip_info *info, - const struct attribute_group **groups); + const struct attribute_group **extra_groups); struct device * devm_hwmon_device_register_with_info(struct device *dev, - const char *name, void *drvdata, - const struct hwmon_chip_info *info, - const struct attribute_group **groups); + const char *name, void *drvdata, + const struct hwmon_chip_info *info, + const struct attribute_group **extra_groups); void hwmon_device_unregister(struct device *dev); void devm_hwmon_device_unregister(struct device *dev); diff --git a/include/linux/iomap.h b/include/linux/iomap.h index 7892f55..f185156 100644 --- a/include/linux/iomap.h +++ b/include/linux/iomap.h @@ -49,6 +49,7 @@ struct iomap { #define IOMAP_WRITE (1 << 0) /* writing, must allocate blocks */ #define IOMAP_ZERO (1 << 1) /* zeroing operation, may skip holes */ #define IOMAP_REPORT (1 << 2) /* report extent status, e.g. FIEMAP */ +#define IOMAP_FAULT (1 << 3) /* mapping for page fault */ struct iomap_ops { /* diff --git a/include/linux/irqchip/arm-gic-v3.h b/include/linux/irqchip/arm-gic-v3.h index 5118d3a..e808f8a 100644 --- a/include/linux/irqchip/arm-gic-v3.h +++ b/include/linux/irqchip/arm-gic-v3.h @@ -295,10 +295,10 @@ #define GITS_BASER_InnerShareable \ GIC_BASER_SHAREABILITY(GITS_BASER, InnerShareable) #define GITS_BASER_PAGE_SIZE_SHIFT (8) -#define GITS_BASER_PAGE_SIZE_4K (0UL << GITS_BASER_PAGE_SIZE_SHIFT) -#define GITS_BASER_PAGE_SIZE_16K (1UL << GITS_BASER_PAGE_SIZE_SHIFT) -#define GITS_BASER_PAGE_SIZE_64K (2UL << GITS_BASER_PAGE_SIZE_SHIFT) -#define GITS_BASER_PAGE_SIZE_MASK (3UL << GITS_BASER_PAGE_SIZE_SHIFT) +#define GITS_BASER_PAGE_SIZE_4K (0ULL << GITS_BASER_PAGE_SIZE_SHIFT) +#define GITS_BASER_PAGE_SIZE_16K (1ULL << GITS_BASER_PAGE_SIZE_SHIFT) +#define GITS_BASER_PAGE_SIZE_64K (2ULL << GITS_BASER_PAGE_SIZE_SHIFT) +#define GITS_BASER_PAGE_SIZE_MASK (3ULL << GITS_BASER_PAGE_SIZE_SHIFT) #define GITS_BASER_PAGES_MAX 256 #define GITS_BASER_PAGES_SHIFT (0) #define GITS_BASER_NR_PAGES(r) (((r) & 0xff) + 1) diff --git a/include/linux/kernel.h b/include/linux/kernel.h index 01b6b46..d234cd3 100644 --- a/include/linux/kernel.h +++ b/include/linux/kernel.h @@ -45,6 +45,7 @@ #define REPEAT_BYTE(x) ((~0ul / 0xff) * (x)) +/* @a is a power of 2 value */ #define ALIGN(x, a) __ALIGN_KERNEL((x), (a)) #define __ALIGN_MASK(x, mask) __ALIGN_KERNEL_MASK((x), (mask)) #define PTR_ALIGN(p, a) ((typeof(p))ALIGN((unsigned long)(p), (a))) diff --git a/include/linux/kvm_host.h b/include/linux/kvm_host.h index 81ba3ba..1c5190d 100644 --- a/include/linux/kvm_host.h +++ b/include/linux/kvm_host.h @@ -438,6 +438,9 @@ struct kvm { pr_info("kvm [%i]: " fmt, task_pid_nr(current), ## __VA_ARGS__) #define kvm_debug(fmt, ...) \ pr_debug("kvm [%i]: " fmt, task_pid_nr(current), ## __VA_ARGS__) +#define kvm_debug_ratelimited(fmt, ...) \ + pr_debug_ratelimited("kvm [%i]: " fmt, task_pid_nr(current), \ + ## __VA_ARGS__) #define kvm_pr_unimpl(fmt, ...) \ pr_err_ratelimited("kvm [%i]: " fmt, \ task_tgid_nr(current), ## __VA_ARGS__) @@ -449,6 +452,9 @@ struct kvm { #define vcpu_debug(vcpu, fmt, ...) \ kvm_debug("vcpu%i " fmt, (vcpu)->vcpu_id, ## __VA_ARGS__) +#define vcpu_debug_ratelimited(vcpu, fmt, ...) \ + kvm_debug_ratelimited("vcpu%i " fmt, (vcpu)->vcpu_id, \ + ## __VA_ARGS__) #define vcpu_err(vcpu, fmt, ...) \ kvm_err("vcpu%i " fmt, (vcpu)->vcpu_id, ## __VA_ARGS__) @@ -1108,6 +1114,10 @@ static inline bool kvm_check_request(int req, struct kvm_vcpu *vcpu) extern bool kvm_rebooting; +extern unsigned int halt_poll_ns; +extern unsigned int halt_poll_ns_grow; +extern unsigned int halt_poll_ns_shrink; + struct kvm_device { struct kvm_device_ops *ops; struct kvm *kvm; diff --git a/include/linux/libata.h b/include/linux/libata.h index 616eef4..c170be5 100644 --- a/include/linux/libata.h +++ b/include/linux/libata.h @@ -166,6 +166,8 @@ enum { ATA_DFLAG_NO_UNLOAD = (1 << 17), /* device doesn't support unload */ ATA_DFLAG_UNLOCK_HPA = (1 << 18), /* unlock HPA */ ATA_DFLAG_NCQ_SEND_RECV = (1 << 19), /* device supports NCQ SEND and RECV */ + ATA_DFLAG_NCQ_PRIO = (1 << 20), /* device supports NCQ priority */ + ATA_DFLAG_NCQ_PRIO_ENABLE = (1 << 21), /* Priority cmds sent to dev */ ATA_DFLAG_INIT_MASK = (1 << 24) - 1, ATA_DFLAG_DETACH = (1 << 24), @@ -342,7 +344,9 @@ enum { ATA_SHIFT_PIO = 0, ATA_SHIFT_MWDMA = ATA_SHIFT_PIO + ATA_NR_PIO_MODES, ATA_SHIFT_UDMA = ATA_SHIFT_MWDMA + ATA_NR_MWDMA_MODES, + ATA_SHIFT_PRIO = 6, + ATA_PRIO_HIGH = 2, /* size of buffer to pad xfers ending on unaligned boundaries */ ATA_DMA_PAD_SZ = 4, @@ -542,6 +546,7 @@ typedef void (*ata_postreset_fn_t)(struct ata_link *link, unsigned int *classes) extern struct device_attribute dev_attr_link_power_management_policy; extern struct device_attribute dev_attr_unload_heads; +extern struct device_attribute dev_attr_ncq_prio_enable; extern struct device_attribute dev_attr_em_message_type; extern struct device_attribute dev_attr_em_message; extern struct device_attribute dev_attr_sw_activity; diff --git a/include/linux/pci_ids.h b/include/linux/pci_ids.h index c58752f..a5e6c7b 100644 --- a/include/linux/pci_ids.h +++ b/include/linux/pci_ids.h @@ -23,8 +23,10 @@ #define PCI_CLASS_STORAGE_SATA 0x0106 #define PCI_CLASS_STORAGE_SATA_AHCI 0x010601 #define PCI_CLASS_STORAGE_SAS 0x0107 +#define PCI_CLASS_STORAGE_EXPRESS 0x010802 #define PCI_CLASS_STORAGE_OTHER 0x0180 + #define PCI_BASE_CLASS_NETWORK 0x02 #define PCI_CLASS_NETWORK_ETHERNET 0x0200 #define PCI_CLASS_NETWORK_TOKEN_RING 0x0201 diff --git a/include/linux/phy/phy-qcom-ufs.h b/include/linux/phy/phy-qcom-ufs.h index 9d18e9f..35c070e 100644 --- a/include/linux/phy/phy-qcom-ufs.h +++ b/include/linux/phy/phy-qcom-ufs.h @@ -18,22 +18,6 @@ #include "phy.h" /** - * ufs_qcom_phy_enable_ref_clk() - Enable the phy - * ref clock. - * @phy: reference to a generic phy - * - * returns 0 for success, and non-zero for error. - */ -int ufs_qcom_phy_enable_ref_clk(struct phy *phy); - -/** - * ufs_qcom_phy_disable_ref_clk() - Disable the phy - * ref clock. - * @phy: reference to a generic phy. - */ -void ufs_qcom_phy_disable_ref_clk(struct phy *phy); - -/** * ufs_qcom_phy_enable_dev_ref_clk() - Enable the device * ref clock. * @phy: reference to a generic phy. @@ -47,8 +31,6 @@ void ufs_qcom_phy_enable_dev_ref_clk(struct phy *phy); */ void ufs_qcom_phy_disable_dev_ref_clk(struct phy *phy); -int ufs_qcom_phy_enable_iface_clk(struct phy *phy); -void ufs_qcom_phy_disable_iface_clk(struct phy *phy); int ufs_qcom_phy_start_serdes(struct phy *phy); int ufs_qcom_phy_set_tx_lane_enable(struct phy *phy, u32 tx_lanes); int ufs_qcom_phy_calibrate_phy(struct phy *phy, bool is_rate_B); diff --git a/include/linux/proc_fs.h b/include/linux/proc_fs.h index 368c7ad..2d2bf59 100644 --- a/include/linux/proc_fs.h +++ b/include/linux/proc_fs.h @@ -21,6 +21,7 @@ extern struct proc_dir_entry *proc_mkdir_data(const char *, umode_t, struct proc_dir_entry *, void *); extern struct proc_dir_entry *proc_mkdir_mode(const char *, umode_t, struct proc_dir_entry *); +struct proc_dir_entry *proc_create_mount_point(const char *name); extern struct proc_dir_entry *proc_create_data(const char *, umode_t, struct proc_dir_entry *, @@ -56,6 +57,7 @@ static inline struct proc_dir_entry *proc_symlink(const char *name, struct proc_dir_entry *parent,const char *dest) { return NULL;} static inline struct proc_dir_entry *proc_mkdir(const char *name, struct proc_dir_entry *parent) {return NULL;} +static inline struct proc_dir_entry *proc_create_mount_point(const char *name) { return NULL; } static inline struct proc_dir_entry *proc_mkdir_data(const char *name, umode_t mode, struct proc_dir_entry *parent, void *data) { return NULL; } static inline struct proc_dir_entry *proc_mkdir_mode(const char *name, diff --git a/include/linux/restart_block.h b/include/linux/restart_block.h new file mode 100644 index 0000000..0d905d8 --- /dev/null +++ b/include/linux/restart_block.h @@ -0,0 +1,51 @@ +/* + * Common syscall restarting data + */ +#ifndef __LINUX_RESTART_BLOCK_H +#define __LINUX_RESTART_BLOCK_H + +#include <linux/compiler.h> +#include <linux/types.h> + +struct timespec; +struct compat_timespec; +struct pollfd; + +/* + * System call restart block. + */ +struct restart_block { + long (*fn)(struct restart_block *); + union { + /* For futex_wait and futex_wait_requeue_pi */ + struct { + u32 __user *uaddr; + u32 val; + u32 flags; + u32 bitset; + u64 time; + u32 __user *uaddr2; + } futex; + /* For nanosleep */ + struct { + clockid_t clockid; + struct timespec __user *rmtp; +#ifdef CONFIG_COMPAT + struct compat_timespec __user *compat_rmtp; +#endif + u64 expires; + } nanosleep; + /* For poll */ + struct { + struct pollfd __user *ufds; + int nfds; + int has_timeout; + unsigned long tv_sec; + unsigned long tv_nsec; + } poll; + }; +}; + +extern long do_no_restart_syscall(struct restart_block *parm); + +#endif /* __LINUX_RESTART_BLOCK_H */ diff --git a/include/linux/spi/spi.h b/include/linux/spi/spi.h index 4b743ac..75c6bd0 100644 --- a/include/linux/spi/spi.h +++ b/include/linux/spi/spi.h @@ -442,6 +442,7 @@ struct spi_master { #define SPI_MASTER_NO_TX BIT(2) /* can't do buffer write */ #define SPI_MASTER_MUST_RX BIT(3) /* requires rx */ #define SPI_MASTER_MUST_TX BIT(4) /* requires tx */ +#define SPI_MASTER_GPIO_SS BIT(5) /* GPIO CS must select slave */ /* * on some hardware transfer / message size may be constrained diff --git a/include/linux/swiotlb.h b/include/linux/swiotlb.h index 5f81f8a..183f37c 100644 --- a/include/linux/swiotlb.h +++ b/include/linux/swiotlb.h @@ -44,11 +44,13 @@ enum dma_sync_target { extern phys_addr_t swiotlb_tbl_map_single(struct device *hwdev, dma_addr_t tbl_dma_addr, phys_addr_t phys, size_t size, - enum dma_data_direction dir); + enum dma_data_direction dir, + unsigned long attrs); extern void swiotlb_tbl_unmap_single(struct device *hwdev, phys_addr_t tlb_addr, - size_t size, enum dma_data_direction dir); + size_t size, enum dma_data_direction dir, + unsigned long attrs); extern void swiotlb_tbl_sync_single(struct device *hwdev, phys_addr_t tlb_addr, @@ -73,14 +75,6 @@ extern void swiotlb_unmap_page(struct device *hwdev, dma_addr_t dev_addr, unsigned long attrs); extern int -swiotlb_map_sg(struct device *hwdev, struct scatterlist *sg, int nents, - enum dma_data_direction dir); - -extern void -swiotlb_unmap_sg(struct device *hwdev, struct scatterlist *sg, int nents, - enum dma_data_direction dir); - -extern int swiotlb_map_sg_attrs(struct device *hwdev, struct scatterlist *sgl, int nelems, enum dma_data_direction dir, unsigned long attrs); diff --git a/include/linux/thread_info.h b/include/linux/thread_info.h index 2873baf..5837387 100644 --- a/include/linux/thread_info.h +++ b/include/linux/thread_info.h @@ -9,50 +9,17 @@ #include <linux/types.h> #include <linux/bug.h> - -struct timespec; -struct compat_timespec; +#include <linux/restart_block.h> #ifdef CONFIG_THREAD_INFO_IN_TASK -#define current_thread_info() ((struct thread_info *)current) -#endif - /* - * System call restart block. + * For CONFIG_THREAD_INFO_IN_TASK kernels we need <asm/current.h> for the + * definition of current, but for !CONFIG_THREAD_INFO_IN_TASK kernels, + * including <asm/current.h> can cause a circular dependency on some platforms. */ -struct restart_block { - long (*fn)(struct restart_block *); - union { - /* For futex_wait and futex_wait_requeue_pi */ - struct { - u32 __user *uaddr; - u32 val; - u32 flags; - u32 bitset; - u64 time; - u32 __user *uaddr2; - } futex; - /* For nanosleep */ - struct { - clockid_t clockid; - struct timespec __user *rmtp; -#ifdef CONFIG_COMPAT - struct compat_timespec __user *compat_rmtp; +#include <asm/current.h> +#define current_thread_info() ((struct thread_info *)current) #endif - u64 expires; - } nanosleep; - /* For poll */ - struct { - struct pollfd __user *ufds; - int nfds; - int has_timeout; - unsigned long tv_sec; - unsigned long tv_nsec; - } poll; - }; -}; - -extern long do_no_restart_syscall(struct restart_block *parm); #include <linux/bitops.h> #include <asm/thread_info.h> diff --git a/include/linux/workqueue.h b/include/linux/workqueue.h index d4f16cf..a26cc437 100644 --- a/include/linux/workqueue.h +++ b/include/linux/workqueue.h @@ -603,14 +603,6 @@ static inline bool schedule_delayed_work(struct delayed_work *dwork, return queue_delayed_work(system_wq, dwork, delay); } -/** - * keventd_up - is workqueue initialized yet? - */ -static inline bool keventd_up(void) -{ - return system_wq != NULL; -} - #ifndef CONFIG_SMP static inline long work_on_cpu(int cpu, long (*fn)(void *), void *arg) { @@ -645,4 +637,7 @@ int workqueue_online_cpu(unsigned int cpu); int workqueue_offline_cpu(unsigned int cpu); #endif +int __init workqueue_init_early(void); +int __init workqueue_init(void); + #endif diff --git a/include/scsi/libfc.h b/include/scsi/libfc.h index 7428a53..96dd0b3 100644 --- a/include/scsi/libfc.h +++ b/include/scsi/libfc.h @@ -44,6 +44,11 @@ #define FC_NO_ERR 0 /* no error */ #define FC_EX_TIMEOUT 1 /* Exchange timeout */ #define FC_EX_CLOSED 2 /* Exchange closed */ +#define FC_EX_ALLOC_ERR 3 /* Exchange allocation failed */ +#define FC_EX_XMIT_ERR 4 /* Exchange transmit failed */ +#define FC_EX_ELS_RJT 5 /* ELS rejected */ +#define FC_EX_INV_LOGIN 6 /* Login not completed */ +#define FC_EX_SEQ_ERR 6 /* Exchange sequence error */ /** * enum fc_lport_state - Local port states @@ -350,7 +355,8 @@ struct fc_fcp_pkt { /* Timeout/error related information */ struct timer_list timer; - int wait_for_comp; + int wait_for_comp; + int timer_delay; u32 recov_retry; struct fc_seq *recov_seq; struct completion tm_done; @@ -385,6 +391,7 @@ struct fc_seq { #define FC_EX_DONE (1 << 0) /* ep is completed */ #define FC_EX_RST_CLEANUP (1 << 1) /* reset is forcing completion */ +#define FC_EX_QUARANTINE (1 << 2) /* exch is quarantined */ /** * struct fc_exch - Fibre Channel Exchange @@ -478,37 +485,6 @@ struct libfc_function_template { void *arg, u32 timer_msec); /* - * Send the FC frame payload using a new exchange and sequence. - * - * The exchange response handler is set in this routine to resp() - * function pointer. It can be called in two scenarios: if a timeout - * occurs or if a response frame is received for the exchange. The - * fc_frame pointer in response handler will also indicate timeout - * as error using IS_ERR related macros. - * - * The exchange destructor handler is also set in this routine. - * The destructor handler is invoked by EM layer when exchange - * is about to free, this can be used by caller to free its - * resources along with exchange free. - * - * The arg is passed back to resp and destructor handler. - * - * The timeout value (in msec) for an exchange is set if non zero - * timer_msec argument is specified. The timer is canceled when - * it fires or when the exchange is done. The exchange timeout handler - * is registered by EM layer. - * - * STATUS: OPTIONAL - */ - struct fc_seq *(*exch_seq_send)(struct fc_lport *, struct fc_frame *, - void (*resp)(struct fc_seq *, - struct fc_frame *, - void *), - void (*destructor)(struct fc_seq *, - void *), - void *, unsigned int timer_msec); - - /* * Sets up the DDP context for a given exchange id on the given * scatterlist if LLD supports DDP for large receive. * @@ -537,73 +513,6 @@ struct libfc_function_template { * STATUS: OPTIONAL */ void (*get_lesb)(struct fc_lport *, struct fc_els_lesb *lesb); - /* - * Send a frame using an existing sequence and exchange. - * - * STATUS: OPTIONAL - */ - int (*seq_send)(struct fc_lport *, struct fc_seq *, - struct fc_frame *); - - /* - * Send an ELS response using information from the received frame. - * - * STATUS: OPTIONAL - */ - void (*seq_els_rsp_send)(struct fc_frame *, enum fc_els_cmd, - struct fc_seq_els_data *); - - /* - * Abort an exchange and sequence. Generally called because of a - * exchange timeout or an abort from the upper layer. - * - * A timer_msec can be specified for abort timeout, if non-zero - * timer_msec value is specified then exchange resp handler - * will be called with timeout error if no response to abort. - * - * STATUS: OPTIONAL - */ - int (*seq_exch_abort)(const struct fc_seq *, - unsigned int timer_msec); - - /* - * Indicate that an exchange/sequence tuple is complete and the memory - * allocated for the related objects may be freed. - * - * STATUS: OPTIONAL - */ - void (*exch_done)(struct fc_seq *); - - /* - * Start a new sequence on the same exchange/sequence tuple. - * - * STATUS: OPTIONAL - */ - struct fc_seq *(*seq_start_next)(struct fc_seq *); - - /* - * Set a response handler for the exchange of the sequence. - * - * STATUS: OPTIONAL - */ - void (*seq_set_resp)(struct fc_seq *sp, - void (*resp)(struct fc_seq *, struct fc_frame *, - void *), - void *arg); - - /* - * Assign a sequence for an incoming request frame. - * - * STATUS: OPTIONAL - */ - struct fc_seq *(*seq_assign)(struct fc_lport *, struct fc_frame *); - - /* - * Release the reference on the sequence returned by seq_assign(). - * - * STATUS: OPTIONAL - */ - void (*seq_release)(struct fc_seq *); /* * Reset an exchange manager, completing all sequences and exchanges. @@ -615,27 +524,6 @@ struct libfc_function_template { void (*exch_mgr_reset)(struct fc_lport *, u32 s_id, u32 d_id); /* - * Flush the rport work queue. Generally used before shutdown. - * - * STATUS: OPTIONAL - */ - void (*rport_flush_queue)(void); - - /* - * Receive a frame for a local port. - * - * STATUS: OPTIONAL - */ - void (*lport_recv)(struct fc_lport *, struct fc_frame *); - - /* - * Reset the local port. - * - * STATUS: OPTIONAL - */ - int (*lport_reset)(struct fc_lport *); - - /* * Set the local port FC_ID. * * This may be provided by the LLD to allow it to be @@ -656,54 +544,6 @@ struct libfc_function_template { struct fc_frame *); /* - * Create a remote port with a given port ID - * - * STATUS: OPTIONAL - */ - struct fc_rport_priv *(*rport_create)(struct fc_lport *, u32); - - /* - * Initiates the RP state machine. It is called from the LP module. - * This function will issue the following commands to the N_Port - * identified by the FC ID provided. - * - * - PLOGI - * - PRLI - * - RTV - * - * STATUS: OPTIONAL - */ - int (*rport_login)(struct fc_rport_priv *); - - /* - * Logoff, and remove the rport from the transport if - * it had been added. This will send a LOGO to the target. - * - * STATUS: OPTIONAL - */ - int (*rport_logoff)(struct fc_rport_priv *); - - /* - * Receive a request from a remote port. - * - * STATUS: OPTIONAL - */ - void (*rport_recv_req)(struct fc_lport *, struct fc_frame *); - - /* - * lookup an rport by it's port ID. - * - * STATUS: OPTIONAL - */ - struct fc_rport_priv *(*rport_lookup)(const struct fc_lport *, u32); - - /* - * Destroy an rport after final kref_put(). - * The argument is a pointer to the kref inside the fc_rport_priv. - */ - void (*rport_destroy)(struct kref *); - - /* * Callback routine after the remote port is logged in * * STATUS: OPTIONAL @@ -1068,18 +908,26 @@ void fc_vport_setlink(struct fc_lport *); void fc_vports_linkchange(struct fc_lport *); int fc_lport_config(struct fc_lport *); int fc_lport_reset(struct fc_lport *); +void fc_lport_recv(struct fc_lport *lport, struct fc_frame *fp); int fc_set_mfs(struct fc_lport *, u32 mfs); struct fc_lport *libfc_vport_create(struct fc_vport *, int privsize); struct fc_lport *fc_vport_id_lookup(struct fc_lport *, u32 port_id); -int fc_lport_bsg_request(struct fc_bsg_job *); +int fc_lport_bsg_request(struct bsg_job *); void fc_lport_set_local_id(struct fc_lport *, u32 port_id); void fc_lport_iterate(void (*func)(struct fc_lport *, void *), void *); /* * REMOTE PORT LAYER *****************************/ -int fc_rport_init(struct fc_lport *); void fc_rport_terminate_io(struct fc_rport *); +struct fc_rport_priv *fc_rport_lookup(const struct fc_lport *lport, + u32 port_id); +struct fc_rport_priv *fc_rport_create(struct fc_lport *, u32); +void fc_rport_destroy(struct kref *kref); +int fc_rport_login(struct fc_rport_priv *rdata); +int fc_rport_logoff(struct fc_rport_priv *rdata); +void fc_rport_recv_req(struct fc_lport *lport, struct fc_frame *fp); +void fc_rport_flush_queue(void); /* * DISCOVERY LAYER @@ -1131,6 +979,21 @@ void fc_fill_hdr(struct fc_frame *, const struct fc_frame *, *****************************/ int fc_exch_init(struct fc_lport *); void fc_exch_update_stats(struct fc_lport *lport); +struct fc_seq *fc_exch_seq_send(struct fc_lport *lport, + struct fc_frame *fp, + void (*resp)(struct fc_seq *, + struct fc_frame *fp, + void *arg), + void (*destructor)(struct fc_seq *, void *), + void *arg, u32 timer_msec); +void fc_seq_els_rsp_send(struct fc_frame *, enum fc_els_cmd, + struct fc_seq_els_data *); +struct fc_seq *fc_seq_start_next(struct fc_seq *sp); +void fc_seq_set_resp(struct fc_seq *sp, + void (*resp)(struct fc_seq *, struct fc_frame *, void *), + void *arg); +struct fc_seq *fc_seq_assign(struct fc_lport *lport, struct fc_frame *fp); +void fc_seq_release(struct fc_seq *sp); struct fc_exch_mgr_anchor *fc_exch_mgr_add(struct fc_lport *, struct fc_exch_mgr *, bool (*match)(struct fc_frame *)); @@ -1142,6 +1005,9 @@ struct fc_exch_mgr *fc_exch_mgr_alloc(struct fc_lport *, enum fc_class class, void fc_exch_mgr_free(struct fc_lport *); void fc_exch_recv(struct fc_lport *, struct fc_frame *); void fc_exch_mgr_reset(struct fc_lport *, u32 s_id, u32 d_id); +int fc_seq_send(struct fc_lport *lport, struct fc_seq *sp, struct fc_frame *fp); +int fc_seq_exch_abort(const struct fc_seq *, unsigned int timer_msec); +void fc_exch_done(struct fc_seq *sp); /* * Functions for fc_functions_template diff --git a/include/scsi/scsi_host.h b/include/scsi/scsi_host.h index 7e4cd53..36680f1 100644 --- a/include/scsi/scsi_host.h +++ b/include/scsi/scsi_host.h @@ -278,6 +278,14 @@ struct scsi_host_template { int (* change_queue_depth)(struct scsi_device *, int); /* + * This functions lets the driver expose the queue mapping + * to the block layer. + * + * Status: OPTIONAL + */ + int (* map_queues)(struct Scsi_Host *shost); + + /* * This function determines the BIOS parameters for a given * harddisk. These tend to be numbers that are made up by * the host adapter. Parameters: diff --git a/include/scsi/scsi_transport_fc.h b/include/scsi/scsi_transport_fc.h index bf66ea6..924c8e6 100644 --- a/include/scsi/scsi_transport_fc.h +++ b/include/scsi/scsi_transport_fc.h @@ -28,9 +28,11 @@ #define SCSI_TRANSPORT_FC_H #include <linux/sched.h> +#include <linux/bsg-lib.h> #include <asm/unaligned.h> #include <scsi/scsi.h> #include <scsi/scsi_netlink.h> +#include <scsi/scsi_host.h> struct scsi_transport_template; @@ -624,48 +626,6 @@ struct fc_host_attrs { #define fc_host_dev_loss_tmo(x) \ (((struct fc_host_attrs *)(x)->shost_data)->dev_loss_tmo) - -struct fc_bsg_buffer { - unsigned int payload_len; - int sg_cnt; - struct scatterlist *sg_list; -}; - -/* Values for fc_bsg_job->state_flags (bitflags) */ -#define FC_RQST_STATE_INPROGRESS 0 -#define FC_RQST_STATE_DONE 1 - -struct fc_bsg_job { - struct Scsi_Host *shost; - struct fc_rport *rport; - struct device *dev; - struct request *req; - spinlock_t job_lock; - unsigned int state_flags; - unsigned int ref_cnt; - void (*job_done)(struct fc_bsg_job *); - - struct fc_bsg_request *request; - struct fc_bsg_reply *reply; - unsigned int request_len; - unsigned int reply_len; - /* - * On entry : reply_len indicates the buffer size allocated for - * the reply. - * - * Upon completion : the message handler must set reply_len - * to indicates the size of the reply to be returned to the - * caller. - */ - - /* DMA payloads for the request/response */ - struct fc_bsg_buffer request_payload; - struct fc_bsg_buffer reply_payload; - - void *dd_data; /* Used for driver-specific storage */ -}; - - /* The functions by which the transport class and the driver communicate */ struct fc_function_template { void (*get_rport_dev_loss_tmo)(struct fc_rport *); @@ -702,8 +662,8 @@ struct fc_function_template { int (* it_nexus_response)(struct Scsi_Host *, u64, int); /* bsg support */ - int (*bsg_request)(struct fc_bsg_job *); - int (*bsg_timeout)(struct fc_bsg_job *); + int (*bsg_request)(struct bsg_job *); + int (*bsg_timeout)(struct bsg_job *); /* allocation lengths for host-specific data */ u32 dd_fcrport_size; @@ -849,4 +809,18 @@ struct fc_vport *fc_vport_create(struct Scsi_Host *shost, int channel, int fc_vport_terminate(struct fc_vport *vport); int fc_block_scsi_eh(struct scsi_cmnd *cmnd); +static inline struct Scsi_Host *fc_bsg_to_shost(struct bsg_job *job) +{ + if (scsi_is_host_device(job->dev)) + return dev_to_shost(job->dev); + return rport_to_shost(dev_to_rport(job->dev)); +} + +static inline struct fc_rport *fc_bsg_to_rport(struct bsg_job *job) +{ + if (scsi_is_fc_rport(job->dev)) + return dev_to_rport(job->dev); + return NULL; +} + #endif /* SCSI_TRANSPORT_FC_H */ diff --git a/include/trace/events/f2fs.h b/include/trace/events/f2fs.h index 5da2c82..01b3c98 100644 --- a/include/trace/events/f2fs.h +++ b/include/trace/events/f2fs.h @@ -1111,6 +1111,27 @@ TRACE_EVENT(f2fs_issue_discard, (unsigned long long)__entry->blklen) ); +TRACE_EVENT(f2fs_issue_reset_zone, + + TP_PROTO(struct super_block *sb, block_t blkstart), + + TP_ARGS(sb, blkstart), + + TP_STRUCT__entry( + __field(dev_t, dev) + __field(block_t, blkstart) + ), + + TP_fast_assign( + __entry->dev = sb->s_dev; + __entry->blkstart = blkstart; + ), + + TP_printk("dev = (%d,%d), reset zone at block = 0x%llx", + show_dev(__entry), + (unsigned long long)__entry->blkstart) +); + TRACE_EVENT(f2fs_issue_flush, TP_PROTO(struct super_block *sb, unsigned int nobarrier, diff --git a/include/uapi/linux/fs.h b/include/uapi/linux/fs.h index c1d11df..36da93f 100644 --- a/include/uapi/linux/fs.h +++ b/include/uapi/linux/fs.h @@ -258,6 +258,20 @@ struct fsxattr { /* Policy provided via an ioctl on the topmost directory */ #define FS_KEY_DESCRIPTOR_SIZE 8 +#define FS_POLICY_FLAGS_PAD_4 0x00 +#define FS_POLICY_FLAGS_PAD_8 0x01 +#define FS_POLICY_FLAGS_PAD_16 0x02 +#define FS_POLICY_FLAGS_PAD_32 0x03 +#define FS_POLICY_FLAGS_PAD_MASK 0x03 +#define FS_POLICY_FLAGS_VALID 0x03 + +/* Encryption algorithms */ +#define FS_ENCRYPTION_MODE_INVALID 0 +#define FS_ENCRYPTION_MODE_AES_256_XTS 1 +#define FS_ENCRYPTION_MODE_AES_256_GCM 2 +#define FS_ENCRYPTION_MODE_AES_256_CBC 3 +#define FS_ENCRYPTION_MODE_AES_256_CTS 4 + struct fscrypt_policy { __u8 version; __u8 contents_encryption_mode; diff --git a/include/uapi/linux/hw_breakpoint.h b/include/uapi/linux/hw_breakpoint.h index b04000a..2b65efd 100644 --- a/include/uapi/linux/hw_breakpoint.h +++ b/include/uapi/linux/hw_breakpoint.h @@ -4,7 +4,11 @@ enum { HW_BREAKPOINT_LEN_1 = 1, HW_BREAKPOINT_LEN_2 = 2, + HW_BREAKPOINT_LEN_3 = 3, HW_BREAKPOINT_LEN_4 = 4, + HW_BREAKPOINT_LEN_5 = 5, + HW_BREAKPOINT_LEN_6 = 6, + HW_BREAKPOINT_LEN_7 = 7, HW_BREAKPOINT_LEN_8 = 8, }; diff --git a/include/uapi/linux/kvm.h b/include/uapi/linux/kvm.h index 4ee67cb..cac48ed 100644 --- a/include/uapi/linux/kvm.h +++ b/include/uapi/linux/kvm.h @@ -651,6 +651,9 @@ struct kvm_enable_cap { }; /* for KVM_PPC_GET_PVINFO */ + +#define KVM_PPC_PVINFO_FLAGS_EV_IDLE (1<<0) + struct kvm_ppc_pvinfo { /* out */ __u32 flags; @@ -682,8 +685,6 @@ struct kvm_ppc_smmu_info { struct kvm_ppc_one_seg_page_size sps[KVM_PPC_PAGE_SIZES_MAX_SZ]; }; -#define KVM_PPC_PVINFO_FLAGS_EV_IDLE (1<<0) - #define KVMIO 0xAE /* machine type bits, to be used as argument to KVM_CREATE_VM */ diff --git a/include/xen/arm/hypercall.h b/include/xen/arm/hypercall.h new file mode 100644 index 0000000..9d874db --- /dev/null +++ b/include/xen/arm/hypercall.h @@ -0,0 +1,87 @@ +/****************************************************************************** + * hypercall.h + * + * Linux-specific hypervisor handling. + * + * Stefano Stabellini <stefano.stabellini@eu.citrix.com>, Citrix, 2012 + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation; or, when distributed + * separately from the Linux kernel or incorporated into other + * software packages, subject to the following license: + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this source file (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, copy, modify, + * merge, publish, distribute, sublicense, and/or sell copies of the Software, + * and to permit persons to whom the Software is furnished to do so, subject to + * the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +#ifndef _ASM_ARM_XEN_HYPERCALL_H +#define _ASM_ARM_XEN_HYPERCALL_H + +#include <linux/bug.h> + +#include <xen/interface/xen.h> +#include <xen/interface/sched.h> +#include <xen/interface/platform.h> + +long privcmd_call(unsigned call, unsigned long a1, + unsigned long a2, unsigned long a3, + unsigned long a4, unsigned long a5); +int HYPERVISOR_xen_version(int cmd, void *arg); +int HYPERVISOR_console_io(int cmd, int count, char *str); +int HYPERVISOR_grant_table_op(unsigned int cmd, void *uop, unsigned int count); +int HYPERVISOR_sched_op(int cmd, void *arg); +int HYPERVISOR_event_channel_op(int cmd, void *arg); +unsigned long HYPERVISOR_hvm_op(int op, void *arg); +int HYPERVISOR_memory_op(unsigned int cmd, void *arg); +int HYPERVISOR_physdev_op(int cmd, void *arg); +int HYPERVISOR_vcpu_op(int cmd, int vcpuid, void *extra_args); +int HYPERVISOR_tmem_op(void *arg); +int HYPERVISOR_vm_assist(unsigned int cmd, unsigned int type); +int HYPERVISOR_platform_op_raw(void *arg); +static inline int HYPERVISOR_platform_op(struct xen_platform_op *op) +{ + op->interface_version = XENPF_INTERFACE_VERSION; + return HYPERVISOR_platform_op_raw(op); +} +int HYPERVISOR_multicall(struct multicall_entry *calls, uint32_t nr); + +static inline int +HYPERVISOR_suspend(unsigned long start_info_mfn) +{ + struct sched_shutdown r = { .reason = SHUTDOWN_suspend }; + + /* start_info_mfn is unused on ARM */ + return HYPERVISOR_sched_op(SCHEDOP_shutdown, &r); +} + +static inline void +MULTI_update_va_mapping(struct multicall_entry *mcl, unsigned long va, + unsigned int new_val, unsigned long flags) +{ + BUG(); +} + +static inline void +MULTI_mmu_update(struct multicall_entry *mcl, struct mmu_update *req, + int count, int *success_count, domid_t domid) +{ + BUG(); +} + +#endif /* _ASM_ARM_XEN_HYPERCALL_H */ diff --git a/include/xen/arm/hypervisor.h b/include/xen/arm/hypervisor.h new file mode 100644 index 0000000..9525151 --- /dev/null +++ b/include/xen/arm/hypervisor.h @@ -0,0 +1,39 @@ +#ifndef _ASM_ARM_XEN_HYPERVISOR_H +#define _ASM_ARM_XEN_HYPERVISOR_H + +#include <linux/init.h> + +extern struct shared_info *HYPERVISOR_shared_info; +extern struct start_info *xen_start_info; + +/* Lazy mode for batching updates / context switch */ +enum paravirt_lazy_mode { + PARAVIRT_LAZY_NONE, + PARAVIRT_LAZY_MMU, + PARAVIRT_LAZY_CPU, +}; + +static inline enum paravirt_lazy_mode paravirt_get_lazy_mode(void) +{ + return PARAVIRT_LAZY_NONE; +} + +extern struct dma_map_ops *xen_dma_ops; + +#ifdef CONFIG_XEN +void __init xen_early_init(void); +#else +static inline void xen_early_init(void) { return; } +#endif + +#ifdef CONFIG_HOTPLUG_CPU +static inline void xen_arch_register_cpu(int num) +{ +} + +static inline void xen_arch_unregister_cpu(int num) +{ +} +#endif + +#endif /* _ASM_ARM_XEN_HYPERVISOR_H */ diff --git a/include/xen/arm/interface.h b/include/xen/arm/interface.h new file mode 100644 index 0000000..75d5968 --- /dev/null +++ b/include/xen/arm/interface.h @@ -0,0 +1,85 @@ +/****************************************************************************** + * Guest OS interface to ARM Xen. + * + * Stefano Stabellini <stefano.stabellini@eu.citrix.com>, Citrix, 2012 + */ + +#ifndef _ASM_ARM_XEN_INTERFACE_H +#define _ASM_ARM_XEN_INTERFACE_H + +#include <linux/types.h> + +#define uint64_aligned_t uint64_t __attribute__((aligned(8))) + +#define __DEFINE_GUEST_HANDLE(name, type) \ + typedef struct { union { type *p; uint64_aligned_t q; }; } \ + __guest_handle_ ## name + +#define DEFINE_GUEST_HANDLE_STRUCT(name) \ + __DEFINE_GUEST_HANDLE(name, struct name) +#define DEFINE_GUEST_HANDLE(name) __DEFINE_GUEST_HANDLE(name, name) +#define GUEST_HANDLE(name) __guest_handle_ ## name + +#define set_xen_guest_handle(hnd, val) \ + do { \ + if (sizeof(hnd) == 8) \ + *(uint64_t *)&(hnd) = 0; \ + (hnd).p = val; \ + } while (0) + +#define __HYPERVISOR_platform_op_raw __HYPERVISOR_platform_op + +#ifndef __ASSEMBLY__ +/* Explicitly size integers that represent pfns in the interface with + * Xen so that we can have one ABI that works for 32 and 64 bit guests. + * Note that this means that the xen_pfn_t type may be capable of + * representing pfn's which the guest cannot represent in its own pfn + * type. However since pfn space is controlled by the guest this is + * fine since it simply wouldn't be able to create any sure pfns in + * the first place. + */ +typedef uint64_t xen_pfn_t; +#define PRI_xen_pfn "llx" +typedef uint64_t xen_ulong_t; +#define PRI_xen_ulong "llx" +typedef int64_t xen_long_t; +#define PRI_xen_long "llx" +/* Guest handles for primitive C types. */ +__DEFINE_GUEST_HANDLE(uchar, unsigned char); +__DEFINE_GUEST_HANDLE(uint, unsigned int); +DEFINE_GUEST_HANDLE(char); +DEFINE_GUEST_HANDLE(int); +DEFINE_GUEST_HANDLE(void); +DEFINE_GUEST_HANDLE(uint64_t); +DEFINE_GUEST_HANDLE(uint32_t); +DEFINE_GUEST_HANDLE(xen_pfn_t); +DEFINE_GUEST_HANDLE(xen_ulong_t); + +/* Maximum number of virtual CPUs in multi-processor guests. */ +#define MAX_VIRT_CPUS 1 + +struct arch_vcpu_info { }; +struct arch_shared_info { }; + +/* TODO: Move pvclock definitions some place arch independent */ +struct pvclock_vcpu_time_info { + u32 version; + u32 pad0; + u64 tsc_timestamp; + u64 system_time; + u32 tsc_to_system_mul; + s8 tsc_shift; + u8 flags; + u8 pad[2]; +} __attribute__((__packed__)); /* 32 bytes */ + +/* It is OK to have a 12 bytes struct with no padding because it is packed */ +struct pvclock_wall_clock { + u32 version; + u32 sec; + u32 nsec; + u32 sec_hi; +} __attribute__((__packed__)); +#endif + +#endif /* _ASM_ARM_XEN_INTERFACE_H */ diff --git a/include/xen/arm/page-coherent.h b/include/xen/arm/page-coherent.h new file mode 100644 index 0000000..95ce6ac --- /dev/null +++ b/include/xen/arm/page-coherent.h @@ -0,0 +1,98 @@ +#ifndef _ASM_ARM_XEN_PAGE_COHERENT_H +#define _ASM_ARM_XEN_PAGE_COHERENT_H + +#include <asm/page.h> +#include <linux/dma-mapping.h> + +void __xen_dma_map_page(struct device *hwdev, struct page *page, + dma_addr_t dev_addr, unsigned long offset, size_t size, + enum dma_data_direction dir, unsigned long attrs); +void __xen_dma_unmap_page(struct device *hwdev, dma_addr_t handle, + size_t size, enum dma_data_direction dir, + unsigned long attrs); +void __xen_dma_sync_single_for_cpu(struct device *hwdev, + dma_addr_t handle, size_t size, enum dma_data_direction dir); + +void __xen_dma_sync_single_for_device(struct device *hwdev, + dma_addr_t handle, size_t size, enum dma_data_direction dir); + +static inline void *xen_alloc_coherent_pages(struct device *hwdev, size_t size, + dma_addr_t *dma_handle, gfp_t flags, unsigned long attrs) +{ + return __generic_dma_ops(hwdev)->alloc(hwdev, size, dma_handle, flags, attrs); +} + +static inline void xen_free_coherent_pages(struct device *hwdev, size_t size, + void *cpu_addr, dma_addr_t dma_handle, unsigned long attrs) +{ + __generic_dma_ops(hwdev)->free(hwdev, size, cpu_addr, dma_handle, attrs); +} + +static inline void xen_dma_map_page(struct device *hwdev, struct page *page, + dma_addr_t dev_addr, unsigned long offset, size_t size, + enum dma_data_direction dir, unsigned long attrs) +{ + unsigned long page_pfn = page_to_xen_pfn(page); + unsigned long dev_pfn = XEN_PFN_DOWN(dev_addr); + unsigned long compound_pages = + (1<<compound_order(page)) * XEN_PFN_PER_PAGE; + bool local = (page_pfn <= dev_pfn) && + (dev_pfn - page_pfn < compound_pages); + + /* + * Dom0 is mapped 1:1, while the Linux page can span across + * multiple Xen pages, it's not possible for it to contain a + * mix of local and foreign Xen pages. So if the first xen_pfn + * == mfn the page is local otherwise it's a foreign page + * grant-mapped in dom0. If the page is local we can safely + * call the native dma_ops function, otherwise we call the xen + * specific function. + */ + if (local) + __generic_dma_ops(hwdev)->map_page(hwdev, page, offset, size, dir, attrs); + else + __xen_dma_map_page(hwdev, page, dev_addr, offset, size, dir, attrs); +} + +static inline void xen_dma_unmap_page(struct device *hwdev, dma_addr_t handle, + size_t size, enum dma_data_direction dir, unsigned long attrs) +{ + unsigned long pfn = PFN_DOWN(handle); + /* + * Dom0 is mapped 1:1, while the Linux page can be spanned accross + * multiple Xen page, it's not possible to have a mix of local and + * foreign Xen page. Dom0 is mapped 1:1, so calling pfn_valid on a + * foreign mfn will always return false. If the page is local we can + * safely call the native dma_ops function, otherwise we call the xen + * specific function. + */ + if (pfn_valid(pfn)) { + if (__generic_dma_ops(hwdev)->unmap_page) + __generic_dma_ops(hwdev)->unmap_page(hwdev, handle, size, dir, attrs); + } else + __xen_dma_unmap_page(hwdev, handle, size, dir, attrs); +} + +static inline void xen_dma_sync_single_for_cpu(struct device *hwdev, + dma_addr_t handle, size_t size, enum dma_data_direction dir) +{ + unsigned long pfn = PFN_DOWN(handle); + if (pfn_valid(pfn)) { + if (__generic_dma_ops(hwdev)->sync_single_for_cpu) + __generic_dma_ops(hwdev)->sync_single_for_cpu(hwdev, handle, size, dir); + } else + __xen_dma_sync_single_for_cpu(hwdev, handle, size, dir); +} + +static inline void xen_dma_sync_single_for_device(struct device *hwdev, + dma_addr_t handle, size_t size, enum dma_data_direction dir) +{ + unsigned long pfn = PFN_DOWN(handle); + if (pfn_valid(pfn)) { + if (__generic_dma_ops(hwdev)->sync_single_for_device) + __generic_dma_ops(hwdev)->sync_single_for_device(hwdev, handle, size, dir); + } else + __xen_dma_sync_single_for_device(hwdev, handle, size, dir); +} + +#endif /* _ASM_ARM_XEN_PAGE_COHERENT_H */ diff --git a/include/xen/arm/page.h b/include/xen/arm/page.h new file mode 100644 index 0000000..415dbc6 --- /dev/null +++ b/include/xen/arm/page.h @@ -0,0 +1,122 @@ +#ifndef _ASM_ARM_XEN_PAGE_H +#define _ASM_ARM_XEN_PAGE_H + +#include <asm/page.h> +#include <asm/pgtable.h> + +#include <linux/pfn.h> +#include <linux/types.h> +#include <linux/dma-mapping.h> + +#include <xen/xen.h> +#include <xen/interface/grant_table.h> + +#define phys_to_machine_mapping_valid(pfn) (1) + +/* Xen machine address */ +typedef struct xmaddr { + phys_addr_t maddr; +} xmaddr_t; + +/* Xen pseudo-physical address */ +typedef struct xpaddr { + phys_addr_t paddr; +} xpaddr_t; + +#define XMADDR(x) ((xmaddr_t) { .maddr = (x) }) +#define XPADDR(x) ((xpaddr_t) { .paddr = (x) }) + +#define INVALID_P2M_ENTRY (~0UL) + +/* + * The pseudo-physical frame (pfn) used in all the helpers is always based + * on Xen page granularity (i.e 4KB). + * + * A Linux page may be split across multiple non-contiguous Xen page so we + * have to keep track with frame based on 4KB page granularity. + * + * PV drivers should never make a direct usage of those helpers (particularly + * pfn_to_gfn and gfn_to_pfn). + */ + +unsigned long __pfn_to_mfn(unsigned long pfn); +extern struct rb_root phys_to_mach; + +/* Pseudo-physical <-> Guest conversion */ +static inline unsigned long pfn_to_gfn(unsigned long pfn) +{ + return pfn; +} + +static inline unsigned long gfn_to_pfn(unsigned long gfn) +{ + return gfn; +} + +/* Pseudo-physical <-> BUS conversion */ +static inline unsigned long pfn_to_bfn(unsigned long pfn) +{ + unsigned long mfn; + + if (phys_to_mach.rb_node != NULL) { + mfn = __pfn_to_mfn(pfn); + if (mfn != INVALID_P2M_ENTRY) + return mfn; + } + + return pfn; +} + +static inline unsigned long bfn_to_pfn(unsigned long bfn) +{ + return bfn; +} + +#define bfn_to_local_pfn(bfn) bfn_to_pfn(bfn) + +/* VIRT <-> GUEST conversion */ +#define virt_to_gfn(v) (pfn_to_gfn(virt_to_phys(v) >> XEN_PAGE_SHIFT)) +#define gfn_to_virt(m) (__va(gfn_to_pfn(m) << XEN_PAGE_SHIFT)) + +/* Only used in PV code. But ARM guests are always HVM. */ +static inline xmaddr_t arbitrary_virt_to_machine(void *vaddr) +{ + BUG(); +} + +/* TODO: this shouldn't be here but it is because the frontend drivers + * are using it (its rolled in headers) even though we won't hit the code path. + * So for right now just punt with this. + */ +static inline pte_t *lookup_address(unsigned long address, unsigned int *level) +{ + BUG(); + return NULL; +} + +extern int set_foreign_p2m_mapping(struct gnttab_map_grant_ref *map_ops, + struct gnttab_map_grant_ref *kmap_ops, + struct page **pages, unsigned int count); + +extern int clear_foreign_p2m_mapping(struct gnttab_unmap_grant_ref *unmap_ops, + struct gnttab_unmap_grant_ref *kunmap_ops, + struct page **pages, unsigned int count); + +bool __set_phys_to_machine(unsigned long pfn, unsigned long mfn); +bool __set_phys_to_machine_multi(unsigned long pfn, unsigned long mfn, + unsigned long nr_pages); + +static inline bool set_phys_to_machine(unsigned long pfn, unsigned long mfn) +{ + return __set_phys_to_machine(pfn, mfn); +} + +#define xen_remap(cookie, size) ioremap_cache((cookie), (size)) +#define xen_unmap(cookie) iounmap((cookie)) + +bool xen_arch_need_swiotlb(struct device *dev, + phys_addr_t phys, + dma_addr_t dev_addr); +unsigned long xen_get_swiotlb_free_pages(unsigned int order); + +#endif /* _ASM_ARM_XEN_PAGE_H */ diff --git a/include/xen/swiotlb-xen.h b/include/xen/swiotlb-xen.h index 7c35e27..a0083be 100644 --- a/include/xen/swiotlb-xen.h +++ b/include/xen/swiotlb-xen.h @@ -51,9 +51,6 @@ xen_swiotlb_sync_sg_for_device(struct device *hwdev, struct scatterlist *sg, int nelems, enum dma_data_direction dir); extern int -xen_swiotlb_dma_mapping_error(struct device *hwdev, dma_addr_t dma_addr); - -extern int xen_swiotlb_dma_supported(struct device *hwdev, u64 mask); extern int diff --git a/include/xen/xenbus.h b/include/xen/xenbus.h index 32b944b..271ba62 100644 --- a/include/xen/xenbus.h +++ b/include/xen/xenbus.h @@ -151,6 +151,10 @@ __scanf(4, 5) int xenbus_scanf(struct xenbus_transaction t, const char *dir, const char *node, const char *fmt, ...); +/* Read an (optional) unsigned value. */ +unsigned int xenbus_read_unsigned(const char *dir, const char *node, + unsigned int default_val); + /* Single printf and write: returns -errno or 0. */ __printf(4, 5) int xenbus_printf(struct xenbus_transaction t, diff --git a/init/main.c b/init/main.c index fa20116..23c275c 100644 --- a/init/main.c +++ b/init/main.c @@ -553,6 +553,14 @@ asmlinkage __visible void __init start_kernel(void) "Interrupts were enabled *very* early, fixing it\n")) local_irq_disable(); idr_init_cache(); + + /* + * Allow workqueue creation and work item queueing/cancelling + * early. Work item execution depends on kthreads and starts after + * workqueue_init(). + */ + workqueue_init_early(); + rcu_init(); /* trace_printk() and trace points may be used after this */ @@ -1009,6 +1017,8 @@ static noinline void __init kernel_init_freeable(void) smp_prepare_cpus(setup_max_cpus); + workqueue_init(); + do_pre_smp_initcalls(); lockup_detector_init(); diff --git a/kernel/power/qos.c b/kernel/power/qos.c index 168ff44..97b0df7 100644 --- a/kernel/power/qos.c +++ b/kernel/power/qos.c @@ -482,16 +482,7 @@ void pm_qos_update_request(struct pm_qos_request *req, return; } - /* - * This function may be called very early during boot, for example, - * from of_clk_init(), where irq needs to stay disabled. - * cancel_delayed_work_sync() assumes that irq is enabled on - * invocation and re-enables it on return. Avoid calling it until - * workqueue is initialized. - */ - if (keventd_up()) - cancel_delayed_work_sync(&req->work); - + cancel_delayed_work_sync(&req->work); __pm_qos_update_request(req, new_value); } EXPORT_SYMBOL_GPL(pm_qos_update_request); diff --git a/kernel/workqueue.c b/kernel/workqueue.c index 479d840..1d9fb65 100644 --- a/kernel/workqueue.c +++ b/kernel/workqueue.c @@ -290,6 +290,8 @@ module_param_named(disable_numa, wq_disable_numa, bool, 0444); static bool wq_power_efficient = IS_ENABLED(CONFIG_WQ_POWER_EFFICIENT_DEFAULT); module_param_named(power_efficient, wq_power_efficient, bool, 0444); +static bool wq_online; /* can kworkers be created yet? */ + static bool wq_numa_enabled; /* unbound NUMA affinity enabled */ /* buf for wq_update_unbound_numa_attrs(), protected by CPU hotplug exclusion */ @@ -2583,6 +2585,9 @@ void flush_workqueue(struct workqueue_struct *wq) }; int next_color; + if (WARN_ON(!wq_online)) + return; + lock_map_acquire(&wq->lockdep_map); lock_map_release(&wq->lockdep_map); @@ -2843,6 +2848,9 @@ bool flush_work(struct work_struct *work) { struct wq_barrier barr; + if (WARN_ON(!wq_online)) + return false; + lock_map_acquire(&work->lockdep_map); lock_map_release(&work->lockdep_map); @@ -2913,7 +2921,13 @@ static bool __cancel_work_timer(struct work_struct *work, bool is_dwork) mark_work_canceling(work); local_irq_restore(flags); - flush_work(work); + /* + * This allows canceling during early boot. We know that @work + * isn't executing. + */ + if (wq_online) + flush_work(work); + clear_work_data(work); /* @@ -3364,7 +3378,7 @@ static struct worker_pool *get_unbound_pool(const struct workqueue_attrs *attrs) goto fail; /* create and start the initial worker */ - if (!create_worker(pool)) + if (wq_online && !create_worker(pool)) goto fail; /* install */ @@ -3429,6 +3443,7 @@ static void pwq_adjust_max_active(struct pool_workqueue *pwq) { struct workqueue_struct *wq = pwq->wq; bool freezable = wq->flags & WQ_FREEZABLE; + unsigned long flags; /* for @wq->saved_max_active */ lockdep_assert_held(&wq->mutex); @@ -3437,7 +3452,8 @@ static void pwq_adjust_max_active(struct pool_workqueue *pwq) if (!freezable && pwq->max_active == wq->saved_max_active) return; - spin_lock_irq(&pwq->pool->lock); + /* this function can be called during early boot w/ irq disabled */ + spin_lock_irqsave(&pwq->pool->lock, flags); /* * During [un]freezing, the caller is responsible for ensuring that @@ -3460,7 +3476,7 @@ static void pwq_adjust_max_active(struct pool_workqueue *pwq) pwq->max_active = 0; } - spin_unlock_irq(&pwq->pool->lock); + spin_unlock_irqrestore(&pwq->pool->lock, flags); } /* initialize newly alloced @pwq which is associated with @wq and @pool */ @@ -4033,6 +4049,7 @@ void destroy_workqueue(struct workqueue_struct *wq) for (i = 0; i < WORK_NR_COLORS; i++) { if (WARN_ON(pwq->nr_in_flight[i])) { mutex_unlock(&wq->mutex); + show_workqueue_state(); return; } } @@ -4041,6 +4058,7 @@ void destroy_workqueue(struct workqueue_struct *wq) WARN_ON(pwq->nr_active) || WARN_ON(!list_empty(&pwq->delayed_works))) { mutex_unlock(&wq->mutex); + show_workqueue_state(); return; } } @@ -5467,7 +5485,17 @@ static void __init wq_numa_init(void) wq_numa_enabled = true; } -static int __init init_workqueues(void) +/** + * workqueue_init_early - early init for workqueue subsystem + * + * This is the first half of two-staged workqueue subsystem initialization + * and invoked as soon as the bare basics - memory allocation, cpumasks and + * idr are up. It sets up all the data structures and system workqueues + * and allows early boot code to create workqueues and queue/cancel work + * items. Actual work item execution starts only after kthreads can be + * created and scheduled right before early initcalls. + */ +int __init workqueue_init_early(void) { int std_nice[NR_STD_WORKER_POOLS] = { 0, HIGHPRI_NICE_LEVEL }; int i, cpu; @@ -5479,8 +5507,6 @@ static int __init init_workqueues(void) pwq_cache = KMEM_CACHE(pool_workqueue, SLAB_PANIC); - wq_numa_init(); - /* initialize CPU pools */ for_each_possible_cpu(cpu) { struct worker_pool *pool; @@ -5500,16 +5526,6 @@ static int __init init_workqueues(void) } } - /* create the initial worker */ - for_each_online_cpu(cpu) { - struct worker_pool *pool; - - for_each_cpu_worker_pool(pool, cpu) { - pool->flags &= ~POOL_DISASSOCIATED; - BUG_ON(!create_worker(pool)); - } - } - /* create default unbound and ordered wq attrs */ for (i = 0; i < NR_STD_WORKER_POOLS; i++) { struct workqueue_attrs *attrs; @@ -5546,8 +5562,59 @@ static int __init init_workqueues(void) !system_power_efficient_wq || !system_freezable_power_efficient_wq); + return 0; +} + +/** + * workqueue_init - bring workqueue subsystem fully online + * + * This is the latter half of two-staged workqueue subsystem initialization + * and invoked as soon as kthreads can be created and scheduled. + * Workqueues have been created and work items queued on them, but there + * are no kworkers executing the work items yet. Populate the worker pools + * with the initial workers and enable future kworker creations. + */ +int __init workqueue_init(void) +{ + struct workqueue_struct *wq; + struct worker_pool *pool; + int cpu, bkt; + + /* + * It'd be simpler to initialize NUMA in workqueue_init_early() but + * CPU to node mapping may not be available that early on some + * archs such as power and arm64. As per-cpu pools created + * previously could be missing node hint and unbound pools NUMA + * affinity, fix them up. + */ + wq_numa_init(); + + mutex_lock(&wq_pool_mutex); + + for_each_possible_cpu(cpu) { + for_each_cpu_worker_pool(pool, cpu) { + pool->node = cpu_to_node(cpu); + } + } + + list_for_each_entry(wq, &workqueues, list) + wq_update_unbound_numa(wq, smp_processor_id(), true); + + mutex_unlock(&wq_pool_mutex); + + /* create the initial workers */ + for_each_online_cpu(cpu) { + for_each_cpu_worker_pool(pool, cpu) { + pool->flags &= ~POOL_DISASSOCIATED; + BUG_ON(!create_worker(pool)); + } + } + + hash_for_each(unbound_pool_hash, bkt, pool, hash_node) + BUG_ON(!create_worker(pool)); + + wq_online = true; wq_watchdog_init(); return 0; } -early_initcall(init_workqueues); diff --git a/lib/debugobjects.c b/lib/debugobjects.c index 056052dc..04c1ef7 100644 --- a/lib/debugobjects.c +++ b/lib/debugobjects.c @@ -199,7 +199,7 @@ static void free_object(struct debug_obj *obj) * initialized: */ if (obj_pool_free > ODEBUG_POOL_SIZE && obj_cache) - sched = keventd_up(); + sched = 1; hlist_add_head(&obj->node, &obj_pool); obj_pool_free++; obj_pool_used--; diff --git a/lib/swiotlb.c b/lib/swiotlb.c index 22e13a0..cb1b54e 100644 --- a/lib/swiotlb.c +++ b/lib/swiotlb.c @@ -425,7 +425,8 @@ static void swiotlb_bounce(phys_addr_t orig_addr, phys_addr_t tlb_addr, phys_addr_t swiotlb_tbl_map_single(struct device *hwdev, dma_addr_t tbl_dma_addr, phys_addr_t orig_addr, size_t size, - enum dma_data_direction dir) + enum dma_data_direction dir, + unsigned long attrs) { unsigned long flags; phys_addr_t tlb_addr; @@ -526,7 +527,8 @@ found: */ for (i = 0; i < nslots; i++) io_tlb_orig_addr[index+i] = orig_addr + (i << IO_TLB_SHIFT); - if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL) + if (!(attrs & DMA_ATTR_SKIP_CPU_SYNC) && + (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL)) swiotlb_bounce(orig_addr, tlb_addr, size, DMA_TO_DEVICE); return tlb_addr; @@ -539,18 +541,20 @@ EXPORT_SYMBOL_GPL(swiotlb_tbl_map_single); static phys_addr_t map_single(struct device *hwdev, phys_addr_t phys, size_t size, - enum dma_data_direction dir) + enum dma_data_direction dir, unsigned long attrs) { dma_addr_t start_dma_addr = phys_to_dma(hwdev, io_tlb_start); - return swiotlb_tbl_map_single(hwdev, start_dma_addr, phys, size, dir); + return swiotlb_tbl_map_single(hwdev, start_dma_addr, phys, size, + dir, attrs); } /* * dma_addr is the kernel virtual address of the bounce buffer to unmap. */ void swiotlb_tbl_unmap_single(struct device *hwdev, phys_addr_t tlb_addr, - size_t size, enum dma_data_direction dir) + size_t size, enum dma_data_direction dir, + unsigned long attrs) { unsigned long flags; int i, count, nslots = ALIGN(size, 1 << IO_TLB_SHIFT) >> IO_TLB_SHIFT; @@ -561,6 +565,7 @@ void swiotlb_tbl_unmap_single(struct device *hwdev, phys_addr_t tlb_addr, * First, sync the memory before unmapping the entry */ if (orig_addr != INVALID_PHYS_ADDR && + !(attrs & DMA_ATTR_SKIP_CPU_SYNC) && ((dir == DMA_FROM_DEVICE) || (dir == DMA_BIDIRECTIONAL))) swiotlb_bounce(orig_addr, tlb_addr, size, DMA_FROM_DEVICE); @@ -654,7 +659,8 @@ swiotlb_alloc_coherent(struct device *hwdev, size_t size, * GFP_DMA memory; fall back on map_single(), which * will grab memory from the lowest available address range. */ - phys_addr_t paddr = map_single(hwdev, 0, size, DMA_FROM_DEVICE); + phys_addr_t paddr = map_single(hwdev, 0, size, + DMA_FROM_DEVICE, 0); if (paddr == SWIOTLB_MAP_ERROR) goto err_warn; @@ -667,9 +673,13 @@ swiotlb_alloc_coherent(struct device *hwdev, size_t size, (unsigned long long)dma_mask, (unsigned long long)dev_addr); - /* DMA_TO_DEVICE to avoid memcpy in unmap_single */ + /* + * DMA_TO_DEVICE to avoid memcpy in unmap_single. + * The DMA_ATTR_SKIP_CPU_SYNC is optional. + */ swiotlb_tbl_unmap_single(hwdev, paddr, - size, DMA_TO_DEVICE); + size, DMA_TO_DEVICE, + DMA_ATTR_SKIP_CPU_SYNC); goto err_warn; } } @@ -698,8 +708,12 @@ swiotlb_free_coherent(struct device *hwdev, size_t size, void *vaddr, if (!is_swiotlb_buffer(paddr)) free_pages((unsigned long)vaddr, get_order(size)); else - /* DMA_TO_DEVICE to avoid memcpy in swiotlb_tbl_unmap_single */ - swiotlb_tbl_unmap_single(hwdev, paddr, size, DMA_TO_DEVICE); + /* + * DMA_TO_DEVICE to avoid memcpy in swiotlb_tbl_unmap_single. + * DMA_ATTR_SKIP_CPU_SYNC is optional. + */ + swiotlb_tbl_unmap_single(hwdev, paddr, size, DMA_TO_DEVICE, + DMA_ATTR_SKIP_CPU_SYNC); } EXPORT_SYMBOL(swiotlb_free_coherent); @@ -714,8 +728,8 @@ swiotlb_full(struct device *dev, size_t size, enum dma_data_direction dir, * When the mapping is small enough return a static buffer to limit * the damage, or panic when the transfer is too big. */ - printk(KERN_ERR "DMA: Out of SW-IOMMU space for %zu bytes at " - "device %s\n", size, dev ? dev_name(dev) : "?"); + dev_err_ratelimited(dev, "DMA: Out of SW-IOMMU space for %zu bytes\n", + size); if (size <= io_tlb_overflow || !do_panic) return; @@ -755,7 +769,7 @@ dma_addr_t swiotlb_map_page(struct device *dev, struct page *page, trace_swiotlb_bounced(dev, dev_addr, size, swiotlb_force); /* Oh well, have to allocate and map a bounce buffer. */ - map = map_single(dev, phys, size, dir); + map = map_single(dev, phys, size, dir, attrs); if (map == SWIOTLB_MAP_ERROR) { swiotlb_full(dev, size, dir, 1); return phys_to_dma(dev, io_tlb_overflow_buffer); @@ -764,12 +778,13 @@ dma_addr_t swiotlb_map_page(struct device *dev, struct page *page, dev_addr = phys_to_dma(dev, map); /* Ensure that the address returned is DMA'ble */ - if (!dma_capable(dev, dev_addr, size)) { - swiotlb_tbl_unmap_single(dev, map, size, dir); - return phys_to_dma(dev, io_tlb_overflow_buffer); - } + if (dma_capable(dev, dev_addr, size)) + return dev_addr; - return dev_addr; + attrs |= DMA_ATTR_SKIP_CPU_SYNC; + swiotlb_tbl_unmap_single(dev, map, size, dir, attrs); + + return phys_to_dma(dev, io_tlb_overflow_buffer); } EXPORT_SYMBOL_GPL(swiotlb_map_page); @@ -782,14 +797,15 @@ EXPORT_SYMBOL_GPL(swiotlb_map_page); * whatever the device wrote there. */ static void unmap_single(struct device *hwdev, dma_addr_t dev_addr, - size_t size, enum dma_data_direction dir) + size_t size, enum dma_data_direction dir, + unsigned long attrs) { phys_addr_t paddr = dma_to_phys(hwdev, dev_addr); BUG_ON(dir == DMA_NONE); if (is_swiotlb_buffer(paddr)) { - swiotlb_tbl_unmap_single(hwdev, paddr, size, dir); + swiotlb_tbl_unmap_single(hwdev, paddr, size, dir, attrs); return; } @@ -809,7 +825,7 @@ void swiotlb_unmap_page(struct device *hwdev, dma_addr_t dev_addr, size_t size, enum dma_data_direction dir, unsigned long attrs) { - unmap_single(hwdev, dev_addr, size, dir); + unmap_single(hwdev, dev_addr, size, dir, attrs); } EXPORT_SYMBOL_GPL(swiotlb_unmap_page); @@ -891,11 +907,12 @@ swiotlb_map_sg_attrs(struct device *hwdev, struct scatterlist *sgl, int nelems, if (swiotlb_force || !dma_capable(hwdev, dev_addr, sg->length)) { phys_addr_t map = map_single(hwdev, sg_phys(sg), - sg->length, dir); + sg->length, dir, attrs); if (map == SWIOTLB_MAP_ERROR) { /* Don't panic here, we expect map_sg users to do proper error handling. */ swiotlb_full(hwdev, sg->length, dir, 0); + attrs |= DMA_ATTR_SKIP_CPU_SYNC; swiotlb_unmap_sg_attrs(hwdev, sgl, i, dir, attrs); sg_dma_len(sgl) = 0; @@ -910,14 +927,6 @@ swiotlb_map_sg_attrs(struct device *hwdev, struct scatterlist *sgl, int nelems, } EXPORT_SYMBOL(swiotlb_map_sg_attrs); -int -swiotlb_map_sg(struct device *hwdev, struct scatterlist *sgl, int nelems, - enum dma_data_direction dir) -{ - return swiotlb_map_sg_attrs(hwdev, sgl, nelems, dir, 0); -} -EXPORT_SYMBOL(swiotlb_map_sg); - /* * Unmap a set of streaming mode DMA translations. Again, cpu read rules * concerning calls here are the same as for swiotlb_unmap_page() above. @@ -933,19 +942,11 @@ swiotlb_unmap_sg_attrs(struct device *hwdev, struct scatterlist *sgl, BUG_ON(dir == DMA_NONE); for_each_sg(sgl, sg, nelems, i) - unmap_single(hwdev, sg->dma_address, sg_dma_len(sg), dir); - + unmap_single(hwdev, sg->dma_address, sg_dma_len(sg), dir, + attrs); } EXPORT_SYMBOL(swiotlb_unmap_sg_attrs); -void -swiotlb_unmap_sg(struct device *hwdev, struct scatterlist *sgl, int nelems, - enum dma_data_direction dir) -{ - return swiotlb_unmap_sg_attrs(hwdev, sgl, nelems, dir, 0); -} -EXPORT_SYMBOL(swiotlb_unmap_sg); - /* * Make physical memory consistent for a set of streaming mode DMA translations * after a transfer. diff --git a/mm/filemap.c b/mm/filemap.c index 5b4dd03..6956838 100644 --- a/mm/filemap.c +++ b/mm/filemap.c @@ -135,10 +135,9 @@ static int page_cache_tree_insert(struct address_space *mapping, } else { /* DAX can replace empty locked entry with a hole */ WARN_ON_ONCE(p != - (void *)(RADIX_TREE_EXCEPTIONAL_ENTRY | - RADIX_DAX_ENTRY_LOCK)); + dax_radix_locked_entry(0, RADIX_DAX_EMPTY)); /* Wakeup waiters for exceptional entry lock */ - dax_wake_mapping_entry_waiter(mapping, page->index, + dax_wake_mapping_entry_waiter(mapping, page->index, p, false); } } diff --git a/mm/percpu.c b/mm/percpu.c index f696385..0686f56 100644 --- a/mm/percpu.c +++ b/mm/percpu.c @@ -886,7 +886,8 @@ static void __percpu *pcpu_alloc(size_t size, size_t align, bool reserved, size = ALIGN(size, 2); - if (unlikely(!size || size > PCPU_MIN_UNIT_SIZE || align > PAGE_SIZE)) { + if (unlikely(!size || size > PCPU_MIN_UNIT_SIZE || align > PAGE_SIZE || + !is_power_of_2(align))) { WARN(true, "illegal size (%zu) or align (%zu) for percpu allocation\n", size, align); return NULL; @@ -552,12 +552,7 @@ static void start_cpu_timer(int cpu) { struct delayed_work *reap_work = &per_cpu(slab_reap_work, cpu); - /* - * When this gets called from do_initcalls via cpucache_init(), - * init_workqueues() has already run, so keventd will be setup - * at that time. - */ - if (keventd_up() && reap_work->work.func == NULL) { + if (reap_work->work.func == NULL) { init_reap_node(cpu); INIT_DEFERRABLE_WORK(reap_work, cache_reap); schedule_delayed_work_on(cpu, reap_work, diff --git a/tools/include/uapi/linux/hw_breakpoint.h b/tools/include/uapi/linux/hw_breakpoint.h index b04000a..2b65efd 100644 --- a/tools/include/uapi/linux/hw_breakpoint.h +++ b/tools/include/uapi/linux/hw_breakpoint.h @@ -4,7 +4,11 @@ enum { HW_BREAKPOINT_LEN_1 = 1, HW_BREAKPOINT_LEN_2 = 2, + HW_BREAKPOINT_LEN_3 = 3, HW_BREAKPOINT_LEN_4 = 4, + HW_BREAKPOINT_LEN_5 = 5, + HW_BREAKPOINT_LEN_6 = 6, + HW_BREAKPOINT_LEN_7 = 7, HW_BREAKPOINT_LEN_8 = 8, }; diff --git a/tools/spi/spidev_test.c b/tools/spi/spidev_test.c index f046b77..816f119 100644 --- a/tools/spi/spidev_test.c +++ b/tools/spi/spidev_test.c @@ -315,7 +315,7 @@ static void transfer_file(int fd, char *filename) pabort("can't stat input file"); tx_fd = open(filename, O_RDONLY); - if (fd < 0) + if (tx_fd < 0) pabort("can't open input file"); tx = malloc(sb.st_size); diff --git a/tools/testing/selftests/breakpoints/Makefile b/tools/testing/selftests/breakpoints/Makefile index 74e533f..61b79e8 100644 --- a/tools/testing/selftests/breakpoints/Makefile +++ b/tools/testing/selftests/breakpoints/Makefile @@ -5,6 +5,9 @@ ARCH ?= $(shell echo $(uname_M) | sed -e s/i.86/x86/ -e s/x86_64/x86/) ifeq ($(ARCH),x86) TEST_PROGS := breakpoint_test endif +ifeq ($(ARCH),aarch64) +TEST_PROGS := breakpoint_test_arm64 +endif TEST_PROGS += step_after_suspend_test @@ -13,4 +16,4 @@ all: $(TEST_PROGS) include ../lib.mk clean: - rm -fr breakpoint_test step_after_suspend_test + rm -fr breakpoint_test breakpoint_test_arm64 step_after_suspend_test diff --git a/tools/testing/selftests/breakpoints/breakpoint_test_arm64.c b/tools/testing/selftests/breakpoints/breakpoint_test_arm64.c new file mode 100644 index 0000000..3897e99 --- /dev/null +++ b/tools/testing/selftests/breakpoints/breakpoint_test_arm64.c @@ -0,0 +1,236 @@ +/* + * Copyright (C) 2016 Google, Inc. + * + * This software is licensed under the terms of the GNU General Public + * License version 2, as published by the Free Software Foundation, and + * may be copied, distributed, and modified under those terms. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * Original Code by Pavel Labath <labath@google.com> + * + * Code modified by Pratyush Anand <panand@redhat.com> + * for testing different byte select for each access size. + * + */ + +#define _GNU_SOURCE + +#include <sys/types.h> +#include <sys/wait.h> +#include <sys/ptrace.h> +#include <sys/param.h> +#include <sys/uio.h> +#include <stdint.h> +#include <stdbool.h> +#include <stddef.h> +#include <string.h> +#include <stdio.h> +#include <unistd.h> +#include <elf.h> +#include <errno.h> +#include <signal.h> + +#include "../kselftest.h" + +static volatile uint8_t var[96] __attribute__((__aligned__(32))); + +static void child(int size, int wr) +{ + volatile uint8_t *addr = &var[32 + wr]; + + if (ptrace(PTRACE_TRACEME, 0, NULL, NULL) != 0) { + perror("ptrace(PTRACE_TRACEME) failed"); + _exit(1); + } + + if (raise(SIGSTOP) != 0) { + perror("raise(SIGSTOP) failed"); + _exit(1); + } + + if ((uintptr_t) addr % size) { + perror("Wrong address write for the given size\n"); + _exit(1); + } + switch (size) { + case 1: + *addr = 47; + break; + case 2: + *(uint16_t *)addr = 47; + break; + case 4: + *(uint32_t *)addr = 47; + break; + case 8: + *(uint64_t *)addr = 47; + break; + case 16: + __asm__ volatile ("stp x29, x30, %0" : "=m" (addr[0])); + break; + case 32: + __asm__ volatile ("stp q29, q30, %0" : "=m" (addr[0])); + break; + } + + _exit(0); +} + +static bool set_watchpoint(pid_t pid, int size, int wp) +{ + const volatile uint8_t *addr = &var[32 + wp]; + const int offset = (uintptr_t)addr % 8; + const unsigned int byte_mask = ((1 << size) - 1) << offset; + const unsigned int type = 2; /* Write */ + const unsigned int enable = 1; + const unsigned int control = byte_mask << 5 | type << 3 | enable; + struct user_hwdebug_state dreg_state; + struct iovec iov; + + memset(&dreg_state, 0, sizeof(dreg_state)); + dreg_state.dbg_regs[0].addr = (uintptr_t)(addr - offset); + dreg_state.dbg_regs[0].ctrl = control; + iov.iov_base = &dreg_state; + iov.iov_len = offsetof(struct user_hwdebug_state, dbg_regs) + + sizeof(dreg_state.dbg_regs[0]); + if (ptrace(PTRACE_SETREGSET, pid, NT_ARM_HW_WATCH, &iov) == 0) + return true; + + if (errno == EIO) { + printf("ptrace(PTRACE_SETREGSET, NT_ARM_HW_WATCH) " + "not supported on this hardware\n"); + ksft_exit_skip(); + } + perror("ptrace(PTRACE_SETREGSET, NT_ARM_HW_WATCH) failed"); + return false; +} + +static bool run_test(int wr_size, int wp_size, int wr, int wp) +{ + int status; + siginfo_t siginfo; + pid_t pid = fork(); + pid_t wpid; + + if (pid < 0) { + perror("fork() failed"); + return false; + } + if (pid == 0) + child(wr_size, wr); + + wpid = waitpid(pid, &status, __WALL); + if (wpid != pid) { + perror("waitpid() failed"); + return false; + } + if (!WIFSTOPPED(status)) { + printf("child did not stop\n"); + return false; + } + if (WSTOPSIG(status) != SIGSTOP) { + printf("child did not stop with SIGSTOP\n"); + return false; + } + + if (!set_watchpoint(pid, wp_size, wp)) + return false; + + if (ptrace(PTRACE_CONT, pid, NULL, NULL) < 0) { + perror("ptrace(PTRACE_SINGLESTEP) failed"); + return false; + } + + alarm(3); + wpid = waitpid(pid, &status, __WALL); + if (wpid != pid) { + perror("waitpid() failed"); + return false; + } + alarm(0); + if (WIFEXITED(status)) { + printf("child did not single-step\t"); + return false; + } + if (!WIFSTOPPED(status)) { + printf("child did not stop\n"); + return false; + } + if (WSTOPSIG(status) != SIGTRAP) { + printf("child did not stop with SIGTRAP\n"); + return false; + } + if (ptrace(PTRACE_GETSIGINFO, pid, NULL, &siginfo) != 0) { + perror("ptrace(PTRACE_GETSIGINFO)"); + return false; + } + if (siginfo.si_code != TRAP_HWBKPT) { + printf("Unexpected si_code %d\n", siginfo.si_code); + return false; + } + + kill(pid, SIGKILL); + wpid = waitpid(pid, &status, 0); + if (wpid != pid) { + perror("waitpid() failed"); + return false; + } + return true; +} + +static void sigalrm(int sig) +{ +} + +int main(int argc, char **argv) +{ + int opt; + bool succeeded = true; + struct sigaction act; + int wr, wp, size; + bool result; + + act.sa_handler = sigalrm; + sigemptyset(&act.sa_mask); + act.sa_flags = 0; + sigaction(SIGALRM, &act, NULL); + for (size = 1; size <= 32; size = size*2) { + for (wr = 0; wr <= 32; wr = wr + size) { + for (wp = wr - size; wp <= wr + size; wp = wp + size) { + printf("Test size = %d write offset = %d watchpoint offset = %d\t", size, wr, wp); + result = run_test(size, MIN(size, 8), wr, wp); + if ((result && wr == wp) || (!result && wr != wp)) { + printf("[OK]\n"); + ksft_inc_pass_cnt(); + } else { + printf("[FAILED]\n"); + ksft_inc_fail_cnt(); + succeeded = false; + } + } + } + } + + for (size = 1; size <= 32; size = size*2) { + printf("Test size = %d write offset = %d watchpoint offset = -8\t", size, -size); + + if (run_test(size, 8, -size, -8)) { + printf("[OK]\n"); + ksft_inc_pass_cnt(); + } else { + printf("[FAILED]\n"); + ksft_inc_fail_cnt(); + succeeded = false; + } + } + + ksft_print_cnts(); + if (succeeded) + ksft_exit_pass(); + else + ksft_exit_fail(); +} diff --git a/virt/kvm/arm/arch_timer.c b/virt/kvm/arm/arch_timer.c index 27a1f63..ae95fc0 100644 --- a/virt/kvm/arm/arch_timer.c +++ b/virt/kvm/arm/arch_timer.c @@ -425,6 +425,11 @@ int kvm_timer_hyp_init(void) info = arch_timer_get_kvm_info(); timecounter = &info->timecounter; + if (!timecounter->cc) { + kvm_err("kvm_arch_timer: uninitialized timecounter\n"); + return -ENODEV; + } + if (info->virtual_irq <= 0) { kvm_err("kvm_arch_timer: invalid virtual timer IRQ: %d\n", info->virtual_irq); @@ -498,17 +503,7 @@ int kvm_timer_enable(struct kvm_vcpu *vcpu) if (ret) return ret; - - /* - * There is a potential race here between VCPUs starting for the first - * time, which may be enabling the timer multiple times. That doesn't - * hurt though, because we're just setting a variable to the same - * variable that it already was. The important thing is that all - * VCPUs have the enabled variable set, before entering the guest, if - * the arch timers are enabled. - */ - if (timecounter) - timer->enabled = 1; + timer->enabled = 1; return 0; } diff --git a/virt/kvm/arm/vgic/vgic-its.c b/virt/kvm/arm/vgic/vgic-its.c index 4660a7d..8c2b3cd 100644 --- a/virt/kvm/arm/vgic/vgic-its.c +++ b/virt/kvm/arm/vgic/vgic-its.c @@ -632,21 +632,22 @@ static bool vgic_its_check_id(struct vgic_its *its, u64 baser, int id) int index; u64 indirect_ptr; gfn_t gfn; + int esz = GITS_BASER_ENTRY_SIZE(baser); if (!(baser & GITS_BASER_INDIRECT)) { phys_addr_t addr; - if (id >= (l1_tbl_size / GITS_BASER_ENTRY_SIZE(baser))) + if (id >= (l1_tbl_size / esz)) return false; - addr = BASER_ADDRESS(baser) + id * GITS_BASER_ENTRY_SIZE(baser); + addr = BASER_ADDRESS(baser) + id * esz; gfn = addr >> PAGE_SHIFT; return kvm_is_visible_gfn(its->dev->kvm, gfn); } /* calculate and check the index into the 1st level */ - index = id / (SZ_64K / GITS_BASER_ENTRY_SIZE(baser)); + index = id / (SZ_64K / esz); if (index >= (l1_tbl_size / sizeof(u64))) return false; @@ -670,8 +671,8 @@ static bool vgic_its_check_id(struct vgic_its *its, u64 baser, int id) indirect_ptr &= GENMASK_ULL(51, 16); /* Find the address of the actual entry */ - index = id % (SZ_64K / GITS_BASER_ENTRY_SIZE(baser)); - indirect_ptr += index * GITS_BASER_ENTRY_SIZE(baser); + index = id % (SZ_64K / esz); + indirect_ptr += index * esz; gfn = indirect_ptr >> PAGE_SHIFT; return kvm_is_visible_gfn(its->dev->kvm, gfn); diff --git a/virt/kvm/arm/vgic/vgic-kvm-device.c b/virt/kvm/arm/vgic/vgic-kvm-device.c index ce1f4ed..fbe87a6 100644 --- a/virt/kvm/arm/vgic/vgic-kvm-device.c +++ b/virt/kvm/arm/vgic/vgic-kvm-device.c @@ -221,11 +221,9 @@ int kvm_register_vgic_device(unsigned long type) ret = kvm_register_device_ops(&kvm_arm_vgic_v3_ops, KVM_DEV_TYPE_ARM_VGIC_V3); -#ifdef CONFIG_KVM_ARM_VGIC_V3_ITS if (ret) break; ret = kvm_vgic_register_its_device(); -#endif break; } diff --git a/virt/kvm/arm/vgic/vgic-mmio-v2.c b/virt/kvm/arm/vgic/vgic-mmio-v2.c index b44b359..78e34bc 100644 --- a/virt/kvm/arm/vgic/vgic-mmio-v2.c +++ b/virt/kvm/arm/vgic/vgic-mmio-v2.c @@ -129,6 +129,7 @@ static void vgic_mmio_write_target(struct kvm_vcpu *vcpu, unsigned long val) { u32 intid = VGIC_ADDR_TO_INTID(addr, 8); + u8 cpu_mask = GENMASK(atomic_read(&vcpu->kvm->online_vcpus) - 1, 0); int i; /* GICD_ITARGETSR[0-7] are read-only */ @@ -141,7 +142,7 @@ static void vgic_mmio_write_target(struct kvm_vcpu *vcpu, spin_lock(&irq->irq_lock); - irq->targets = (val >> (i * 8)) & 0xff; + irq->targets = (val >> (i * 8)) & cpu_mask; target = irq->targets ? __ffs(irq->targets) : 0; irq->target_vcpu = kvm_get_vcpu(vcpu->kvm, target); diff --git a/virt/kvm/arm/vgic/vgic-mmio-v3.c b/virt/kvm/arm/vgic/vgic-mmio-v3.c index 0d3c76a..50f42f0 100644 --- a/virt/kvm/arm/vgic/vgic-mmio-v3.c +++ b/virt/kvm/arm/vgic/vgic-mmio-v3.c @@ -42,7 +42,6 @@ u64 update_64bit_reg(u64 reg, unsigned int offset, unsigned int len, return reg | ((u64)val << lower); } -#ifdef CONFIG_KVM_ARM_VGIC_V3_ITS bool vgic_has_its(struct kvm *kvm) { struct vgic_dist *dist = &kvm->arch.vgic; @@ -52,7 +51,6 @@ bool vgic_has_its(struct kvm *kvm) return dist->has_its; } -#endif static unsigned long vgic_mmio_read_v3_misc(struct kvm_vcpu *vcpu, gpa_t addr, unsigned int len) diff --git a/virt/kvm/arm/vgic/vgic.h b/virt/kvm/arm/vgic/vgic.h index 9d9e014..859f65c 100644 --- a/virt/kvm/arm/vgic/vgic.h +++ b/virt/kvm/arm/vgic/vgic.h @@ -84,37 +84,11 @@ int vgic_v3_probe(const struct gic_kvm_info *info); int vgic_v3_map_resources(struct kvm *kvm); int vgic_register_redist_iodevs(struct kvm *kvm, gpa_t dist_base_address); -#ifdef CONFIG_KVM_ARM_VGIC_V3_ITS int vgic_register_its_iodevs(struct kvm *kvm); bool vgic_has_its(struct kvm *kvm); int kvm_vgic_register_its_device(void); void vgic_enable_lpis(struct kvm_vcpu *vcpu); int vgic_its_inject_msi(struct kvm *kvm, struct kvm_msi *msi); -#else -static inline int vgic_register_its_iodevs(struct kvm *kvm) -{ - return -ENODEV; -} - -static inline bool vgic_has_its(struct kvm *kvm) -{ - return false; -} - -static inline int kvm_vgic_register_its_device(void) -{ - return -ENODEV; -} - -static inline void vgic_enable_lpis(struct kvm_vcpu *vcpu) -{ -} - -static inline int vgic_its_inject_msi(struct kvm *kvm, struct kvm_msi *msi) -{ - return -ENODEV; -} -#endif int kvm_register_vgic_device(unsigned long type); int vgic_lazy_init(struct kvm *kvm); diff --git a/virt/kvm/kvm_main.c b/virt/kvm/kvm_main.c index c55f5d6..823544c 100644 --- a/virt/kvm/kvm_main.c +++ b/virt/kvm/kvm_main.c @@ -70,16 +70,19 @@ MODULE_AUTHOR("Qumranet"); MODULE_LICENSE("GPL"); /* Architectures should define their poll value according to the halt latency */ -static unsigned int halt_poll_ns = KVM_HALT_POLL_NS_DEFAULT; +unsigned int halt_poll_ns = KVM_HALT_POLL_NS_DEFAULT; module_param(halt_poll_ns, uint, S_IRUGO | S_IWUSR); +EXPORT_SYMBOL_GPL(halt_poll_ns); /* Default doubles per-vcpu halt_poll_ns. */ -static unsigned int halt_poll_ns_grow = 2; +unsigned int halt_poll_ns_grow = 2; module_param(halt_poll_ns_grow, uint, S_IRUGO | S_IWUSR); +EXPORT_SYMBOL_GPL(halt_poll_ns_grow); /* Default resets per-vcpu halt_poll_ns . */ -static unsigned int halt_poll_ns_shrink; +unsigned int halt_poll_ns_shrink; module_param(halt_poll_ns_shrink, uint, S_IRUGO | S_IWUSR); +EXPORT_SYMBOL_GPL(halt_poll_ns_shrink); /* * Ordering of locks: @@ -595,7 +598,7 @@ static int kvm_create_vm_debugfs(struct kvm *kvm, int fd) stat_data->kvm = kvm; stat_data->offset = p->offset; kvm->debugfs_stat_data[p - debugfs_entries] = stat_data; - if (!debugfs_create_file(p->name, 0444, + if (!debugfs_create_file(p->name, 0644, kvm->debugfs_dentry, stat_data, stat_fops_per_vm[p->kind])) @@ -3669,11 +3672,23 @@ static int vm_stat_get_per_vm(void *data, u64 *val) return 0; } +static int vm_stat_clear_per_vm(void *data, u64 val) +{ + struct kvm_stat_data *stat_data = (struct kvm_stat_data *)data; + + if (val) + return -EINVAL; + + *(ulong *)((void *)stat_data->kvm + stat_data->offset) = 0; + + return 0; +} + static int vm_stat_get_per_vm_open(struct inode *inode, struct file *file) { __simple_attr_check_format("%llu\n", 0ull); return kvm_debugfs_open(inode, file, vm_stat_get_per_vm, - NULL, "%llu\n"); + vm_stat_clear_per_vm, "%llu\n"); } static const struct file_operations vm_stat_get_per_vm_fops = { @@ -3699,11 +3714,26 @@ static int vcpu_stat_get_per_vm(void *data, u64 *val) return 0; } +static int vcpu_stat_clear_per_vm(void *data, u64 val) +{ + int i; + struct kvm_stat_data *stat_data = (struct kvm_stat_data *)data; + struct kvm_vcpu *vcpu; + + if (val) + return -EINVAL; + + kvm_for_each_vcpu(i, vcpu, stat_data->kvm) + *(u64 *)((void *)vcpu + stat_data->offset) = 0; + + return 0; +} + static int vcpu_stat_get_per_vm_open(struct inode *inode, struct file *file) { __simple_attr_check_format("%llu\n", 0ull); return kvm_debugfs_open(inode, file, vcpu_stat_get_per_vm, - NULL, "%llu\n"); + vcpu_stat_clear_per_vm, "%llu\n"); } static const struct file_operations vcpu_stat_get_per_vm_fops = { @@ -3738,7 +3768,26 @@ static int vm_stat_get(void *_offset, u64 *val) return 0; } -DEFINE_SIMPLE_ATTRIBUTE(vm_stat_fops, vm_stat_get, NULL, "%llu\n"); +static int vm_stat_clear(void *_offset, u64 val) +{ + unsigned offset = (long)_offset; + struct kvm *kvm; + struct kvm_stat_data stat_tmp = {.offset = offset}; + + if (val) + return -EINVAL; + + spin_lock(&kvm_lock); + list_for_each_entry(kvm, &vm_list, vm_list) { + stat_tmp.kvm = kvm; + vm_stat_clear_per_vm((void *)&stat_tmp, 0); + } + spin_unlock(&kvm_lock); + + return 0; +} + +DEFINE_SIMPLE_ATTRIBUTE(vm_stat_fops, vm_stat_get, vm_stat_clear, "%llu\n"); static int vcpu_stat_get(void *_offset, u64 *val) { @@ -3758,7 +3807,27 @@ static int vcpu_stat_get(void *_offset, u64 *val) return 0; } -DEFINE_SIMPLE_ATTRIBUTE(vcpu_stat_fops, vcpu_stat_get, NULL, "%llu\n"); +static int vcpu_stat_clear(void *_offset, u64 val) +{ + unsigned offset = (long)_offset; + struct kvm *kvm; + struct kvm_stat_data stat_tmp = {.offset = offset}; + + if (val) + return -EINVAL; + + spin_lock(&kvm_lock); + list_for_each_entry(kvm, &vm_list, vm_list) { + stat_tmp.kvm = kvm; + vcpu_stat_clear_per_vm((void *)&stat_tmp, 0); + } + spin_unlock(&kvm_lock); + + return 0; +} + +DEFINE_SIMPLE_ATTRIBUTE(vcpu_stat_fops, vcpu_stat_get, vcpu_stat_clear, + "%llu\n"); static const struct file_operations *stat_fops[] = { [KVM_STAT_VCPU] = &vcpu_stat_fops, @@ -3776,7 +3845,7 @@ static int kvm_init_debug(void) kvm_debugfs_num_entries = 0; for (p = debugfs_entries; p->name; ++p, kvm_debugfs_num_entries++) { - if (!debugfs_create_file(p->name, 0444, kvm_debugfs_dir, + if (!debugfs_create_file(p->name, 0644, kvm_debugfs_dir, (void *)(long)p->offset, stat_fops[p->kind])) goto out_dir; |
