Merge branch 'akpm' (patches from Andrew)
Merge misc updates from Andrew Morton:
- a few MM hotfixes
- kthread, tools, scripts, ntfs and ocfs2
- some of MM
Subsystems affected by this patch series: kthread, tools, scripts, ntfs,
ocfs2 and mm (hofixes, pagealloc, slab-generic, slab, slub, kcsan,
debug, pagecache, gup, swap, shmem, memcg, pagemap, mremap, mincore,
sparsemem, vmalloc, kasan, pagealloc, hugetlb and vmscan).
* emailed patches from Andrew Morton <akpm@linux-foundation.org>: (162 commits)
mm: vmscan: consistent update to pgrefill
mm/vmscan.c: fix typo
khugepaged: khugepaged_test_exit() check mmget_still_valid()
khugepaged: retract_page_tables() remember to test exit
khugepaged: collapse_pte_mapped_thp() protect the pmd lock
khugepaged: collapse_pte_mapped_thp() flush the right range
mm/hugetlb: fix calculation of adjust_range_if_pmd_sharing_possible
mm: thp: replace HTTP links with HTTPS ones
mm/page_alloc: fix memalloc_nocma_{save/restore} APIs
mm/page_alloc.c: skip setting nodemask when we are in interrupt
mm/page_alloc: fallbacks at most has 3 elements
mm/page_alloc: silence a KASAN false positive
mm/page_alloc.c: remove unnecessary end_bitidx for [set|get]_pfnblock_flags_mask()
mm/page_alloc.c: simplify pageblock bitmap access
mm/page_alloc.c: extract the common part in pfn_to_bitidx()
mm/page_alloc.c: replace the definition of NR_MIGRATETYPE_BITS with PB_migratetype_bits
mm/shuffle: remove dynamic reconfiguration
mm/memory_hotplug: document why shuffle_zone() is relevant
mm/page_alloc: remove nr_free_pagecache_pages()
mm: remove vm_total_pages
...
diff --git a/Documentation/ABI/testing/sysfs-bus-event_source-devices-hv_24x7 b/Documentation/ABI/testing/sysfs-bus-event_source-devices-hv_24x7
index e8698af..f7e32f2 100644
--- a/Documentation/ABI/testing/sysfs-bus-event_source-devices-hv_24x7
+++ b/Documentation/ABI/testing/sysfs-bus-event_source-devices-hv_24x7
@@ -43,6 +43,13 @@
This sysfs interface exposes the number of cores per chip
present in the system.
+What: /sys/devices/hv_24x7/interface/cpumask
+Date: July 2020
+Contact: Linux on PowerPC Developer List <linuxppc-dev@lists.ozlabs.org>
+Description: read only
+ This sysfs file exposes the cpumask which is designated to make
+ HCALLs to retrieve hv-24x7 pmu event counter data.
+
What: /sys/bus/event_source/devices/hv_24x7/event_descs/<event-name>
Date: February 2014
Contact: Linux on PowerPC Developer List <linuxppc-dev@lists.ozlabs.org>
diff --git a/Documentation/ABI/testing/sysfs-bus-papr-pmem b/Documentation/ABI/testing/sysfs-bus-papr-pmem
index 5b10d03..c1a6727 100644
--- a/Documentation/ABI/testing/sysfs-bus-papr-pmem
+++ b/Documentation/ABI/testing/sysfs-bus-papr-pmem
@@ -25,3 +25,30 @@
NVDIMM have been scrubbed.
* "locked" : Indicating that NVDIMM contents cant
be modified until next power cycle.
+
+What: /sys/bus/nd/devices/nmemX/papr/perf_stats
+Date: May, 2020
+KernelVersion: v5.9
+Contact: linuxppc-dev <linuxppc-dev@lists.ozlabs.org>, linux-nvdimm@lists.01.org,
+Description:
+ (RO) Report various performance stats related to papr-scm NVDIMM
+ device. Each stat is reported on a new line with each line
+ composed of a stat-identifier followed by it value. Below are
+ currently known dimm performance stats which are reported:
+
+ * "CtlResCt" : Controller Reset Count
+ * "CtlResTm" : Controller Reset Elapsed Time
+ * "PonSecs " : Power-on Seconds
+ * "MemLife " : Life Remaining
+ * "CritRscU" : Critical Resource Utilization
+ * "HostLCnt" : Host Load Count
+ * "HostSCnt" : Host Store Count
+ * "HostSDur" : Host Store Duration
+ * "HostLDur" : Host Load Duration
+ * "MedRCnt " : Media Read Count
+ * "MedWCnt " : Media Write Count
+ * "MedRDur " : Media Read Duration
+ * "MedWDur " : Media Write Duration
+ * "CchRHCnt" : Cache Read Hit Count
+ * "CchWHCnt" : Cache Write Hit Count
+ * "FastWCnt" : Fast Write Count
\ No newline at end of file
diff --git a/Documentation/ABI/testing/sysfs-class-ocxl b/Documentation/ABI/testing/sysfs-class-ocxl
index b5b1fa1..ae1276e 100644
--- a/Documentation/ABI/testing/sysfs-class-ocxl
+++ b/Documentation/ABI/testing/sysfs-class-ocxl
@@ -33,3 +33,14 @@
Contact: linuxppc-dev@lists.ozlabs.org
Description: read/write
Give access the global mmio area for the AFU
+
+What: /sys/class/ocxl/<afu name>/reload_on_reset
+Date: February 2020
+Contact: linuxppc-dev@lists.ozlabs.org
+Description: read/write
+ Control whether the FPGA is reloaded on a link reset. Enabled
+ through a vendor-specific logic block on the FPGA.
+ 0 Do not reload FPGA image from flash
+ 1 Reload FPGA image from flash
+ unavailable
+ The device does not support this capability
diff --git a/Documentation/ABI/testing/sysfs-driver-ufs b/Documentation/ABI/testing/sysfs-driver-ufs
index 016724e..d1a3521 100644
--- a/Documentation/ABI/testing/sysfs-driver-ufs
+++ b/Documentation/ABI/testing/sysfs-driver-ufs
@@ -883,3 +883,139 @@
Description: This entry shows the target state of an UFS UIC link
for the chosen system power management level.
The file is read only.
+
+What: /sys/bus/platform/drivers/ufshcd/*/device_descriptor/wb_presv_us_en
+Date: June 2020
+Contact: Asutosh Das <asutoshd@codeaurora.org>
+Description: This entry shows if preserve user-space was configured
+ The file is read only.
+
+What: /sys/bus/platform/drivers/ufshcd/*/device_descriptor/wb_shared_alloc_units
+Date: June 2020
+Contact: Asutosh Das <asutoshd@codeaurora.org>
+Description: This entry shows the shared allocated units of WB buffer
+ The file is read only.
+
+What: /sys/bus/platform/drivers/ufshcd/*/device_descriptor/wb_type
+Date: June 2020
+Contact: Asutosh Das <asutoshd@codeaurora.org>
+Description: This entry shows the configured WB type.
+ 0x1 for shared buffer mode. 0x0 for dedicated buffer mode.
+ The file is read only.
+
+What: /sys/bus/platform/drivers/ufshcd/*/geometry_descriptor/wb_buff_cap_adj
+Date: June 2020
+Contact: Asutosh Das <asutoshd@codeaurora.org>
+Description: This entry shows the total user-space decrease in shared
+ buffer mode.
+ The value of this parameter is 3 for TLC NAND when SLC mode
+ is used as WriteBooster Buffer. 2 for MLC NAND.
+ The file is read only.
+
+What: /sys/bus/platform/drivers/ufshcd/*/geometry_descriptor/wb_max_alloc_units
+Date: June 2020
+Contact: Asutosh Das <asutoshd@codeaurora.org>
+Description: This entry shows the Maximum total WriteBooster Buffer size
+ which is supported by the entire device.
+ The file is read only.
+
+What: /sys/bus/platform/drivers/ufshcd/*/geometry_descriptor/wb_max_wb_luns
+Date: June 2020
+Contact: Asutosh Das <asutoshd@codeaurora.org>
+Description: This entry shows the maximum number of luns that can support
+ WriteBooster.
+ The file is read only.
+
+What: /sys/bus/platform/drivers/ufshcd/*/geometry_descriptor/wb_sup_red_type
+Date: June 2020
+Contact: Asutosh Das <asutoshd@codeaurora.org>
+Description: The supportability of user space reduction mode
+ and preserve user space mode.
+ 00h: WriteBooster Buffer can be configured only in
+ user space reduction type.
+ 01h: WriteBooster Buffer can be configured only in
+ preserve user space type.
+ 02h: Device can be configured in either user space
+ reduction type or preserve user space type.
+ The file is read only.
+
+What: /sys/bus/platform/drivers/ufshcd/*/geometry_descriptor/wb_sup_wb_type
+Date: June 2020
+Contact: Asutosh Das <asutoshd@codeaurora.org>
+Description: The supportability of WriteBooster Buffer type.
+ 00h: LU based WriteBooster Buffer configuration
+ 01h: Single shared WriteBooster Buffer
+ configuration
+ 02h: Supporting both LU based WriteBooster
+ Buffer and Single shared WriteBooster Buffer
+ configuration
+ The file is read only.
+
+What: /sys/bus/platform/drivers/ufshcd/*/flags/wb_enable
+Date: June 2020
+Contact: Asutosh Das <asutoshd@codeaurora.org>
+Description: This entry shows the status of WriteBooster.
+ 0: WriteBooster is not enabled.
+ 1: WriteBooster is enabled
+ The file is read only.
+
+What: /sys/bus/platform/drivers/ufshcd/*/flags/wb_flush_en
+Date: June 2020
+Contact: Asutosh Das <asutoshd@codeaurora.org>
+Description: This entry shows if flush is enabled.
+ 0: Flush operation is not performed.
+ 1: Flush operation is performed.
+ The file is read only.
+
+What: /sys/bus/platform/drivers/ufshcd/*/flags/wb_flush_during_h8
+Date: June 2020
+Contact: Asutosh Das <asutoshd@codeaurora.org>
+Description: Flush WriteBooster Buffer during hibernate state.
+ 0: Device is not allowed to flush the
+ WriteBooster Buffer during link hibernate
+ state.
+ 1: Device is allowed to flush the
+ WriteBooster Buffer during link hibernate
+ state
+ The file is read only.
+
+What: /sys/bus/platform/drivers/ufshcd/*/attributes/wb_avail_buf
+Date: June 2020
+Contact: Asutosh Das <asutoshd@codeaurora.org>
+Description: This entry shows the amount of unused WriteBooster buffer
+ available.
+ The file is read only.
+
+What: /sys/bus/platform/drivers/ufshcd/*/attributes/wb_cur_buf
+Date: June 2020
+Contact: Asutosh Das <asutoshd@codeaurora.org>
+Description: This entry shows the amount of unused current buffer.
+ The file is read only.
+
+What: /sys/bus/platform/drivers/ufshcd/*/attributes/wb_flush_status
+Date: June 2020
+Contact: Asutosh Das <asutoshd@codeaurora.org>
+Description: This entry shows the flush operation status.
+ 00h: idle
+ 01h: Flush operation in progress
+ 02h: Flush operation stopped prematurely.
+ 03h: Flush operation completed successfully
+ 04h: Flush operation general failure
+ The file is read only.
+
+What: /sys/bus/platform/drivers/ufshcd/*/attributes/wb_life_time_est
+Date: June 2020
+Contact: Asutosh Das <asutoshd@codeaurora.org>
+Description: This entry shows an indication of the WriteBooster Buffer
+ lifetime based on the amount of performed program/erase cycles
+ 01h: 0% - 10% WriteBooster Buffer life time used
+ ...
+ 0Ah: 90% - 100% WriteBooster Buffer life time used
+ The file is read only.
+
+What: /sys/class/scsi_device/*/device/unit_descriptor/wb_buf_alloc_units
+Date: June 2020
+Contact: Asutosh Das <asutoshd@codeaurora.org>
+Description: This entry shows the configured size of WriteBooster buffer.
+ 0400h corresponds to 4GB.
+ The file is read only.
diff --git a/Documentation/admin-guide/kernel-parameters.txt b/Documentation/admin-guide/kernel-parameters.txt
index c955522..98ea67f 100644
--- a/Documentation/admin-guide/kernel-parameters.txt
+++ b/Documentation/admin-guide/kernel-parameters.txt
@@ -916,6 +916,10 @@
disable_radix [PPC]
Disable RADIX MMU mode on POWER9
+ radix_hcall_invalidate=on [PPC/PSERIES]
+ Disable RADIX GTSE feature and use hcall for TLB
+ invalidate.
+
disable_tlbie [PPC]
Disable TLBIE instruction. Currently does not work
with KVM, with HASH MMU, or with coherent accelerators.
diff --git a/Documentation/core-api/cpu_hotplug.rst b/Documentation/core-api/cpu_hotplug.rst
index f646687..298c9c8 100644
--- a/Documentation/core-api/cpu_hotplug.rst
+++ b/Documentation/core-api/cpu_hotplug.rst
@@ -50,13 +50,6 @@
This option is limited to the X86 and S390 architecture.
-``cede_offline={"off","on"}``
- Use this option to disable/enable putting offlined processors to an extended
- ``H_CEDE`` state on supported pseries platforms. If nothing is specified,
- ``cede_offline`` is set to "on".
-
- This option is limited to the PowerPC architecture.
-
``cpu0_hotplug``
Allow to shutdown CPU0.
diff --git a/Documentation/devicetree/bindings/arm/arm,scmi.txt b/Documentation/devicetree/bindings/arm/arm,scmi.txt
index 1f293ea..55deb68 100644
--- a/Documentation/devicetree/bindings/arm/arm,scmi.txt
+++ b/Documentation/devicetree/bindings/arm/arm,scmi.txt
@@ -102,7 +102,7 @@
[0] http://infocenter.arm.com/help/topic/com.arm.doc.den0056a/index.html
[1] Documentation/devicetree/bindings/clock/clock-bindings.txt
[2] Documentation/devicetree/bindings/power/power-domain.yaml
-[3] Documentation/devicetree/bindings/thermal/thermal.txt
+[3] Documentation/devicetree/bindings/thermal/thermal*.yaml
[4] Documentation/devicetree/bindings/sram/sram.yaml
[5] Documentation/devicetree/bindings/reset/reset.txt
diff --git a/Documentation/devicetree/bindings/arm/arm,scpi.txt b/Documentation/devicetree/bindings/arm/arm,scpi.txt
index dd04d9d..bcd6c3e 100644
--- a/Documentation/devicetree/bindings/arm/arm,scpi.txt
+++ b/Documentation/devicetree/bindings/arm/arm,scpi.txt
@@ -108,7 +108,7 @@
[0] http://infocenter.arm.com/help/topic/com.arm.doc.dui0922b/index.html
[1] Documentation/devicetree/bindings/clock/clock-bindings.txt
-[2] Documentation/devicetree/bindings/thermal/thermal.txt
+[2] Documentation/devicetree/bindings/thermal/thermal*.yaml
[3] Documentation/devicetree/bindings/sram/sram.yaml
[4] Documentation/devicetree/bindings/power/power-domain.yaml
diff --git a/Documentation/devicetree/bindings/arm/freescale/fsl,scu.txt b/Documentation/devicetree/bindings/arm/freescale/fsl,scu.txt
index 10b8459..6064d98 100644
--- a/Documentation/devicetree/bindings/arm/freescale/fsl,scu.txt
+++ b/Documentation/devicetree/bindings/arm/freescale/fsl,scu.txt
@@ -176,7 +176,7 @@
"fsl,imx8qxp-sc-thermal"
followed by "fsl,imx-sc-thermal";
-- #thermal-sensor-cells: See Documentation/devicetree/bindings/thermal/thermal.txt
+- #thermal-sensor-cells: See Documentation/devicetree/bindings/thermal/thermal-sensor.yaml
for a description.
Example (imx8qxp):
diff --git a/Documentation/devicetree/bindings/arm/marvell/ap80x-system-controller.txt b/Documentation/devicetree/bindings/arm/marvell/ap80x-system-controller.txt
index 098d932..e315112 100644
--- a/Documentation/devicetree/bindings/arm/marvell/ap80x-system-controller.txt
+++ b/Documentation/devicetree/bindings/arm/marvell/ap80x-system-controller.txt
@@ -111,7 +111,7 @@
--------
For common binding part and usage, refer to
-Documentation/devicetree/bindings/thermal/thermal.txt
+Documentation/devicetree/bindings/thermal/thermal*.yaml
The thermal IP can probe the temperature all around the processor. It
may feature several channels, each of them wired to one sensor.
diff --git a/Documentation/devicetree/bindings/arm/marvell/cp110-system-controller.txt b/Documentation/devicetree/bindings/arm/marvell/cp110-system-controller.txt
index f982a8e..a21f770 100644
--- a/Documentation/devicetree/bindings/arm/marvell/cp110-system-controller.txt
+++ b/Documentation/devicetree/bindings/arm/marvell/cp110-system-controller.txt
@@ -203,7 +203,7 @@
critical point to any subnode of the thermal-zone node.
For common binding part and usage, refer to
-Documentation/devicetree/bindings/thermal/thermal.txt
+Documentation/devicetree/bindings/thermal/thermal*.yaml
Required properties:
- compatible: must be one of:
diff --git a/Documentation/devicetree/bindings/cpufreq/cpufreq-dt.txt b/Documentation/devicetree/bindings/cpufreq/cpufreq-dt.txt
index 332aed8..56f4423 100644
--- a/Documentation/devicetree/bindings/cpufreq/cpufreq-dt.txt
+++ b/Documentation/devicetree/bindings/cpufreq/cpufreq-dt.txt
@@ -18,7 +18,8 @@
in unit of nanoseconds.
- voltage-tolerance: Specify the CPU voltage tolerance in percentage.
- #cooling-cells:
- Please refer to Documentation/devicetree/bindings/thermal/thermal.txt.
+ Please refer to
+ Documentation/devicetree/bindings/thermal/thermal-cooling-devices.yaml.
Examples:
diff --git a/Documentation/devicetree/bindings/cpufreq/cpufreq-mediatek.txt b/Documentation/devicetree/bindings/cpufreq/cpufreq-mediatek.txt
index 0551c78..ea4994b 100644
--- a/Documentation/devicetree/bindings/cpufreq/cpufreq-mediatek.txt
+++ b/Documentation/devicetree/bindings/cpufreq/cpufreq-mediatek.txt
@@ -21,8 +21,8 @@
flow is handled by hardware, hence no software "voltage tracking" is
needed.
- #cooling-cells:
- Please refer to Documentation/devicetree/bindings/thermal/thermal.txt
- for detail.
+ For details, please refer to
+ Documentation/devicetree/bindings/thermal/thermal-cooling-devices.yaml
Example 1 (MT7623 SoC):
diff --git a/Documentation/devicetree/bindings/cpufreq/nvidia,tegra20-cpufreq.txt b/Documentation/devicetree/bindings/cpufreq/nvidia,tegra20-cpufreq.txt
index daeca6a..52a24b8 100644
--- a/Documentation/devicetree/bindings/cpufreq/nvidia,tegra20-cpufreq.txt
+++ b/Documentation/devicetree/bindings/cpufreq/nvidia,tegra20-cpufreq.txt
@@ -5,7 +5,7 @@
- clocks: Must contain an entry for the CPU clock.
See ../clocks/clock-bindings.txt for details.
- operating-points-v2: See ../bindings/opp/opp.txt for details.
-- #cooling-cells: Should be 2. See ../thermal/thermal.txt for details.
+- #cooling-cells: Should be 2. See ../thermal/thermal-cooling-devices.yaml for details.
For each opp entry in 'operating-points-v2' table:
- opp-supported-hw: Two bitfields indicating:
diff --git a/Documentation/devicetree/bindings/hwmon/gpio-fan.txt b/Documentation/devicetree/bindings/hwmon/gpio-fan.txt
index 2becdcf..f4cfa35 100644
--- a/Documentation/devicetree/bindings/hwmon/gpio-fan.txt
+++ b/Documentation/devicetree/bindings/hwmon/gpio-fan.txt
@@ -12,7 +12,8 @@
- alarm-gpios: This pin going active indicates something is wrong with
the fan, and a udev event will be fired.
- #cooling-cells: If used as a cooling device, must be <2>
- Also see: Documentation/devicetree/bindings/thermal/thermal.txt
+ Also see:
+ Documentation/devicetree/bindings/thermal/thermal-cooling-devices.yaml
min and max states are derived from the speed-map of the fan.
Note: At least one the "gpios" or "alarm-gpios" properties must be set.
diff --git a/Documentation/devicetree/bindings/hwmon/lm90.txt b/Documentation/devicetree/bindings/hwmon/lm90.txt
index c76a7ac..398dcb9 100644
--- a/Documentation/devicetree/bindings/hwmon/lm90.txt
+++ b/Documentation/devicetree/bindings/hwmon/lm90.txt
@@ -34,8 +34,8 @@
LM90 "-ALERT" pin output.
See interrupt-controller/interrupts.txt for the format.
-- #thermal-sensor-cells: should be set to 1. See thermal/thermal.txt for
- details. See <include/dt-bindings/thermal/lm90.h> for the
+- #thermal-sensor-cells: should be set to 1. See thermal/thermal-sensor.yaml
+ for details. See <include/dt-bindings/thermal/lm90.h> for the
definition of the local, remote and 2nd remote sensor index
constants.
diff --git a/Documentation/devicetree/bindings/thermal/allwinner,sun8i-a83t-ths.yaml b/Documentation/devicetree/bindings/thermal/allwinner,sun8i-a83t-ths.yaml
index 8736926..44ba676 100644
--- a/Documentation/devicetree/bindings/thermal/allwinner,sun8i-a83t-ths.yaml
+++ b/Documentation/devicetree/bindings/thermal/allwinner,sun8i-a83t-ths.yaml
@@ -50,7 +50,7 @@
nvmem-cell-names:
const: calibration
- # See ./thermal.txt for details
+ # See Documentation/devicetree/bindings/thermal/thermal-sensor.yaml for details
"#thermal-sensor-cells":
enum:
- 0
diff --git a/Documentation/devicetree/bindings/thermal/amazon,al-thermal.txt b/Documentation/devicetree/bindings/thermal/amazon,al-thermal.txt
index 703979d..12fc4ef 100644
--- a/Documentation/devicetree/bindings/thermal/amazon,al-thermal.txt
+++ b/Documentation/devicetree/bindings/thermal/amazon,al-thermal.txt
@@ -6,7 +6,7 @@
Required properties:
- compatible: "amazon,al-thermal".
- reg: The physical base address and length of the sensor's registers.
-- #thermal-sensor-cells: Must be 1. See ./thermal.txt for a description.
+- #thermal-sensor-cells: Must be 1. See Documentation/devicetree/bindings/thermal/thermal-sensor.yaml for a description.
Example:
thermal: thermal {
diff --git a/Documentation/devicetree/bindings/thermal/brcm,avs-ro-thermal.yaml b/Documentation/devicetree/bindings/thermal/brcm,avs-ro-thermal.yaml
index f3e68ed..1ab5070 100644
--- a/Documentation/devicetree/bindings/thermal/brcm,avs-ro-thermal.yaml
+++ b/Documentation/devicetree/bindings/thermal/brcm,avs-ro-thermal.yaml
@@ -23,7 +23,7 @@
compatible:
const: brcm,bcm2711-thermal
- # See ./thermal.txt for details
+ # See Documentation/devicetree/bindings/thermal/thermal-sensor.yaml for details
"#thermal-sensor-cells":
const: 0
diff --git a/Documentation/devicetree/bindings/thermal/brcm,bcm2835-thermal.txt b/Documentation/devicetree/bindings/thermal/brcm,bcm2835-thermal.txt
index da8c5b7..a3e9ec5 100644
--- a/Documentation/devicetree/bindings/thermal/brcm,bcm2835-thermal.txt
+++ b/Documentation/devicetree/bindings/thermal/brcm,bcm2835-thermal.txt
@@ -7,7 +7,7 @@
"brcm,bcm2836-thermal" or "brcm,bcm2837-thermal"
reg: Address range of the thermal registers.
clocks: Phandle of the clock used by the thermal sensor.
-#thermal-sensor-cells: should be 0 (see thermal.txt)
+#thermal-sensor-cells: should be 0 (see Documentation/devicetree/bindings/thermal/thermal-sensor.yaml)
Example:
diff --git a/Documentation/devicetree/bindings/thermal/hisilicon-thermal.txt b/Documentation/devicetree/bindings/thermal/hisilicon-thermal.txt
index cef716a..4b19d80e 100644
--- a/Documentation/devicetree/bindings/thermal/hisilicon-thermal.txt
+++ b/Documentation/devicetree/bindings/thermal/hisilicon-thermal.txt
@@ -9,7 +9,7 @@
by /SOCTHERM/tsensor.
- clock-names: Input clock name, should be 'thermal_clk'.
- clocks: phandles for clock specified in "clock-names" property.
-- #thermal-sensor-cells: Should be 1. See ./thermal.txt for a description.
+- #thermal-sensor-cells: Should be 1. See Documentation/devicetree/bindings/thermal/thermal-sensor.yaml for a description.
Example :
diff --git a/Documentation/devicetree/bindings/thermal/max77620_thermal.txt b/Documentation/devicetree/bindings/thermal/max77620_thermal.txt
index 323a3b3..82ed5d4 100644
--- a/Documentation/devicetree/bindings/thermal/max77620_thermal.txt
+++ b/Documentation/devicetree/bindings/thermal/max77620_thermal.txt
@@ -8,12 +8,12 @@
Required properties:
-------------------
-#thermal-sensor-cells: Please refer <devicetree/bindings/thermal/thermal.txt>
- for more details.
+#thermal-sensor-cells: For more details, please refer to
+ <devicetree/bindings/thermal/thermal-sensor.yaml>
The value must be 0.
For more details, please refer generic thermal DT binding document
-<devicetree/bindings/thermal/thermal.txt>.
+<devicetree/bindings/thermal/thermal*.yaml>.
Please refer <devicetree/bindings/mfd/max77620.txt> for mfd DT binding
document for the MAX77620.
diff --git a/Documentation/devicetree/bindings/thermal/mediatek-thermal.txt b/Documentation/devicetree/bindings/thermal/mediatek-thermal.txt
index f8d7831..1e249c42 100644
--- a/Documentation/devicetree/bindings/thermal/mediatek-thermal.txt
+++ b/Documentation/devicetree/bindings/thermal/mediatek-thermal.txt
@@ -23,7 +23,7 @@
- resets: Reference to the reset controller controlling the thermal controller.
- mediatek,auxadc: A phandle to the AUXADC which the thermal controller uses
- mediatek,apmixedsys: A phandle to the APMIXEDSYS controller.
-- #thermal-sensor-cells : Should be 0. See ./thermal.txt for a description.
+- #thermal-sensor-cells : Should be 0. See Documentation/devicetree/bindings/thermal/thermal-sensor.yaml for a description.
Optional properties:
- nvmem-cells: A phandle to the calibration data provided by a nvmem device. If
diff --git a/Documentation/devicetree/bindings/thermal/nvidia,tegra124-soctherm.txt b/Documentation/devicetree/bindings/thermal/nvidia,tegra124-soctherm.txt
index f02f385..db880e7 100644
--- a/Documentation/devicetree/bindings/thermal/nvidia,tegra124-soctherm.txt
+++ b/Documentation/devicetree/bindings/thermal/nvidia,tegra124-soctherm.txt
@@ -28,9 +28,10 @@
See ../reset/reset.txt for details.
- reset-names : Must include the following entries:
- soctherm
-- #thermal-sensor-cells : Should be 1. See ./thermal.txt for a description
- of this property. See <dt-bindings/thermal/tegra124-soctherm.h> for a
- list of valid values when referring to thermal sensors.
+- #thermal-sensor-cells : Should be 1. For a description of this property, see
+ Documentation/devicetree/bindings/thermal/thermal-sensor.yaml.
+ See <dt-bindings/thermal/tegra124-soctherm.h> for a list of valid values
+ when referring to thermal sensors.
- throttle-cfgs: A sub-node which is a container of configuration for each
hardware throttle events. These events can be set as cooling devices.
* throttle events: Sub-nodes must be named as "light" or "heavy".
@@ -62,7 +63,8 @@
TEGRA_SOCTHERM_THROT_LEVEL_MED (75%),
TEGRA_SOCTHERM_THROT_LEVEL_HIGH (85%).
- #cooling-cells: Should be 1. This cooling device only support on/off state.
- See ./thermal.txt for a description of this property.
+ For a description of this property see:
+ Documentation/devicetree/bindings/thermal/thermal-cooling-devices.yaml
Optional properties: The following properties are T210 specific and
valid only for OCx throttle events.
diff --git a/Documentation/devicetree/bindings/thermal/nvidia,tegra186-bpmp-thermal.txt b/Documentation/devicetree/bindings/thermal/nvidia,tegra186-bpmp-thermal.txt
index e17c07b..fc87f6a 100644
--- a/Documentation/devicetree/bindings/thermal/nvidia,tegra186-bpmp-thermal.txt
+++ b/Documentation/devicetree/bindings/thermal/nvidia,tegra186-bpmp-thermal.txt
@@ -8,7 +8,7 @@
The BPMP thermal node must be located directly inside the main BPMP node. See
../firmware/nvidia,tegra186-bpmp.txt for details of the BPMP binding.
-This node represents a thermal sensor. See thermal.txt for details of the
+This node represents a thermal sensor. See Documentation/devicetree/bindings/thermal/thermal-sensor.yaml for details of the
core thermal binding.
Required properties:
diff --git a/Documentation/devicetree/bindings/thermal/qcom-spmi-temp-alarm.txt b/Documentation/devicetree/bindings/thermal/qcom-spmi-temp-alarm.txt
index 0273a92..2d5b2ad 100644
--- a/Documentation/devicetree/bindings/thermal/qcom-spmi-temp-alarm.txt
+++ b/Documentation/devicetree/bindings/thermal/qcom-spmi-temp-alarm.txt
@@ -8,7 +8,7 @@
- compatible: Should contain "qcom,spmi-temp-alarm".
- reg: Specifies the SPMI address.
- interrupts: PMIC temperature alarm interrupt.
-- #thermal-sensor-cells: Should be 0. See thermal.txt for a description.
+- #thermal-sensor-cells: Should be 0. See Documentation/devicetree/bindings/thermal/thermal-sensor.yaml for a description.
Optional properties:
- io-channels: Should contain IIO channel specifier for the ADC channel,
diff --git a/Documentation/devicetree/bindings/thermal/qcom-tsens.yaml b/Documentation/devicetree/bindings/thermal/qcom-tsens.yaml
index adac01c..95462e0 100644
--- a/Documentation/devicetree/bindings/thermal/qcom-tsens.yaml
+++ b/Documentation/devicetree/bindings/thermal/qcom-tsens.yaml
@@ -23,6 +23,7 @@
items:
- enum:
- qcom,msm8916-tsens
+ - qcom,msm8939-tsens
- qcom,msm8974-tsens
- const: qcom,tsens-v0_1
diff --git a/Documentation/devicetree/bindings/thermal/rockchip-thermal.txt b/Documentation/devicetree/bindings/thermal/rockchip-thermal.txt
index c6aac9b..7f94669 100644
--- a/Documentation/devicetree/bindings/thermal/rockchip-thermal.txt
+++ b/Documentation/devicetree/bindings/thermal/rockchip-thermal.txt
@@ -24,7 +24,7 @@
- pinctrl-1 : The "default" pinctrl state, it will be set after reset the
TSADC controller.
- pinctrl-2 : The "sleep" pinctrl state, it will be in for suspend.
-- #thermal-sensor-cells : Should be 1. See ./thermal.txt for a description.
+- #thermal-sensor-cells : Should be 1. See Documentation/devicetree/bindings/thermal/thermal-sensor.yaml for a description.
Optional properties:
- rockchip,hw-tshut-temp : The hardware-controlled shutdown temperature value.
diff --git a/Documentation/devicetree/bindings/thermal/tango-thermal.txt b/Documentation/devicetree/bindings/thermal/tango-thermal.txt
index 212198d..2c918d7 100644
--- a/Documentation/devicetree/bindings/thermal/tango-thermal.txt
+++ b/Documentation/devicetree/bindings/thermal/tango-thermal.txt
@@ -4,7 +4,7 @@
(in the CPU, video decoder, and PCIe controller).
Required properties:
-- #thermal-sensor-cells: Should be 0 (see thermal.txt)
+- #thermal-sensor-cells: Should be 0 (see Documentation/devicetree/bindings/thermal/thermal-sensor.yaml)
- compatible: "sigma,smp8758-thermal"
- reg: Address range of the thermal registers
diff --git a/Documentation/devicetree/bindings/thermal/thermal-generic-adc.txt b/Documentation/devicetree/bindings/thermal/thermal-generic-adc.txt
index 691a09d..e136946 100644
--- a/Documentation/devicetree/bindings/thermal/thermal-generic-adc.txt
+++ b/Documentation/devicetree/bindings/thermal/thermal-generic-adc.txt
@@ -8,7 +8,7 @@
Required properties:
===================
- compatible: Must be "generic-adc-thermal".
-- #thermal-sensor-cells: Should be 1. See ./thermal.txt for a description
+- #thermal-sensor-cells: Should be 1. See Documentation/devicetree/bindings/thermal/thermal-sensor.yaml for a description
of this property.
Optional properties:
===================
diff --git a/Documentation/devicetree/bindings/thermal/thermal.txt b/Documentation/devicetree/bindings/thermal/thermal.txt
deleted file mode 100644
index f78bec1..0000000
--- a/Documentation/devicetree/bindings/thermal/thermal.txt
+++ /dev/null
@@ -1,586 +0,0 @@
-* Thermal Framework Device Tree descriptor
-
-This file describes a generic binding to provide a way of
-defining hardware thermal structure using device tree.
-A thermal structure includes thermal zones and their components,
-such as trip points, polling intervals, sensors and cooling devices
-binding descriptors.
-
-The target of device tree thermal descriptors is to describe only
-the hardware thermal aspects. The thermal device tree bindings are
-not about how the system must control or which algorithm or policy
-must be taken in place.
-
-There are five types of nodes involved to describe thermal bindings:
-- thermal sensors: devices which may be used to take temperature
- measurements.
-- cooling devices: devices which may be used to dissipate heat.
-- trip points: describe key temperatures at which cooling is recommended. The
- set of points should be chosen based on hardware limits.
-- cooling maps: used to describe links between trip points and cooling devices;
-- thermal zones: used to describe thermal data within the hardware;
-
-The following is a description of each of these node types.
-
-* Thermal sensor devices
-
-Thermal sensor devices are nodes providing temperature sensing capabilities on
-thermal zones. Typical devices are I2C ADC converters and bandgaps. These are
-nodes providing temperature data to thermal zones. Thermal sensor devices may
-control one or more internal sensors.
-
-Required property:
-- #thermal-sensor-cells: Used to provide sensor device specific information
- Type: unsigned while referring to it. Typically 0 on thermal sensor
- Size: one cell nodes with only one sensor, and at least 1 on nodes
- with several internal sensors, in order
- to identify uniquely the sensor instances within
- the IC. See thermal zone binding for more details
- on how consumers refer to sensor devices.
-
-* Cooling device nodes
-
-Cooling devices are nodes providing control on power dissipation. There
-are essentially two ways to provide control on power dissipation. First
-is by means of regulating device performance, which is known as passive
-cooling. A typical passive cooling is a CPU that has dynamic voltage and
-frequency scaling (DVFS), and uses lower frequencies as cooling states.
-Second is by means of activating devices in order to remove
-the dissipated heat, which is known as active cooling, e.g. regulating
-fan speeds. In both cases, cooling devices shall have a way to determine
-the state of cooling in which the device is.
-
-Any cooling device has a range of cooling states (i.e. different levels
-of heat dissipation). For example a fan's cooling states correspond to
-the different fan speeds possible. Cooling states are referred to by
-single unsigned integers, where larger numbers mean greater heat
-dissipation. The precise set of cooling states associated with a device
-should be defined in a particular device's binding.
-For more examples of cooling devices, refer to the example sections below.
-
-Required properties:
-- #cooling-cells: Used to provide cooling device specific information
- Type: unsigned while referring to it. Must be at least 2, in order
- Size: one cell to specify minimum and maximum cooling state used
- in the reference. The first cell is the minimum
- cooling state requested and the second cell is
- the maximum cooling state requested in the reference.
- See Cooling device maps section below for more details
- on how consumers refer to cooling devices.
-
-* Trip points
-
-The trip node is a node to describe a point in the temperature domain
-in which the system takes an action. This node describes just the point,
-not the action.
-
-Required properties:
-- temperature: An integer indicating the trip temperature level,
- Type: signed in millicelsius.
- Size: one cell
-
-- hysteresis: A low hysteresis value on temperature property (above).
- Type: unsigned This is a relative value, in millicelsius.
- Size: one cell
-
-- type: a string containing the trip type. Expected values are:
- "active": A trip point to enable active cooling
- "passive": A trip point to enable passive cooling
- "hot": A trip point to notify emergency
- "critical": Hardware not reliable.
- Type: string
-
-* Cooling device maps
-
-The cooling device maps node is a node to describe how cooling devices
-get assigned to trip points of the zone. The cooling devices are expected
-to be loaded in the target system.
-
-Required properties:
-- cooling-device: A list of phandles of cooling devices with their specifiers,
- Type: phandle + referring to which cooling devices are used in this
- cooling specifier binding. In the cooling specifier, the first cell
- is the minimum cooling state and the second cell
- is the maximum cooling state used in this map.
-- trip: A phandle of a trip point node within the same thermal
- Type: phandle of zone.
- trip point node
-
-Optional property:
-- contribution: The cooling contribution to the thermal zone of the
- Type: unsigned referred cooling device at the referred trip point.
- Size: one cell The contribution is a ratio of the sum
- of all cooling contributions within a thermal zone.
-
-Note: Using the THERMAL_NO_LIMIT (-1UL) constant in the cooling-device phandle
-limit specifier means:
-(i) - minimum state allowed for minimum cooling state used in the reference.
-(ii) - maximum state allowed for maximum cooling state used in the reference.
-Refer to include/dt-bindings/thermal/thermal.h for definition of this constant.
-
-* Thermal zone nodes
-
-The thermal zone node is the node containing all the required info
-for describing a thermal zone, including its cooling device bindings. The
-thermal zone node must contain, apart from its own properties, one sub-node
-containing trip nodes and one sub-node containing all the zone cooling maps.
-
-Required properties:
-- polling-delay: The maximum number of milliseconds to wait between polls
- Type: unsigned when checking this thermal zone.
- Size: one cell
-
-- polling-delay-passive: The maximum number of milliseconds to wait
- Type: unsigned between polls when performing passive cooling.
- Size: one cell
-
-- thermal-sensors: A list of thermal sensor phandles and sensor specifier
- Type: list of used while monitoring the thermal zone.
- phandles + sensor
- specifier
-
-- trips: A sub-node which is a container of only trip point nodes
- Type: sub-node required to describe the thermal zone.
-
-Optional property:
-- cooling-maps: A sub-node which is a container of only cooling device
- Type: sub-node map nodes, used to describe the relation between trips
- and cooling devices.
-
-- coefficients: An array of integers (one signed cell) containing
- Type: array coefficients to compose a linear relation between
- Elem size: one cell the sensors listed in the thermal-sensors property.
- Elem type: signed Coefficients defaults to 1, in case this property
- is not specified. A simple linear polynomial is used:
- Z = c0 * x0 + c1 * x1 + ... + c(n-1) * x(n-1) + cn.
-
- The coefficients are ordered and they match with sensors
- by means of sensor ID. Additional coefficients are
- interpreted as constant offset.
-
-- sustainable-power: An estimate of the sustainable power (in mW) that the
- Type: unsigned thermal zone can dissipate at the desired
- Size: one cell control temperature. For reference, the
- sustainable power of a 4'' phone is typically
- 2000mW, while on a 10'' tablet is around
- 4500mW.
-
-Note: The delay properties are bound to the maximum dT/dt (temperature
-derivative over time) in two situations for a thermal zone:
-(i) - when passive cooling is activated (polling-delay-passive); and
-(ii) - when the zone just needs to be monitored (polling-delay) or
-when active cooling is activated.
-
-The maximum dT/dt is highly bound to hardware power consumption and dissipation
-capability. The delays should be chosen to account for said max dT/dt,
-such that a device does not cross several trip boundaries unexpectedly
-between polls. Choosing the right polling delays shall avoid having the
-device in temperature ranges that may damage the silicon structures and
-reduce silicon lifetime.
-
-* The thermal-zones node
-
-The "thermal-zones" node is a container for all thermal zone nodes. It shall
-contain only sub-nodes describing thermal zones as in the section
-"Thermal zone nodes". The "thermal-zones" node appears under "/".
-
-* Examples
-
-Below are several examples on how to use thermal data descriptors
-using device tree bindings:
-
-(a) - CPU thermal zone
-
-The CPU thermal zone example below describes how to setup one thermal zone
-using one single sensor as temperature source and many cooling devices and
-power dissipation control sources.
-
-#include <dt-bindings/thermal/thermal.h>
-
-cpus {
- /*
- * Here is an example of describing a cooling device for a DVFS
- * capable CPU. The CPU node describes its four OPPs.
- * The cooling states possible are 0..3, and they are
- * used as OPP indexes. The minimum cooling state is 0, which means
- * all four OPPs can be available to the system. The maximum
- * cooling state is 3, which means only the lowest OPPs (198MHz@0.85V)
- * can be available in the system.
- */
- cpu0: cpu@0 {
- ...
- operating-points = <
- /* kHz uV */
- 970000 1200000
- 792000 1100000
- 396000 950000
- 198000 850000
- >;
- #cooling-cells = <2>; /* min followed by max */
- };
- ...
-};
-
-&i2c1 {
- ...
- /*
- * A simple fan controller which supports 10 speeds of operation
- * (represented as 0-9).
- */
- fan0: fan@48 {
- ...
- #cooling-cells = <2>; /* min followed by max */
- };
-};
-
-ocp {
- ...
- /*
- * A simple IC with a single bandgap temperature sensor.
- */
- bandgap0: bandgap@0000ed00 {
- ...
- #thermal-sensor-cells = <0>;
- };
-};
-
-thermal-zones {
- cpu_thermal: cpu-thermal {
- polling-delay-passive = <250>; /* milliseconds */
- polling-delay = <1000>; /* milliseconds */
-
- thermal-sensors = <&bandgap0>;
-
- trips {
- cpu_alert0: cpu-alert0 {
- temperature = <90000>; /* millicelsius */
- hysteresis = <2000>; /* millicelsius */
- type = "active";
- };
- cpu_alert1: cpu-alert1 {
- temperature = <100000>; /* millicelsius */
- hysteresis = <2000>; /* millicelsius */
- type = "passive";
- };
- cpu_crit: cpu-crit {
- temperature = <125000>; /* millicelsius */
- hysteresis = <2000>; /* millicelsius */
- type = "critical";
- };
- };
-
- cooling-maps {
- map0 {
- trip = <&cpu_alert0>;
- cooling-device = <&fan0 THERMAL_NO_LIMIT 4>;
- };
- map1 {
- trip = <&cpu_alert1>;
- cooling-device = <&fan0 5 THERMAL_NO_LIMIT>, <&cpu0 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>;
- };
- };
- };
-};
-
-In the example above, the ADC sensor (bandgap0) at address 0x0000ED00 is
-used to monitor the zone 'cpu-thermal' using its sole sensor. A fan
-device (fan0) is controlled via I2C bus 1, at address 0x48, and has ten
-different cooling states 0-9. It is used to remove the heat out of
-the thermal zone 'cpu-thermal' using its cooling states
-from its minimum to 4, when it reaches trip point 'cpu_alert0'
-at 90C, as an example of active cooling. The same cooling device is used at
-'cpu_alert1', but from 5 to its maximum state. The cpu@0 device is also
-linked to the same thermal zone, 'cpu-thermal', as a passive cooling device,
-using all its cooling states at trip point 'cpu_alert1',
-which is a trip point at 100C. On the thermal zone 'cpu-thermal', at the
-temperature of 125C, represented by the trip point 'cpu_crit', the silicon
-is not reliable anymore.
-
-(b) - IC with several internal sensors
-
-The example below describes how to deploy several thermal zones based off a
-single sensor IC, assuming it has several internal sensors. This is a common
-case on SoC designs with several internal IPs that may need different thermal
-requirements, and thus may have their own sensor to monitor or detect internal
-hotspots in their silicon.
-
-#include <dt-bindings/thermal/thermal.h>
-
-ocp {
- ...
- /*
- * A simple IC with several bandgap temperature sensors.
- */
- bandgap0: bandgap@0000ed00 {
- ...
- #thermal-sensor-cells = <1>;
- };
-};
-
-thermal-zones {
- cpu_thermal: cpu-thermal {
- polling-delay-passive = <250>; /* milliseconds */
- polling-delay = <1000>; /* milliseconds */
-
- /* sensor ID */
- thermal-sensors = <&bandgap0 0>;
-
- trips {
- /* each zone within the SoC may have its own trips */
- cpu_alert: cpu-alert {
- temperature = <100000>; /* millicelsius */
- hysteresis = <2000>; /* millicelsius */
- type = "passive";
- };
- cpu_crit: cpu-crit {
- temperature = <125000>; /* millicelsius */
- hysteresis = <2000>; /* millicelsius */
- type = "critical";
- };
- };
-
- cooling-maps {
- /* each zone within the SoC may have its own cooling */
- ...
- };
- };
-
- gpu_thermal: gpu-thermal {
- polling-delay-passive = <120>; /* milliseconds */
- polling-delay = <1000>; /* milliseconds */
-
- /* sensor ID */
- thermal-sensors = <&bandgap0 1>;
-
- trips {
- /* each zone within the SoC may have its own trips */
- gpu_alert: gpu-alert {
- temperature = <90000>; /* millicelsius */
- hysteresis = <2000>; /* millicelsius */
- type = "passive";
- };
- gpu_crit: gpu-crit {
- temperature = <105000>; /* millicelsius */
- hysteresis = <2000>; /* millicelsius */
- type = "critical";
- };
- };
-
- cooling-maps {
- /* each zone within the SoC may have its own cooling */
- ...
- };
- };
-
- dsp_thermal: dsp-thermal {
- polling-delay-passive = <50>; /* milliseconds */
- polling-delay = <1000>; /* milliseconds */
-
- /* sensor ID */
- thermal-sensors = <&bandgap0 2>;
-
- trips {
- /* each zone within the SoC may have its own trips */
- dsp_alert: dsp-alert {
- temperature = <90000>; /* millicelsius */
- hysteresis = <2000>; /* millicelsius */
- type = "passive";
- };
- dsp_crit: gpu-crit {
- temperature = <135000>; /* millicelsius */
- hysteresis = <2000>; /* millicelsius */
- type = "critical";
- };
- };
-
- cooling-maps {
- /* each zone within the SoC may have its own cooling */
- ...
- };
- };
-};
-
-In the example above, there is one bandgap IC which has the capability to
-monitor three sensors. The hardware has been designed so that sensors are
-placed on different places in the DIE to monitor different temperature
-hotspots: one for CPU thermal zone, one for GPU thermal zone and the
-other to monitor a DSP thermal zone.
-
-Thus, there is a need to assign each sensor provided by the bandgap IC
-to different thermal zones. This is achieved by means of using the
-#thermal-sensor-cells property and using the first cell of the sensor
-specifier as sensor ID. In the example, then, <bandgap 0> is used to
-monitor CPU thermal zone, <bandgap 1> is used to monitor GPU thermal
-zone and <bandgap 2> is used to monitor DSP thermal zone. Each zone
-may be uncorrelated, having its own dT/dt requirements, trips
-and cooling maps.
-
-
-(c) - Several sensors within one single thermal zone
-
-The example below illustrates how to use more than one sensor within
-one thermal zone.
-
-#include <dt-bindings/thermal/thermal.h>
-
-&i2c1 {
- ...
- /*
- * A simple IC with a single temperature sensor.
- */
- adc: sensor@49 {
- ...
- #thermal-sensor-cells = <0>;
- };
-};
-
-ocp {
- ...
- /*
- * A simple IC with a single bandgap temperature sensor.
- */
- bandgap0: bandgap@0000ed00 {
- ...
- #thermal-sensor-cells = <0>;
- };
-};
-
-thermal-zones {
- cpu_thermal: cpu-thermal {
- polling-delay-passive = <250>; /* milliseconds */
- polling-delay = <1000>; /* milliseconds */
-
- thermal-sensors = <&bandgap0>, /* cpu */
- <&adc>; /* pcb north */
-
- /* hotspot = 100 * bandgap - 120 * adc + 484 */
- coefficients = <100 -120 484>;
-
- trips {
- ...
- };
-
- cooling-maps {
- ...
- };
- };
-};
-
-In some cases, there is a need to use more than one sensor to extrapolate
-a thermal hotspot in the silicon. The above example illustrates this situation.
-For instance, it may be the case that a sensor external to CPU IP may be placed
-close to CPU hotspot and together with internal CPU sensor, it is used
-to determine the hotspot. Assuming this is the case for the above example,
-the hypothetical extrapolation rule would be:
- hotspot = 100 * bandgap - 120 * adc + 484
-
-In other context, the same idea can be used to add fixed offset. For instance,
-consider the hotspot extrapolation rule below:
- hotspot = 1 * adc + 6000
-
-In the above equation, the hotspot is always 6C higher than what is read
-from the ADC sensor. The binding would be then:
- thermal-sensors = <&adc>;
-
- /* hotspot = 1 * adc + 6000 */
- coefficients = <1 6000>;
-
-(d) - Board thermal
-
-The board thermal example below illustrates how to setup one thermal zone
-with many sensors and many cooling devices.
-
-#include <dt-bindings/thermal/thermal.h>
-
-&i2c1 {
- ...
- /*
- * An IC with several temperature sensor.
- */
- adc_dummy: sensor@50 {
- ...
- #thermal-sensor-cells = <1>; /* sensor internal ID */
- };
-};
-
-thermal-zones {
- batt-thermal {
- polling-delay-passive = <500>; /* milliseconds */
- polling-delay = <2500>; /* milliseconds */
-
- /* sensor ID */
- thermal-sensors = <&adc_dummy 4>;
-
- trips {
- ...
- };
-
- cooling-maps {
- ...
- };
- };
-
- board_thermal: board-thermal {
- polling-delay-passive = <1000>; /* milliseconds */
- polling-delay = <2500>; /* milliseconds */
-
- /* sensor ID */
- thermal-sensors = <&adc_dummy 0>, /* pcb top edge */
- <&adc_dummy 1>, /* lcd */
- <&adc_dummy 2>; /* back cover */
- /*
- * An array of coefficients describing the sensor
- * linear relation. E.g.:
- * z = c1*x1 + c2*x2 + c3*x3
- */
- coefficients = <1200 -345 890>;
-
- sustainable-power = <2500>;
-
- trips {
- /* Trips are based on resulting linear equation */
- cpu_trip: cpu-trip {
- temperature = <60000>; /* millicelsius */
- hysteresis = <2000>; /* millicelsius */
- type = "passive";
- };
- gpu_trip: gpu-trip {
- temperature = <55000>; /* millicelsius */
- hysteresis = <2000>; /* millicelsius */
- type = "passive";
- }
- lcd_trip: lcp-trip {
- temperature = <53000>; /* millicelsius */
- hysteresis = <2000>; /* millicelsius */
- type = "passive";
- };
- crit_trip: crit-trip {
- temperature = <68000>; /* millicelsius */
- hysteresis = <2000>; /* millicelsius */
- type = "critical";
- };
- };
-
- cooling-maps {
- map0 {
- trip = <&cpu_trip>;
- cooling-device = <&cpu0 0 2>;
- contribution = <55>;
- };
- map1 {
- trip = <&gpu_trip>;
- cooling-device = <&gpu0 0 2>;
- contribution = <20>;
- };
- map2 {
- trip = <&lcd_trip>;
- cooling-device = <&lcd0 5 10>;
- contribution = <15>;
- };
- };
- };
-};
-
-The above example is a mix of previous examples, a sensor IP with several internal
-sensors used to monitor different zones, one of them is composed by several sensors and
-with different cooling devices.
diff --git a/Documentation/features/core/jump-labels/arch-support.txt b/Documentation/features/core/jump-labels/arch-support.txt
index f8ec5c1..894d969 100644
--- a/Documentation/features/core/jump-labels/arch-support.txt
+++ b/Documentation/features/core/jump-labels/arch-support.txt
@@ -23,7 +23,7 @@
| openrisc: | TODO |
| parisc: | ok |
| powerpc: | ok |
- | riscv: | TODO |
+ | riscv: | ok |
| s390: | ok |
| sh: | TODO |
| sparc: | ok |
diff --git a/Documentation/features/debug/debug-vm-pgtable/arch-support.txt b/Documentation/features/debug/debug-vm-pgtable/arch-support.txt
index ca6bacb..53da483 100644
--- a/Documentation/features/debug/debug-vm-pgtable/arch-support.txt
+++ b/Documentation/features/debug/debug-vm-pgtable/arch-support.txt
@@ -23,7 +23,7 @@
| openrisc: | TODO |
| parisc: | TODO |
| powerpc: | ok |
- | riscv: | TODO |
+ | riscv: | ok |
| s390: | ok |
| sh: | TODO |
| sparc: | TODO |
diff --git a/Documentation/features/sched/membarrier-sync-core/arch-support.txt b/Documentation/features/sched/membarrier-sync-core/arch-support.txt
index 68658a6..47e6903 100644
--- a/Documentation/features/sched/membarrier-sync-core/arch-support.txt
+++ b/Documentation/features/sched/membarrier-sync-core/arch-support.txt
@@ -5,7 +5,7 @@
#
# Architecture requirements
#
-# * arm/arm64
+# * arm/arm64/powerpc
#
# Rely on implicit context synchronization as a result of exception return
# when returning from IPI handler, and when returning to user-space.
@@ -45,7 +45,7 @@
| nios2: | TODO |
| openrisc: | TODO |
| parisc: | TODO |
- | powerpc: | TODO |
+ | powerpc: | ok |
| riscv: | TODO |
| s390: | TODO |
| sh: | TODO |
diff --git a/Documentation/filesystems/quota.rst b/Documentation/filesystems/quota.rst
index a30cdd4..abd4303 100644
--- a/Documentation/filesystems/quota.rst
+++ b/Documentation/filesystems/quota.rst
@@ -18,7 +18,7 @@
filesystem.
For more details about quota design, see the documentation in quota-tools package
-(http://sourceforge.net/projects/linuxquota).
+(https://sourceforge.net/projects/linuxquota).
Quota netlink interface
=======================
@@ -31,11 +31,11 @@
and processed accordingly.
The interface uses generic netlink framework (see
-http://lwn.net/Articles/208755/ and http://people.suug.ch/~tgr/libnl/ for more
-details about this layer). The name of the quota generic netlink interface
-is "VFS_DQUOT". Definitions of constants below are in <linux/quota.h>.
-Since the quota netlink protocol is not namespace aware, quota netlink messages
-are sent only in initial network namespace.
+https://lwn.net/Articles/208755/ and http://www.infradead.org/~tgr/libnl/ for
+more details about this layer). The name of the quota generic netlink interface
+is "VFS_DQUOT". Definitions of constants below are in <linux/quota.h>. Since
+the quota netlink protocol is not namespace aware, quota netlink messages are
+sent only in initial network namespace.
Currently, the interface supports only one message type QUOTA_NL_C_WARNING.
This command is used to send a notification about any of the above mentioned
diff --git a/Documentation/filesystems/udf.rst b/Documentation/filesystems/udf.rst
index d9badbf..f9489dd 100644
--- a/Documentation/filesystems/udf.rst
+++ b/Documentation/filesystems/udf.rst
@@ -72,4 +72,4 @@
Documentation on UDF and ECMA 167 is available FREE from:
- http://www.osta.org/
- - http://www.ecma-international.org/
+ - https://www.ecma-international.org/
diff --git a/Documentation/memory-barriers.txt b/Documentation/memory-barriers.txt
index 4e55aba..9618633 100644
--- a/Documentation/memory-barriers.txt
+++ b/Documentation/memory-barriers.txt
@@ -1935,6 +1935,20 @@
relaxed I/O accessors and the Documentation/DMA-API.txt file for more
information on consistent memory.
+ (*) pmem_wmb();
+
+ This is for use with persistent memory to ensure that stores for which
+ modifications are written to persistent storage reached a platform
+ durability domain.
+
+ For example, after a non-temporal write to pmem region, we use pmem_wmb()
+ to ensure that stores have reached a platform durability domain. This ensures
+ that stores have updated persistent storage before any data access or
+ data transfer caused by subsequent instructions is initiated. This is
+ in addition to the ordering done by wmb().
+
+ For load from persistent memory, existing read memory barriers are sufficient
+ to ensure read ordering.
===============================
IMPLICIT KERNEL MEMORY BARRIERS
diff --git a/Documentation/powerpc/cpu_families.rst b/Documentation/powerpc/cpu_families.rst
index 1e063c5..9b84e04 100644
--- a/Documentation/powerpc/cpu_families.rst
+++ b/Documentation/powerpc/cpu_families.rst
@@ -9,7 +9,9 @@
Book3S (aka sPAPR)
------------------
-- Hash MMU
+- Hash MMU (except 603 and e300)
+- Software loaded TLB (603 and e300)
+- Selectable Software loaded TLB in addition to hash MMU (755, 7450, e600)
- Mix of 32 & 64 bit::
+--------------+ +----------------+
@@ -24,9 +26,9 @@
| |
| |
v v
- +--------------+ +----------------+ +-------+
- | 604 | | 750 (G3) | ---> | 750CX |
- +--------------+ +----------------+ +-------+
+ +--------------+ +-----+ +----------------+ +-------+
+ | 604 | | 755 | <--- | 750 (G3) | ---> | 750CX |
+ +--------------+ +-----+ +----------------+ +-------+
| | |
| | |
v v v
diff --git a/Documentation/powerpc/mpc52xx.rst b/Documentation/powerpc/mpc52xx.rst
index 8676ac6..3026070 100644
--- a/Documentation/powerpc/mpc52xx.rst
+++ b/Documentation/powerpc/mpc52xx.rst
@@ -2,7 +2,7 @@
Linux 2.6.x on MPC52xx family
=============================
-For the latest info, go to http://www.246tNt.com/mpc52xx/
+For the latest info, go to https://www.246tNt.com/mpc52xx/
To compile/use :
diff --git a/Documentation/powerpc/syscall64-abi.rst b/Documentation/powerpc/syscall64-abi.rst
index e49f69f..46caaad 100644
--- a/Documentation/powerpc/syscall64-abi.rst
+++ b/Documentation/powerpc/syscall64-abi.rst
@@ -5,6 +5,15 @@
syscall
=======
+Invocation
+----------
+The syscall is made with the sc instruction, and returns with execution
+continuing at the instruction following the sc instruction.
+
+If PPC_FEATURE2_SCV appears in the AT_HWCAP2 ELF auxiliary vector, the
+scv 0 instruction is an alternative that may provide better performance,
+with some differences to calling sequence.
+
syscall calling sequence\ [1]_ matches the Power Architecture 64-bit ELF ABI
specification C function calling sequence, including register preservation
rules, with the following differences.
@@ -12,16 +21,23 @@
.. [1] Some syscalls (typically low-level management functions) may have
different calling sequences (e.g., rt_sigreturn).
-Parameters and return value
----------------------------
+Parameters
+----------
The system call number is specified in r0.
There is a maximum of 6 integer parameters to a syscall, passed in r3-r8.
-Both a return value and a return error code are returned. cr0.SO is the return
-error code, and r3 is the return value or error code. When cr0.SO is clear,
-the syscall succeeded and r3 is the return value. When cr0.SO is set, the
-syscall failed and r3 is the error code that generally corresponds to errno.
+Return value
+------------
+- For the sc instruction, both a value and an error condition are returned.
+ cr0.SO is the error condition, and r3 is the return value. When cr0.SO is
+ clear, the syscall succeeded and r3 is the return value. When cr0.SO is set,
+ the syscall failed and r3 is the error value (that normally corresponds to
+ errno).
+
+- For the scv 0 instruction, the return value indicates failure if it is
+ -4095..-1 (i.e., it is >= -MAX_ERRNO (-4095) as an unsigned comparison),
+ in which case the error value is the negated return value.
Stack
-----
@@ -34,22 +50,23 @@
following differences:
=========== ============= ========================================
+--- For the sc instruction, differences with the ELF ABI ---
r0 Volatile (System call number.)
r3 Volatile (Parameter 1, and return value.)
r4-r8 Volatile (Parameters 2-6.)
-cr0 Volatile (cr0.SO is the return error condition)
+cr0 Volatile (cr0.SO is the return error condition.)
cr1, cr5-7 Nonvolatile
lr Nonvolatile
+
+--- For the scv 0 instruction, differences with the ELF ABI ---
+r0 Volatile (System call number.)
+r3 Volatile (Parameter 1, and return value.)
+r4-r8 Volatile (Parameters 2-6.)
=========== ============= ========================================
All floating point and vector data registers as well as control and status
registers are nonvolatile.
-Invocation
-----------
-The syscall is performed with the sc instruction, and returns with execution
-continuing at the instruction following the sc instruction.
-
Transactional Memory
--------------------
Syscall behavior can change if the processor is in transactional or suspended
@@ -75,6 +92,7 @@
returning to the caller. This case is not well defined or supported, so this
behavior should not be relied upon.
+scv 0 syscalls will always behave as PPC_FEATURE2_HTM_NOSC.
vsyscall
========
diff --git a/Documentation/scsi/advansys.rst b/Documentation/scsi/advansys.rst
index e0367e1..7ea12b1 100644
--- a/Documentation/scsi/advansys.rst
+++ b/Documentation/scsi/advansys.rst
@@ -125,7 +125,7 @@
c. klogd is started with the appropriate -c parameter
(e.g. klogd -c 8)
- This will cause printk() messages to be be displayed on the
+ This will cause printk() messages to be displayed on the
current console. Refer to the klogd(8) and syslogd(8) man pages
for details.
diff --git a/Documentation/scsi/scsi-parameters.rst b/Documentation/scsi/scsi-parameters.rst
index 9aba897..e5f68b43 100644
--- a/Documentation/scsi/scsi-parameters.rst
+++ b/Documentation/scsi/scsi-parameters.rst
@@ -94,7 +94,7 @@
(/proc/sys/dev/scsi/logging_level).
There is also a nice 'scsi_logging_level' script in the
S390-tools package, available for download at
- http://www-128.ibm.com/developerworks/linux/linux390/s390-tools-1.5.4.html
+ https://github.com/ibm-s390-tools/s390-tools/blob/master/scripts/scsi_logging_level
scsi_mod.scan= [SCSI] sync (default) scans SCSI busses as they are
discovered. async scans them in kernel threads,
diff --git a/Documentation/virt/kvm/api.rst b/Documentation/virt/kvm/api.rst
index cdfd981..eb3a131 100644
--- a/Documentation/virt/kvm/api.rst
+++ b/Documentation/virt/kvm/api.rst
@@ -2160,9 +2160,12 @@
PPC KVM_REG_PPC_MMCRA 64
PPC KVM_REG_PPC_MMCR2 64
PPC KVM_REG_PPC_MMCRS 64
+ PPC KVM_REG_PPC_MMCR3 64
PPC KVM_REG_PPC_SIAR 64
PPC KVM_REG_PPC_SDAR 64
PPC KVM_REG_PPC_SIER 64
+ PPC KVM_REG_PPC_SIER2 64
+ PPC KVM_REG_PPC_SIER3 64
PPC KVM_REG_PPC_PMC1 32
PPC KVM_REG_PPC_PMC2 32
PPC KVM_REG_PPC_PMC3 32
diff --git a/MAINTAINERS b/MAINTAINERS
index 4e497d1..9590b98 100644
--- a/MAINTAINERS
+++ b/MAINTAINERS
@@ -2309,7 +2309,7 @@
F: drivers/phy/qualcomm/
F: drivers/power/*/msm*
F: drivers/reset/reset-qcom-*
-F: drivers/scsi/ufs/ufs-qcom.*
+F: drivers/scsi/ufs/ufs-qcom*
F: drivers/spi/spi-geni-qcom.c
F: drivers/spi/spi-qcom-qspi.c
F: drivers/spi/spi-qup.c
@@ -3621,6 +3621,7 @@
M: Devesh Sharma <devesh.sharma@broadcom.com>
M: Somnath Kotur <somnath.kotur@broadcom.com>
M: Sriharsha Basavapatna <sriharsha.basavapatna@broadcom.com>
+M: Naresh Kumar PBS <nareshkumar.pbs@broadcom.com>
L: linux-rdma@vger.kernel.org
S: Supported
W: http://www.broadcom.com
@@ -13307,7 +13308,6 @@
PCI ENHANCED ERROR HANDLING (EEH) FOR POWERPC
M: Russell Currey <ruscur@russell.cc>
-M: Sam Bobroff <sbobroff@linux.ibm.com>
M: Oliver O'Halloran <oohall@gmail.com>
L: linuxppc-dev@lists.ozlabs.org
S: Supported
@@ -17085,6 +17085,14 @@
F: drivers/thermal/cpuidle_cooling.c
F: include/linux/cpu_cooling.h
+THERMAL/POWER_ALLOCATOR
+M: Lukasz Luba <lukasz.luba@arm.com>
+L: linux-pm@vger.kernel.org
+S: Maintained
+F: Documentation/driver-api/thermal/power_allocator.rst
+F: drivers/thermal/gov_power_allocator.c
+F: include/trace/events/thermal_power_allocator.h
+
THINKPAD ACPI EXTRAS DRIVER
M: Henrique de Moraes Holschuh <ibm-acpi@hmh.eng.br>
L: ibm-acpi-devel@lists.sourceforge.net
diff --git a/arch/arc/kernel/ptrace.c b/arch/arc/kernel/ptrace.c
index f49a054..8833919 100644
--- a/arch/arc/kernel/ptrace.c
+++ b/arch/arc/kernel/ptrace.c
@@ -18,88 +18,61 @@
static int genregs_get(struct task_struct *target,
const struct user_regset *regset,
- unsigned int pos, unsigned int count,
- void *kbuf, void __user *ubuf)
+ struct membuf to)
{
const struct pt_regs *ptregs = task_pt_regs(target);
const struct callee_regs *cregs = task_callee_regs(target);
- int ret = 0;
unsigned int stop_pc_val;
-#define REG_O_CHUNK(START, END, PTR) \
- if (!ret) \
- ret = user_regset_copyout(&pos, &count, &kbuf, &ubuf, PTR, \
- offsetof(struct user_regs_struct, START), \
- offsetof(struct user_regs_struct, END));
+ membuf_zero(&to, 4); // pad
+ membuf_store(&to, ptregs->bta);
+ membuf_store(&to, ptregs->lp_start);
+ membuf_store(&to, ptregs->lp_end);
+ membuf_store(&to, ptregs->lp_count);
+ membuf_store(&to, ptregs->status32);
+ membuf_store(&to, ptregs->ret);
+ membuf_store(&to, ptregs->blink);
+ membuf_store(&to, ptregs->fp);
+ membuf_store(&to, ptregs->r26); // gp
+ membuf_store(&to, ptregs->r12);
+ membuf_store(&to, ptregs->r11);
+ membuf_store(&to, ptregs->r10);
+ membuf_store(&to, ptregs->r9);
+ membuf_store(&to, ptregs->r8);
+ membuf_store(&to, ptregs->r7);
+ membuf_store(&to, ptregs->r6);
+ membuf_store(&to, ptregs->r5);
+ membuf_store(&to, ptregs->r4);
+ membuf_store(&to, ptregs->r3);
+ membuf_store(&to, ptregs->r2);
+ membuf_store(&to, ptregs->r1);
+ membuf_store(&to, ptregs->r0);
+ membuf_store(&to, ptregs->sp);
+ membuf_zero(&to, 4); // pad2
+ membuf_store(&to, cregs->r25);
+ membuf_store(&to, cregs->r24);
+ membuf_store(&to, cregs->r23);
+ membuf_store(&to, cregs->r22);
+ membuf_store(&to, cregs->r21);
+ membuf_store(&to, cregs->r20);
+ membuf_store(&to, cregs->r19);
+ membuf_store(&to, cregs->r18);
+ membuf_store(&to, cregs->r17);
+ membuf_store(&to, cregs->r16);
+ membuf_store(&to, cregs->r15);
+ membuf_store(&to, cregs->r14);
+ membuf_store(&to, cregs->r13);
+ membuf_store(&to, target->thread.fault_address); // efa
-#define REG_O_ONE(LOC, PTR) \
- if (!ret) \
- ret = user_regset_copyout(&pos, &count, &kbuf, &ubuf, PTR, \
- offsetof(struct user_regs_struct, LOC), \
- offsetof(struct user_regs_struct, LOC) + 4);
-
-#define REG_O_ZERO(LOC) \
- if (!ret) \
- ret = user_regset_copyout_zero(&pos, &count, &kbuf, &ubuf, \
- offsetof(struct user_regs_struct, LOC), \
- offsetof(struct user_regs_struct, LOC) + 4);
-
- REG_O_ZERO(pad);
- REG_O_ONE(scratch.bta, &ptregs->bta);
- REG_O_ONE(scratch.lp_start, &ptregs->lp_start);
- REG_O_ONE(scratch.lp_end, &ptregs->lp_end);
- REG_O_ONE(scratch.lp_count, &ptregs->lp_count);
- REG_O_ONE(scratch.status32, &ptregs->status32);
- REG_O_ONE(scratch.ret, &ptregs->ret);
- REG_O_ONE(scratch.blink, &ptregs->blink);
- REG_O_ONE(scratch.fp, &ptregs->fp);
- REG_O_ONE(scratch.gp, &ptregs->r26);
- REG_O_ONE(scratch.r12, &ptregs->r12);
- REG_O_ONE(scratch.r11, &ptregs->r11);
- REG_O_ONE(scratch.r10, &ptregs->r10);
- REG_O_ONE(scratch.r9, &ptregs->r9);
- REG_O_ONE(scratch.r8, &ptregs->r8);
- REG_O_ONE(scratch.r7, &ptregs->r7);
- REG_O_ONE(scratch.r6, &ptregs->r6);
- REG_O_ONE(scratch.r5, &ptregs->r5);
- REG_O_ONE(scratch.r4, &ptregs->r4);
- REG_O_ONE(scratch.r3, &ptregs->r3);
- REG_O_ONE(scratch.r2, &ptregs->r2);
- REG_O_ONE(scratch.r1, &ptregs->r1);
- REG_O_ONE(scratch.r0, &ptregs->r0);
- REG_O_ONE(scratch.sp, &ptregs->sp);
-
- REG_O_ZERO(pad2);
-
- REG_O_ONE(callee.r25, &cregs->r25);
- REG_O_ONE(callee.r24, &cregs->r24);
- REG_O_ONE(callee.r23, &cregs->r23);
- REG_O_ONE(callee.r22, &cregs->r22);
- REG_O_ONE(callee.r21, &cregs->r21);
- REG_O_ONE(callee.r20, &cregs->r20);
- REG_O_ONE(callee.r19, &cregs->r19);
- REG_O_ONE(callee.r18, &cregs->r18);
- REG_O_ONE(callee.r17, &cregs->r17);
- REG_O_ONE(callee.r16, &cregs->r16);
- REG_O_ONE(callee.r15, &cregs->r15);
- REG_O_ONE(callee.r14, &cregs->r14);
- REG_O_ONE(callee.r13, &cregs->r13);
-
- REG_O_ONE(efa, &target->thread.fault_address);
-
- if (!ret) {
- if (in_brkpt_trap(ptregs)) {
- stop_pc_val = target->thread.fault_address;
- pr_debug("\t\tstop_pc (brk-pt)\n");
- } else {
- stop_pc_val = ptregs->ret;
- pr_debug("\t\tstop_pc (others)\n");
- }
-
- REG_O_ONE(stop_pc, &stop_pc_val);
+ if (in_brkpt_trap(ptregs)) {
+ stop_pc_val = target->thread.fault_address;
+ pr_debug("\t\tstop_pc (brk-pt)\n");
+ } else {
+ stop_pc_val = ptregs->ret;
+ pr_debug("\t\tstop_pc (others)\n");
}
- return ret;
+ return membuf_store(&to, stop_pc_val); // stop_pc
}
static int genregs_set(struct task_struct *target,
@@ -184,25 +157,20 @@
#ifdef CONFIG_ISA_ARCV2
static int arcv2regs_get(struct task_struct *target,
const struct user_regset *regset,
- unsigned int pos, unsigned int count,
- void *kbuf, void __user *ubuf)
+ struct membuf to)
{
const struct pt_regs *regs = task_pt_regs(target);
- int ret, copy_sz;
if (IS_ENABLED(CONFIG_ARC_HAS_ACCL_REGS))
- copy_sz = sizeof(struct user_regs_arcv2);
- else
- copy_sz = 4; /* r30 only */
+ /*
+ * itemized copy not needed like above as layout of regs (r30,r58,r59)
+ * is exactly same in kernel (pt_regs) and userspace (user_regs_arcv2)
+ */
+ return membuf_write(&to, ®s->r30, sizeof(struct user_regs_arcv2));
- /*
- * itemized copy not needed like above as layout of regs (r30,r58,r59)
- * is exactly same in kernel (pt_regs) and userspace (user_regs_arcv2)
- */
- ret = user_regset_copyout(&pos, &count, &kbuf, &ubuf, ®s->r30,
- 0, copy_sz);
- return ret;
+ membuf_write(&to, ®s->r30, 4); /* r30 only */
+ return membuf_zero(&to, sizeof(struct user_regs_arcv2) - 4);
}
static int arcv2regs_set(struct task_struct *target,
@@ -237,7 +205,7 @@
.n = ELF_NGREG,
.size = sizeof(unsigned long),
.align = sizeof(unsigned long),
- .get = genregs_get,
+ .regset_get = genregs_get,
.set = genregs_set,
},
#ifdef CONFIG_ISA_ARCV2
@@ -246,7 +214,7 @@
.n = ELF_ARCV2REG,
.size = sizeof(unsigned long),
.align = sizeof(unsigned long),
- .get = arcv2regs_get,
+ .regset_get = arcv2regs_get,
.set = arcv2regs_set,
},
#endif
diff --git a/arch/arm/kernel/atags_parse.c b/arch/arm/kernel/atags_parse.c
index ce02f92..6c12d9f 100644
--- a/arch/arm/kernel/atags_parse.c
+++ b/arch/arm/kernel/atags_parse.c
@@ -91,8 +91,6 @@
static int __init parse_tag_ramdisk(const struct tag *tag)
{
rd_image_start = tag->u.ramdisk.start;
- rd_doload = (tag->u.ramdisk.flags & 1) == 0;
- rd_prompt = (tag->u.ramdisk.flags & 2) == 0;
if (tag->u.ramdisk.size)
rd_size = tag->u.ramdisk.size;
diff --git a/arch/arm/kernel/ptrace.c b/arch/arm/kernel/ptrace.c
index d0f7c88..2771e68 100644
--- a/arch/arm/kernel/ptrace.c
+++ b/arch/arm/kernel/ptrace.c
@@ -569,14 +569,9 @@
static int gpr_get(struct task_struct *target,
const struct user_regset *regset,
- unsigned int pos, unsigned int count,
- void *kbuf, void __user *ubuf)
+ struct membuf to)
{
- struct pt_regs *regs = task_pt_regs(target);
-
- return user_regset_copyout(&pos, &count, &kbuf, &ubuf,
- regs,
- 0, sizeof(*regs));
+ return membuf_write(&to, task_pt_regs(target), sizeof(struct pt_regs));
}
static int gpr_set(struct task_struct *target,
@@ -602,12 +597,10 @@
static int fpa_get(struct task_struct *target,
const struct user_regset *regset,
- unsigned int pos, unsigned int count,
- void *kbuf, void __user *ubuf)
+ struct membuf to)
{
- return user_regset_copyout(&pos, &count, &kbuf, &ubuf,
- &task_thread_info(target)->fpstate,
- 0, sizeof(struct user_fp));
+ return membuf_write(&to, &task_thread_info(target)->fpstate,
+ sizeof(struct user_fp));
}
static int fpa_set(struct task_struct *target,
@@ -642,41 +635,20 @@
* vfp_set() ignores this chunk
*
* 1 word for the FPSCR
- *
- * The bounds-checking logic built into user_regset_copyout and friends
- * means that we can make a simple sequence of calls to map the relevant data
- * to/from the specified slice of the user regset structure.
*/
static int vfp_get(struct task_struct *target,
const struct user_regset *regset,
- unsigned int pos, unsigned int count,
- void *kbuf, void __user *ubuf)
+ struct membuf to)
{
- int ret;
struct thread_info *thread = task_thread_info(target);
struct vfp_hard_struct const *vfp = &thread->vfpstate.hard;
- const size_t user_fpregs_offset = offsetof(struct user_vfp, fpregs);
const size_t user_fpscr_offset = offsetof(struct user_vfp, fpscr);
vfp_sync_hwstate(thread);
- ret = user_regset_copyout(&pos, &count, &kbuf, &ubuf,
- &vfp->fpregs,
- user_fpregs_offset,
- user_fpregs_offset + sizeof(vfp->fpregs));
- if (ret)
- return ret;
-
- ret = user_regset_copyout_zero(&pos, &count, &kbuf, &ubuf,
- user_fpregs_offset + sizeof(vfp->fpregs),
- user_fpscr_offset);
- if (ret)
- return ret;
-
- return user_regset_copyout(&pos, &count, &kbuf, &ubuf,
- &vfp->fpscr,
- user_fpscr_offset,
- user_fpscr_offset + sizeof(vfp->fpscr));
+ membuf_write(&to, vfp->fpregs, sizeof(vfp->fpregs));
+ membuf_zero(&to, user_fpscr_offset - sizeof(vfp->fpregs));
+ return membuf_store(&to, vfp->fpscr);
}
/*
@@ -739,7 +711,7 @@
.n = ELF_NGREG,
.size = sizeof(u32),
.align = sizeof(u32),
- .get = gpr_get,
+ .regset_get = gpr_get,
.set = gpr_set
},
[REGSET_FPR] = {
@@ -751,7 +723,7 @@
.n = sizeof(struct user_fp) / sizeof(u32),
.size = sizeof(u32),
.align = sizeof(u32),
- .get = fpa_get,
+ .regset_get = fpa_get,
.set = fpa_set
},
#ifdef CONFIG_VFP
@@ -764,7 +736,7 @@
.n = ARM_VFPREGS_SIZE / sizeof(u32),
.size = sizeof(u32),
.align = sizeof(u32),
- .get = vfp_get,
+ .regset_get = vfp_get,
.set = vfp_set
},
#endif /* CONFIG_VFP */
diff --git a/arch/arm/xen/mm.c b/arch/arm/xen/mm.c
index d40e9e5..396797f 100644
--- a/arch/arm/xen/mm.c
+++ b/arch/arm/xen/mm.c
@@ -1,5 +1,6 @@
// SPDX-License-Identifier: GPL-2.0-only
#include <linux/cpu.h>
+#include <linux/dma-direct.h>
#include <linux/dma-noncoherent.h>
#include <linux/gfp.h>
#include <linux/highmem.h>
@@ -42,15 +43,18 @@
static bool hypercall_cflush = false;
/* buffers in highmem or foreign pages cannot cross page boundaries */
-static void dma_cache_maint(dma_addr_t handle, size_t size, u32 op)
+static void dma_cache_maint(struct device *dev, dma_addr_t handle,
+ size_t size, u32 op)
{
struct gnttab_cache_flush cflush;
- cflush.a.dev_bus_addr = handle & XEN_PAGE_MASK;
cflush.offset = xen_offset_in_page(handle);
cflush.op = op;
+ handle &= XEN_PAGE_MASK;
do {
+ cflush.a.dev_bus_addr = dma_to_phys(dev, handle);
+
if (size + cflush.offset > XEN_PAGE_SIZE)
cflush.length = XEN_PAGE_SIZE - cflush.offset;
else
@@ -59,7 +63,7 @@
HYPERVISOR_grant_table_op(GNTTABOP_cache_flush, &cflush, 1);
cflush.offset = 0;
- cflush.a.dev_bus_addr += cflush.length;
+ handle += cflush.length;
size -= cflush.length;
} while (size);
}
@@ -71,24 +75,20 @@
* pfn_valid returns true the pages is local and we can use the native
* dma-direct functions, otherwise we call the Xen specific version.
*/
-void xen_dma_sync_for_cpu(dma_addr_t handle, phys_addr_t paddr, size_t size,
- enum dma_data_direction dir)
+void xen_dma_sync_for_cpu(struct device *dev, dma_addr_t handle,
+ size_t size, enum dma_data_direction dir)
{
- if (pfn_valid(PFN_DOWN(handle)))
- arch_sync_dma_for_cpu(paddr, size, dir);
- else if (dir != DMA_TO_DEVICE)
- dma_cache_maint(handle, size, GNTTAB_CACHE_INVAL);
+ if (dir != DMA_TO_DEVICE)
+ dma_cache_maint(dev, handle, size, GNTTAB_CACHE_INVAL);
}
-void xen_dma_sync_for_device(dma_addr_t handle, phys_addr_t paddr, size_t size,
- enum dma_data_direction dir)
+void xen_dma_sync_for_device(struct device *dev, dma_addr_t handle,
+ size_t size, enum dma_data_direction dir)
{
- if (pfn_valid(PFN_DOWN(handle)))
- arch_sync_dma_for_device(paddr, size, dir);
- else if (dir == DMA_FROM_DEVICE)
- dma_cache_maint(handle, size, GNTTAB_CACHE_INVAL);
+ if (dir == DMA_FROM_DEVICE)
+ dma_cache_maint(dev, handle, size, GNTTAB_CACHE_INVAL);
else
- dma_cache_maint(handle, size, GNTTAB_CACHE_CLEAN);
+ dma_cache_maint(dev, handle, size, GNTTAB_CACHE_CLEAN);
}
bool xen_arch_need_swiotlb(struct device *dev,
@@ -96,7 +96,7 @@
dma_addr_t dev_addr)
{
unsigned int xen_pfn = XEN_PFN_DOWN(phys);
- unsigned int bfn = XEN_PFN_DOWN(dev_addr);
+ unsigned int bfn = XEN_PFN_DOWN(dma_to_phys(dev, dev_addr));
/*
* The swiotlb buffer should be used if
diff --git a/arch/arm64/kernel/kaslr.c b/arch/arm64/kernel/kaslr.c
index 9ded423..b181e05 100644
--- a/arch/arm64/kernel/kaslr.c
+++ b/arch/arm64/kernel/kaslr.c
@@ -84,6 +84,7 @@
void *fdt;
u64 seed, offset, mask, module_range;
const u8 *cmdline, *str;
+ unsigned long raw;
int size;
/*
@@ -122,15 +123,12 @@
}
/*
- * Mix in any entropy obtainable architecturally, open coded
- * since this runs extremely early.
+ * Mix in any entropy obtainable architecturally if enabled
+ * and supported.
*/
- if (__early_cpu_has_rndr()) {
- unsigned long raw;
- if (__arm64_rndr(&raw))
- seed ^= raw;
- }
+ if (arch_get_random_seed_long_early(&raw))
+ seed ^= raw;
if (!seed) {
kaslr_status = KASLR_DISABLED_NO_SEED;
diff --git a/arch/arm64/kernel/ptrace.c b/arch/arm64/kernel/ptrace.c
index 1e02e98..d8ebfd8 100644
--- a/arch/arm64/kernel/ptrace.c
+++ b/arch/arm64/kernel/ptrace.c
@@ -474,11 +474,10 @@
static int hw_break_get(struct task_struct *target,
const struct user_regset *regset,
- unsigned int pos, unsigned int count,
- void *kbuf, void __user *ubuf)
+ struct membuf to)
{
unsigned int note_type = regset->core_note_type;
- int ret, idx = 0, offset, limit;
+ int ret, idx = 0;
u32 info, ctrl;
u64 addr;
@@ -487,49 +486,21 @@
if (ret)
return ret;
- ret = user_regset_copyout(&pos, &count, &kbuf, &ubuf, &info, 0,
- sizeof(info));
- if (ret)
- return ret;
-
- /* Pad */
- offset = offsetof(struct user_hwdebug_state, pad);
- ret = user_regset_copyout_zero(&pos, &count, &kbuf, &ubuf, offset,
- offset + PTRACE_HBP_PAD_SZ);
- if (ret)
- return ret;
-
+ membuf_write(&to, &info, sizeof(info));
+ membuf_zero(&to, sizeof(u32));
/* (address, ctrl) registers */
- offset = offsetof(struct user_hwdebug_state, dbg_regs);
- limit = regset->n * regset->size;
- while (count && offset < limit) {
+ while (to.left) {
ret = ptrace_hbp_get_addr(note_type, target, idx, &addr);
if (ret)
return ret;
- ret = user_regset_copyout(&pos, &count, &kbuf, &ubuf, &addr,
- offset, offset + PTRACE_HBP_ADDR_SZ);
- if (ret)
- return ret;
- offset += PTRACE_HBP_ADDR_SZ;
-
ret = ptrace_hbp_get_ctrl(note_type, target, idx, &ctrl);
if (ret)
return ret;
- ret = user_regset_copyout(&pos, &count, &kbuf, &ubuf, &ctrl,
- offset, offset + PTRACE_HBP_CTRL_SZ);
- if (ret)
- return ret;
- offset += PTRACE_HBP_CTRL_SZ;
-
- ret = user_regset_copyout_zero(&pos, &count, &kbuf, &ubuf,
- offset,
- offset + PTRACE_HBP_PAD_SZ);
- if (ret)
- return ret;
- offset += PTRACE_HBP_PAD_SZ;
+ membuf_store(&to, addr);
+ membuf_store(&to, ctrl);
+ membuf_zero(&to, sizeof(u32));
idx++;
}
-
return 0;
}
@@ -589,11 +560,10 @@
static int gpr_get(struct task_struct *target,
const struct user_regset *regset,
- unsigned int pos, unsigned int count,
- void *kbuf, void __user *ubuf)
+ struct membuf to)
{
struct user_pt_regs *uregs = &task_pt_regs(target)->user_regs;
- return user_regset_copyout(&pos, &count, &kbuf, &ubuf, uregs, 0, -1);
+ return membuf_write(&to, uregs, sizeof(*uregs));
}
static int gpr_set(struct task_struct *target, const struct user_regset *regset,
@@ -626,8 +596,7 @@
*/
static int __fpr_get(struct task_struct *target,
const struct user_regset *regset,
- unsigned int pos, unsigned int count,
- void *kbuf, void __user *ubuf, unsigned int start_pos)
+ struct membuf to)
{
struct user_fpsimd_state *uregs;
@@ -635,13 +604,11 @@
uregs = &target->thread.uw.fpsimd_state;
- return user_regset_copyout(&pos, &count, &kbuf, &ubuf, uregs,
- start_pos, start_pos + sizeof(*uregs));
+ return membuf_write(&to, uregs, sizeof(*uregs));
}
static int fpr_get(struct task_struct *target, const struct user_regset *regset,
- unsigned int pos, unsigned int count,
- void *kbuf, void __user *ubuf)
+ struct membuf to)
{
if (!system_supports_fpsimd())
return -EINVAL;
@@ -649,7 +616,7 @@
if (target == current)
fpsimd_preserve_current_state();
- return __fpr_get(target, regset, pos, count, kbuf, ubuf, 0);
+ return __fpr_get(target, regset, to);
}
static int __fpr_set(struct task_struct *target,
@@ -699,15 +666,12 @@
}
static int tls_get(struct task_struct *target, const struct user_regset *regset,
- unsigned int pos, unsigned int count,
- void *kbuf, void __user *ubuf)
+ struct membuf to)
{
- unsigned long *tls = &target->thread.uw.tp_value;
-
if (target == current)
tls_preserve_current_state();
- return user_regset_copyout(&pos, &count, &kbuf, &ubuf, tls, 0, -1);
+ return membuf_store(&to, target->thread.uw.tp_value);
}
static int tls_set(struct task_struct *target, const struct user_regset *regset,
@@ -727,13 +691,9 @@
static int system_call_get(struct task_struct *target,
const struct user_regset *regset,
- unsigned int pos, unsigned int count,
- void *kbuf, void __user *ubuf)
+ struct membuf to)
{
- int syscallno = task_pt_regs(target)->syscallno;
-
- return user_regset_copyout(&pos, &count, &kbuf, &ubuf,
- &syscallno, 0, -1);
+ return membuf_store(&to, task_pt_regs(target)->syscallno);
}
static int system_call_set(struct task_struct *target,
@@ -780,24 +740,10 @@
return ALIGN(header->size, SVE_VQ_BYTES);
}
-static unsigned int sve_get_size(struct task_struct *target,
- const struct user_regset *regset)
-{
- struct user_sve_header header;
-
- if (!system_supports_sve())
- return 0;
-
- sve_init_header_from_task(&header, target);
- return sve_size_from_header(&header);
-}
-
static int sve_get(struct task_struct *target,
const struct user_regset *regset,
- unsigned int pos, unsigned int count,
- void *kbuf, void __user *ubuf)
+ struct membuf to)
{
- int ret;
struct user_sve_header header;
unsigned int vq;
unsigned long start, end;
@@ -809,10 +755,7 @@
sve_init_header_from_task(&header, target);
vq = sve_vq_from_vl(header.vl);
- ret = user_regset_copyout(&pos, &count, &kbuf, &ubuf, &header,
- 0, sizeof(header));
- if (ret)
- return ret;
+ membuf_write(&to, &header, sizeof(header));
if (target == current)
fpsimd_preserve_current_state();
@@ -821,26 +764,18 @@
BUILD_BUG_ON(SVE_PT_FPSIMD_OFFSET != sizeof(header));
if ((header.flags & SVE_PT_REGS_MASK) == SVE_PT_REGS_FPSIMD)
- return __fpr_get(target, regset, pos, count, kbuf, ubuf,
- SVE_PT_FPSIMD_OFFSET);
+ return __fpr_get(target, regset, to);
/* Otherwise: full SVE case */
BUILD_BUG_ON(SVE_PT_SVE_OFFSET != sizeof(header));
start = SVE_PT_SVE_OFFSET;
end = SVE_PT_SVE_FFR_OFFSET(vq) + SVE_PT_SVE_FFR_SIZE(vq);
- ret = user_regset_copyout(&pos, &count, &kbuf, &ubuf,
- target->thread.sve_state,
- start, end);
- if (ret)
- return ret;
+ membuf_write(&to, target->thread.sve_state, end - start);
start = end;
end = SVE_PT_SVE_FPSR_OFFSET(vq);
- ret = user_regset_copyout_zero(&pos, &count, &kbuf, &ubuf,
- start, end);
- if (ret)
- return ret;
+ membuf_zero(&to, end - start);
/*
* Copy fpsr, and fpcr which must follow contiguously in
@@ -848,16 +783,11 @@
*/
start = end;
end = SVE_PT_SVE_FPCR_OFFSET(vq) + SVE_PT_SVE_FPCR_SIZE;
- ret = user_regset_copyout(&pos, &count, &kbuf, &ubuf,
- &target->thread.uw.fpsimd_state.fpsr,
- start, end);
- if (ret)
- return ret;
+ membuf_write(&to, &target->thread.uw.fpsimd_state.fpsr, end - start);
start = end;
end = sve_size_from_header(&header);
- return user_regset_copyout_zero(&pos, &count, &kbuf, &ubuf,
- start, end);
+ return membuf_zero(&to, end - start);
}
static int sve_set(struct task_struct *target,
@@ -961,8 +891,7 @@
#ifdef CONFIG_ARM64_PTR_AUTH
static int pac_mask_get(struct task_struct *target,
const struct user_regset *regset,
- unsigned int pos, unsigned int count,
- void *kbuf, void __user *ubuf)
+ struct membuf to)
{
/*
* The PAC bits can differ across data and instruction pointers
@@ -978,7 +907,7 @@
if (!system_supports_address_auth())
return -EINVAL;
- return user_regset_copyout(&pos, &count, &kbuf, &ubuf, &uregs, 0, -1);
+ return membuf_write(&to, &uregs, sizeof(uregs));
}
#ifdef CONFIG_CHECKPOINT_RESTORE
@@ -1017,8 +946,7 @@
static int pac_address_keys_get(struct task_struct *target,
const struct user_regset *regset,
- unsigned int pos, unsigned int count,
- void *kbuf, void __user *ubuf)
+ struct membuf to)
{
struct ptrauth_keys_user *keys = &target->thread.keys_user;
struct user_pac_address_keys user_keys;
@@ -1028,8 +956,7 @@
pac_address_keys_to_user(&user_keys, keys);
- return user_regset_copyout(&pos, &count, &kbuf, &ubuf,
- &user_keys, 0, -1);
+ return membuf_write(&to, &user_keys, sizeof(user_keys));
}
static int pac_address_keys_set(struct task_struct *target,
@@ -1068,8 +995,7 @@
static int pac_generic_keys_get(struct task_struct *target,
const struct user_regset *regset,
- unsigned int pos, unsigned int count,
- void *kbuf, void __user *ubuf)
+ struct membuf to)
{
struct ptrauth_keys_user *keys = &target->thread.keys_user;
struct user_pac_generic_keys user_keys;
@@ -1079,8 +1005,7 @@
pac_generic_keys_to_user(&user_keys, keys);
- return user_regset_copyout(&pos, &count, &kbuf, &ubuf,
- &user_keys, 0, -1);
+ return membuf_write(&to, &user_keys, sizeof(user_keys));
}
static int pac_generic_keys_set(struct task_struct *target,
@@ -1134,7 +1059,7 @@
.n = sizeof(struct user_pt_regs) / sizeof(u64),
.size = sizeof(u64),
.align = sizeof(u64),
- .get = gpr_get,
+ .regset_get = gpr_get,
.set = gpr_set
},
[REGSET_FPR] = {
@@ -1147,7 +1072,7 @@
.size = sizeof(u32),
.align = sizeof(u32),
.active = fpr_active,
- .get = fpr_get,
+ .regset_get = fpr_get,
.set = fpr_set
},
[REGSET_TLS] = {
@@ -1155,7 +1080,7 @@
.n = 1,
.size = sizeof(void *),
.align = sizeof(void *),
- .get = tls_get,
+ .regset_get = tls_get,
.set = tls_set,
},
#ifdef CONFIG_HAVE_HW_BREAKPOINT
@@ -1164,7 +1089,7 @@
.n = sizeof(struct user_hwdebug_state) / sizeof(u32),
.size = sizeof(u32),
.align = sizeof(u32),
- .get = hw_break_get,
+ .regset_get = hw_break_get,
.set = hw_break_set,
},
[REGSET_HW_WATCH] = {
@@ -1172,7 +1097,7 @@
.n = sizeof(struct user_hwdebug_state) / sizeof(u32),
.size = sizeof(u32),
.align = sizeof(u32),
- .get = hw_break_get,
+ .regset_get = hw_break_get,
.set = hw_break_set,
},
#endif
@@ -1181,7 +1106,7 @@
.n = 1,
.size = sizeof(int),
.align = sizeof(int),
- .get = system_call_get,
+ .regset_get = system_call_get,
.set = system_call_set,
},
#ifdef CONFIG_ARM64_SVE
@@ -1191,9 +1116,8 @@
SVE_VQ_BYTES),
.size = SVE_VQ_BYTES,
.align = SVE_VQ_BYTES,
- .get = sve_get,
+ .regset_get = sve_get,
.set = sve_set,
- .get_size = sve_get_size,
},
#endif
#ifdef CONFIG_ARM64_PTR_AUTH
@@ -1202,7 +1126,7 @@
.n = sizeof(struct user_pac_mask) / sizeof(u64),
.size = sizeof(u64),
.align = sizeof(u64),
- .get = pac_mask_get,
+ .regset_get = pac_mask_get,
/* this cannot be set dynamically */
},
#ifdef CONFIG_CHECKPOINT_RESTORE
@@ -1211,7 +1135,7 @@
.n = sizeof(struct user_pac_address_keys) / sizeof(__uint128_t),
.size = sizeof(__uint128_t),
.align = sizeof(__uint128_t),
- .get = pac_address_keys_get,
+ .regset_get = pac_address_keys_get,
.set = pac_address_keys_set,
},
[REGSET_PACG_KEYS] = {
@@ -1219,7 +1143,7 @@
.n = sizeof(struct user_pac_generic_keys) / sizeof(__uint128_t),
.size = sizeof(__uint128_t),
.align = sizeof(__uint128_t),
- .get = pac_generic_keys_get,
+ .regset_get = pac_generic_keys_get,
.set = pac_generic_keys_set,
},
#endif
@@ -1237,57 +1161,31 @@
REGSET_COMPAT_VFP,
};
+static inline compat_ulong_t compat_get_user_reg(struct task_struct *task, int idx)
+{
+ struct pt_regs *regs = task_pt_regs(task);
+
+ switch (idx) {
+ case 15:
+ return regs->pc;
+ case 16:
+ return pstate_to_compat_psr(regs->pstate);
+ case 17:
+ return regs->orig_x0;
+ default:
+ return regs->regs[idx];
+ }
+}
+
static int compat_gpr_get(struct task_struct *target,
const struct user_regset *regset,
- unsigned int pos, unsigned int count,
- void *kbuf, void __user *ubuf)
+ struct membuf to)
{
- int ret = 0;
- unsigned int i, start, num_regs;
+ int i = 0;
- /* Calculate the number of AArch32 registers contained in count */
- num_regs = count / regset->size;
-
- /* Convert pos into an register number */
- start = pos / regset->size;
-
- if (start + num_regs > regset->n)
- return -EIO;
-
- for (i = 0; i < num_regs; ++i) {
- unsigned int idx = start + i;
- compat_ulong_t reg;
-
- switch (idx) {
- case 15:
- reg = task_pt_regs(target)->pc;
- break;
- case 16:
- reg = task_pt_regs(target)->pstate;
- reg = pstate_to_compat_psr(reg);
- break;
- case 17:
- reg = task_pt_regs(target)->orig_x0;
- break;
- default:
- reg = task_pt_regs(target)->regs[idx];
- }
-
- if (kbuf) {
- memcpy(kbuf, ®, sizeof(reg));
- kbuf += sizeof(reg);
- } else {
- ret = copy_to_user(ubuf, ®, sizeof(reg));
- if (ret) {
- ret = -EFAULT;
- break;
- }
-
- ubuf += sizeof(reg);
- }
- }
-
- return ret;
+ while (to.left)
+ membuf_store(&to, compat_get_user_reg(target, i++));
+ return 0;
}
static int compat_gpr_set(struct task_struct *target,
@@ -1354,12 +1252,10 @@
static int compat_vfp_get(struct task_struct *target,
const struct user_regset *regset,
- unsigned int pos, unsigned int count,
- void *kbuf, void __user *ubuf)
+ struct membuf to)
{
struct user_fpsimd_state *uregs;
compat_ulong_t fpscr;
- int ret, vregs_end_pos;
if (!system_supports_fpsimd())
return -EINVAL;
@@ -1373,19 +1269,10 @@
* The VFP registers are packed into the fpsimd_state, so they all sit
* nicely together for us. We just need to create the fpscr separately.
*/
- vregs_end_pos = VFP_STATE_SIZE - sizeof(compat_ulong_t);
- ret = user_regset_copyout(&pos, &count, &kbuf, &ubuf, uregs,
- 0, vregs_end_pos);
-
- if (count && !ret) {
- fpscr = (uregs->fpsr & VFP_FPSCR_STAT_MASK) |
- (uregs->fpcr & VFP_FPSCR_CTRL_MASK);
-
- ret = user_regset_copyout(&pos, &count, &kbuf, &ubuf, &fpscr,
- vregs_end_pos, VFP_STATE_SIZE);
- }
-
- return ret;
+ membuf_write(&to, uregs, VFP_STATE_SIZE - sizeof(compat_ulong_t));
+ fpscr = (uregs->fpsr & VFP_FPSCR_STAT_MASK) |
+ (uregs->fpcr & VFP_FPSCR_CTRL_MASK);
+ return membuf_store(&to, fpscr);
}
static int compat_vfp_set(struct task_struct *target,
@@ -1420,11 +1307,10 @@
}
static int compat_tls_get(struct task_struct *target,
- const struct user_regset *regset, unsigned int pos,
- unsigned int count, void *kbuf, void __user *ubuf)
+ const struct user_regset *regset,
+ struct membuf to)
{
- compat_ulong_t tls = (compat_ulong_t)target->thread.uw.tp_value;
- return user_regset_copyout(&pos, &count, &kbuf, &ubuf, &tls, 0, -1);
+ return membuf_store(&to, (compat_ulong_t)target->thread.uw.tp_value);
}
static int compat_tls_set(struct task_struct *target,
@@ -1449,7 +1335,7 @@
.n = COMPAT_ELF_NGREG,
.size = sizeof(compat_elf_greg_t),
.align = sizeof(compat_elf_greg_t),
- .get = compat_gpr_get,
+ .regset_get = compat_gpr_get,
.set = compat_gpr_set
},
[REGSET_COMPAT_VFP] = {
@@ -1458,7 +1344,7 @@
.size = sizeof(compat_ulong_t),
.align = sizeof(compat_ulong_t),
.active = fpr_active,
- .get = compat_vfp_get,
+ .regset_get = compat_vfp_get,
.set = compat_vfp_set
},
};
@@ -1474,7 +1360,7 @@
.n = COMPAT_ELF_NGREG,
.size = sizeof(compat_elf_greg_t),
.align = sizeof(compat_elf_greg_t),
- .get = compat_gpr_get,
+ .regset_get = compat_gpr_get,
.set = compat_gpr_set
},
[REGSET_FPR] = {
@@ -1482,7 +1368,7 @@
.n = VFP_STATE_SIZE / sizeof(compat_ulong_t),
.size = sizeof(compat_ulong_t),
.align = sizeof(compat_ulong_t),
- .get = compat_vfp_get,
+ .regset_get = compat_vfp_get,
.set = compat_vfp_set
},
[REGSET_TLS] = {
@@ -1490,7 +1376,7 @@
.n = 1,
.size = sizeof(compat_ulong_t),
.align = sizeof(compat_ulong_t),
- .get = compat_tls_get,
+ .regset_get = compat_tls_get,
.set = compat_tls_set,
},
#ifdef CONFIG_HAVE_HW_BREAKPOINT
@@ -1499,7 +1385,7 @@
.n = sizeof(struct user_hwdebug_state) / sizeof(u32),
.size = sizeof(u32),
.align = sizeof(u32),
- .get = hw_break_get,
+ .regset_get = hw_break_get,
.set = hw_break_set,
},
[REGSET_HW_WATCH] = {
@@ -1507,7 +1393,7 @@
.n = sizeof(struct user_hwdebug_state) / sizeof(u32),
.size = sizeof(u32),
.align = sizeof(u32),
- .get = hw_break_get,
+ .regset_get = hw_break_get,
.set = hw_break_set,
},
#endif
@@ -1516,7 +1402,7 @@
.n = 1,
.size = sizeof(int),
.align = sizeof(int),
- .get = system_call_get,
+ .regset_get = system_call_get,
.set = system_call_set,
},
};
@@ -1541,9 +1427,7 @@
else if (off == COMPAT_PT_TEXT_END_ADDR)
tmp = tsk->mm->end_code;
else if (off < sizeof(compat_elf_gregset_t))
- return copy_regset_to_user(tsk, &user_aarch32_view,
- REGSET_COMPAT_GPR, off,
- sizeof(compat_ulong_t), ret);
+ tmp = compat_get_user_reg(tsk, off >> 2);
else if (off >= COMPAT_USER_SZ)
return -EIO;
else
@@ -1555,8 +1439,8 @@
static int compat_ptrace_write_user(struct task_struct *tsk, compat_ulong_t off,
compat_ulong_t val)
{
- int ret;
- mm_segment_t old_fs = get_fs();
+ struct pt_regs newregs = *task_pt_regs(tsk);
+ unsigned int idx = off / 4;
if (off & 3 || off >= COMPAT_USER_SZ)
return -EIO;
@@ -1564,14 +1448,25 @@
if (off >= sizeof(compat_elf_gregset_t))
return 0;
- set_fs(KERNEL_DS);
- ret = copy_regset_from_user(tsk, &user_aarch32_view,
- REGSET_COMPAT_GPR, off,
- sizeof(compat_ulong_t),
- &val);
- set_fs(old_fs);
+ switch (idx) {
+ case 15:
+ newregs.pc = val;
+ break;
+ case 16:
+ newregs.pstate = compat_psr_to_pstate(val);
+ break;
+ case 17:
+ newregs.orig_x0 = val;
+ break;
+ default:
+ newregs.regs[idx] = val;
+ }
- return ret;
+ if (!valid_user_regs(&newregs.user_regs, tsk))
+ return -EINVAL;
+
+ *task_pt_regs(tsk) = newregs;
+ return 0;
}
#ifdef CONFIG_HAVE_HW_BREAKPOINT
diff --git a/arch/c6x/kernel/ptrace.c b/arch/c6x/kernel/ptrace.c
index 67af156..3cdaa8c 100644
--- a/arch/c6x/kernel/ptrace.c
+++ b/arch/c6x/kernel/ptrace.c
@@ -57,14 +57,9 @@
static int gpr_get(struct task_struct *target,
const struct user_regset *regset,
- unsigned int pos, unsigned int count,
- void *kbuf, void __user *ubuf)
+ struct membuf to)
{
- struct pt_regs *regs = task_pt_regs(target);
-
- return user_regset_copyout(&pos, &count, &kbuf, &ubuf,
- regs,
- 0, sizeof(*regs));
+ return membuf_write(&to, task_pt_regs(target), sizeof(struct pt_regs));
}
enum c6x_regset {
@@ -77,7 +72,7 @@
.n = ELF_NGREG,
.size = sizeof(u32),
.align = sizeof(u32),
- .get = gpr_get,
+ .regset_get = gpr_get,
},
};
diff --git a/arch/csky/kernel/ptrace.c b/arch/csky/kernel/ptrace.c
index b06612c..d822144 100644
--- a/arch/csky/kernel/ptrace.c
+++ b/arch/csky/kernel/ptrace.c
@@ -76,17 +76,14 @@
static int gpr_get(struct task_struct *target,
const struct user_regset *regset,
- unsigned int pos, unsigned int count,
- void *kbuf, void __user *ubuf)
+ struct membuf to)
{
- struct pt_regs *regs;
-
- regs = task_pt_regs(target);
+ struct pt_regs *regs = task_pt_regs(target);
/* Abiv1 regs->tls is fake and we need sync here. */
regs->tls = task_thread_info(target)->tp_value;
- return user_regset_copyout(&pos, &count, &kbuf, &ubuf, regs, 0, -1);
+ return membuf_write(&to, regs, sizeof(regs));
}
static int gpr_set(struct task_struct *target,
@@ -114,8 +111,7 @@
static int fpr_get(struct task_struct *target,
const struct user_regset *regset,
- unsigned int pos, unsigned int count,
- void *kbuf, void __user *ubuf)
+ struct membuf to)
{
struct user_fp *regs = (struct user_fp *)&target->thread.user_fp;
@@ -131,9 +127,9 @@
for (i = 0; i < 32; i++)
tmp.vr[64 + i] = regs->vr[32 + i];
- return user_regset_copyout(&pos, &count, &kbuf, &ubuf, &tmp, 0, -1);
+ return membuf_write(&to, &tmp, sizeof(tmp));
#else
- return user_regset_copyout(&pos, &count, &kbuf, &ubuf, regs, 0, -1);
+ return membuf_write(&to, regs, sizeof(*regs));
#endif
}
@@ -173,16 +169,16 @@
.n = sizeof(struct pt_regs) / sizeof(u32),
.size = sizeof(u32),
.align = sizeof(u32),
- .get = &gpr_get,
- .set = &gpr_set,
+ .regset_get = gpr_get,
+ .set = gpr_set,
},
[REGSET_FPR] = {
.core_note_type = NT_PRFPREG,
.n = sizeof(struct user_fp) / sizeof(u32),
.size = sizeof(u32),
.align = sizeof(u32),
- .get = &fpr_get,
- .set = &fpr_set,
+ .regset_get = fpr_get,
+ .set = fpr_set,
},
};
diff --git a/arch/h8300/kernel/ptrace.c b/arch/h8300/kernel/ptrace.c
index 0dc1c8f..a11db00 100644
--- a/arch/h8300/kernel/ptrace.c
+++ b/arch/h8300/kernel/ptrace.c
@@ -87,20 +87,15 @@
static int regs_get(struct task_struct *target,
const struct user_regset *regset,
- unsigned int pos, unsigned int count,
- void *kbuf, void __user *ubuf)
+ struct membuf to)
{
int r;
- struct user_regs_struct regs;
- long *reg = (long *)®s;
- /* build user regs in buffer */
- BUILD_BUG_ON(sizeof(regs) % sizeof(long) != 0);
- for (r = 0; r < sizeof(regs) / sizeof(long); r++)
- *reg++ = h8300_get_reg(target, r);
+ BUILD_BUG_ON(sizeof(struct user_regs_struct) % sizeof(long) != 0);
+ for (r = 0; r < ELF_NGREG; r++)
+ membuf_store(&to, h8300_get_reg(target, r));
- return user_regset_copyout(&pos, &count, &kbuf, &ubuf,
- ®s, 0, sizeof(regs));
+ return 0;
}
static int regs_set(struct task_struct *target,
@@ -139,7 +134,7 @@
.n = ELF_NGREG,
.size = sizeof(long),
.align = sizeof(long),
- .get = regs_get,
+ .regset_get = regs_get,
.set = regs_set,
},
};
diff --git a/arch/hexagon/kernel/ptrace.c b/arch/hexagon/kernel/ptrace.c
index dcbf7ea..a5a89e9 100644
--- a/arch/hexagon/kernel/ptrace.c
+++ b/arch/hexagon/kernel/ptrace.c
@@ -35,58 +35,38 @@
static int genregs_get(struct task_struct *target,
const struct user_regset *regset,
- unsigned int pos, unsigned int count,
- void *kbuf, void __user *ubuf)
+ srtuct membuf to)
{
- int ret;
- unsigned int dummy;
struct pt_regs *regs = task_pt_regs(target);
-
- if (!regs)
- return -EIO;
-
/* The general idea here is that the copyout must happen in
* exactly the same order in which the userspace expects these
* regs. Now, the sequence in userspace does not match the
* sequence in the kernel, so everything past the 32 gprs
* happens one at a time.
*/
- ret = user_regset_copyout(&pos, &count, &kbuf, &ubuf,
- ®s->r00, 0, 32*sizeof(unsigned long));
-
-#define ONEXT(KPT_REG, USR_REG) \
- if (!ret) \
- ret = user_regset_copyout(&pos, &count, &kbuf, &ubuf, \
- KPT_REG, offsetof(struct user_regs_struct, USR_REG), \
- offsetof(struct user_regs_struct, USR_REG) + \
- sizeof(unsigned long));
-
+ membuf_write(&to, ®s->r00, 32*sizeof(unsigned long));
/* Must be exactly same sequence as struct user_regs_struct */
- ONEXT(®s->sa0, sa0);
- ONEXT(®s->lc0, lc0);
- ONEXT(®s->sa1, sa1);
- ONEXT(®s->lc1, lc1);
- ONEXT(®s->m0, m0);
- ONEXT(®s->m1, m1);
- ONEXT(®s->usr, usr);
- ONEXT(®s->preds, p3_0);
- ONEXT(®s->gp, gp);
- ONEXT(®s->ugp, ugp);
- ONEXT(&pt_elr(regs), pc);
- dummy = pt_cause(regs);
- ONEXT(&dummy, cause);
- ONEXT(&pt_badva(regs), badva);
+ membuf_store(&to, regs->sa0);
+ membuf_store(&to, regs->lc0);
+ membuf_store(&to, regs->sa1);
+ membuf_store(&to, regs->lc1);
+ membuf_store(&to, regs->m0);
+ membuf_store(&to, regs->m1);
+ membuf_store(&to, regs->usr);
+ membuf_store(&to, regs->p3_0);
+ membuf_store(&to, regs->gp);
+ membuf_store(&to, regs->ugp);
+ membuf_store(&to, pt_elr(regs)); // pc
+ membuf_store(&to, (unsigned long)pt_cause(regs)); // cause
+ membuf_store(&to, pt_badva(regs)); // badva
#if CONFIG_HEXAGON_ARCH_VERSION >=4
- ONEXT(®s->cs0, cs0);
- ONEXT(®s->cs1, cs1);
+ membuf_store(&to, regs->cs0);
+ membuf_store(&to, regs->cs1);
+ return membuf_zero(&to, sizeof(unsigned long));
+#else
+ return membuf_zero(&to, 3 * sizeof(unsigned long));
#endif
-
- /* Pad the rest with zeros, if needed */
- if (!ret)
- ret = user_regset_copyout_zero(&pos, &count, &kbuf, &ubuf,
- offsetof(struct user_regs_struct, pad1), -1);
- return ret;
}
static int genregs_set(struct task_struct *target,
@@ -159,7 +139,7 @@
.n = ELF_NGREG,
.size = sizeof(unsigned long),
.align = sizeof(unsigned long),
- .get = genregs_get,
+ .regset_get = genregs_get,
.set = genregs_set,
},
};
diff --git a/arch/ia64/kernel/ptrace.c b/arch/ia64/kernel/ptrace.c
index 82aaacf..33ca9fa 100644
--- a/arch/ia64/kernel/ptrace.c
+++ b/arch/ia64/kernel/ptrace.c
@@ -1273,52 +1273,43 @@
int ret;
};
+static const ptrdiff_t pt_offsets[32] =
+{
+#define R(n) offsetof(struct pt_regs, r##n)
+ [0] = -1, R(1), R(2), R(3),
+ [4] = -1, [5] = -1, [6] = -1, [7] = -1,
+ R(8), R(9), R(10), R(11), R(12), R(13), R(14), R(15),
+ R(16), R(17), R(18), R(19), R(20), R(21), R(22), R(23),
+ R(24), R(25), R(26), R(27), R(28), R(29), R(30), R(31),
+#undef R
+};
+
static int
access_elf_gpreg(struct task_struct *target, struct unw_frame_info *info,
unsigned long addr, unsigned long *data, int write_access)
{
- struct pt_regs *pt;
- unsigned long *ptr = NULL;
- int ret;
- char nat = 0;
+ struct pt_regs *pt = task_pt_regs(target);
+ unsigned reg = addr / sizeof(unsigned long);
+ ptrdiff_t d = pt_offsets[reg];
- pt = task_pt_regs(target);
- switch (addr) {
- case ELF_GR_OFFSET(1):
- ptr = &pt->r1;
- break;
- case ELF_GR_OFFSET(2):
- case ELF_GR_OFFSET(3):
- ptr = (void *)&pt->r2 + (addr - ELF_GR_OFFSET(2));
- break;
- case ELF_GR_OFFSET(4) ... ELF_GR_OFFSET(7):
+ if (d >= 0) {
+ unsigned long *ptr = (void *)pt + d;
+ if (write_access)
+ *ptr = *data;
+ else
+ *data = *ptr;
+ return 0;
+ } else {
+ char nat = 0;
if (write_access) {
/* read NaT bit first: */
unsigned long dummy;
-
- ret = unw_get_gr(info, addr/8, &dummy, &nat);
+ int ret = unw_get_gr(info, reg, &dummy, &nat);
if (ret < 0)
return ret;
}
- return unw_access_gr(info, addr/8, data, &nat, write_access);
- case ELF_GR_OFFSET(8) ... ELF_GR_OFFSET(11):
- ptr = (void *)&pt->r8 + addr - ELF_GR_OFFSET(8);
- break;
- case ELF_GR_OFFSET(12):
- case ELF_GR_OFFSET(13):
- ptr = (void *)&pt->r12 + addr - ELF_GR_OFFSET(12);
- break;
- case ELF_GR_OFFSET(14):
- ptr = &pt->r14;
- break;
- case ELF_GR_OFFSET(15):
- ptr = &pt->r15;
+ return unw_access_gr(info, reg, data, &nat, write_access);
}
- if (write_access)
- *ptr = *data;
- else
- *data = *ptr;
- return 0;
}
static int
@@ -1490,7 +1481,7 @@
access_elf_reg(struct task_struct *target, struct unw_frame_info *info,
unsigned long addr, unsigned long *data, int write_access)
{
- if (addr >= ELF_GR_OFFSET(1) && addr <= ELF_GR_OFFSET(15))
+ if (addr >= ELF_GR_OFFSET(1) && addr <= ELF_GR_OFFSET(31))
return access_elf_gpreg(target, info, addr, data, write_access);
else if (addr >= ELF_BR_OFFSET(0) && addr <= ELF_BR_OFFSET(7))
return access_elf_breg(target, info, addr, data, write_access);
@@ -1498,12 +1489,17 @@
return access_elf_areg(target, info, addr, data, write_access);
}
+struct regset_membuf {
+ struct membuf to;
+ int ret;
+};
+
void do_gpregs_get(struct unw_frame_info *info, void *arg)
{
- struct pt_regs *pt;
- struct regset_getset *dst = arg;
- elf_greg_t tmp[16];
- unsigned int i, index, min_copy;
+ struct regset_membuf *dst = arg;
+ struct membuf to = dst->to;
+ unsigned int n;
+ elf_greg_t reg;
if (unw_unwind_to_user(info) < 0)
return;
@@ -1521,165 +1517,53 @@
/* Skip r0 */
- if (dst->count > 0 && dst->pos < ELF_GR_OFFSET(1)) {
- dst->ret = user_regset_copyout_zero(&dst->pos, &dst->count,
- &dst->u.get.kbuf,
- &dst->u.get.ubuf,
- 0, ELF_GR_OFFSET(1));
- if (dst->ret || dst->count == 0)
+ membuf_zero(&to, 8);
+ for (n = 8; to.left && n < ELF_AR_END_OFFSET; n += 8) {
+ if (access_elf_reg(info->task, info, n, ®, 0) < 0) {
+ dst->ret = -EIO;
return;
- }
-
- /* gr1 - gr15 */
- if (dst->count > 0 && dst->pos < ELF_GR_OFFSET(16)) {
- index = (dst->pos - ELF_GR_OFFSET(1)) / sizeof(elf_greg_t);
- min_copy = ELF_GR_OFFSET(16) > (dst->pos + dst->count) ?
- (dst->pos + dst->count) : ELF_GR_OFFSET(16);
- for (i = dst->pos; i < min_copy; i += sizeof(elf_greg_t),
- index++)
- if (access_elf_reg(dst->target, info, i,
- &tmp[index], 0) < 0) {
- dst->ret = -EIO;
- return;
- }
- dst->ret = user_regset_copyout(&dst->pos, &dst->count,
- &dst->u.get.kbuf, &dst->u.get.ubuf, tmp,
- ELF_GR_OFFSET(1), ELF_GR_OFFSET(16));
- if (dst->ret || dst->count == 0)
- return;
- }
-
- /* r16-r31 */
- if (dst->count > 0 && dst->pos < ELF_NAT_OFFSET) {
- pt = task_pt_regs(dst->target);
- dst->ret = user_regset_copyout(&dst->pos, &dst->count,
- &dst->u.get.kbuf, &dst->u.get.ubuf, &pt->r16,
- ELF_GR_OFFSET(16), ELF_NAT_OFFSET);
- if (dst->ret || dst->count == 0)
- return;
- }
-
- /* nat, pr, b0 - b7 */
- if (dst->count > 0 && dst->pos < ELF_CR_IIP_OFFSET) {
- index = (dst->pos - ELF_NAT_OFFSET) / sizeof(elf_greg_t);
- min_copy = ELF_CR_IIP_OFFSET > (dst->pos + dst->count) ?
- (dst->pos + dst->count) : ELF_CR_IIP_OFFSET;
- for (i = dst->pos; i < min_copy; i += sizeof(elf_greg_t),
- index++)
- if (access_elf_reg(dst->target, info, i,
- &tmp[index], 0) < 0) {
- dst->ret = -EIO;
- return;
- }
- dst->ret = user_regset_copyout(&dst->pos, &dst->count,
- &dst->u.get.kbuf, &dst->u.get.ubuf, tmp,
- ELF_NAT_OFFSET, ELF_CR_IIP_OFFSET);
- if (dst->ret || dst->count == 0)
- return;
- }
-
- /* ip cfm psr ar.rsc ar.bsp ar.bspstore ar.rnat
- * ar.ccv ar.unat ar.fpsr ar.pfs ar.lc ar.ec ar.csd ar.ssd
- */
- if (dst->count > 0 && dst->pos < (ELF_AR_END_OFFSET)) {
- index = (dst->pos - ELF_CR_IIP_OFFSET) / sizeof(elf_greg_t);
- min_copy = ELF_AR_END_OFFSET > (dst->pos + dst->count) ?
- (dst->pos + dst->count) : ELF_AR_END_OFFSET;
- for (i = dst->pos; i < min_copy; i += sizeof(elf_greg_t),
- index++)
- if (access_elf_reg(dst->target, info, i,
- &tmp[index], 0) < 0) {
- dst->ret = -EIO;
- return;
- }
- dst->ret = user_regset_copyout(&dst->pos, &dst->count,
- &dst->u.get.kbuf, &dst->u.get.ubuf, tmp,
- ELF_CR_IIP_OFFSET, ELF_AR_END_OFFSET);
+ }
+ membuf_store(&to, reg);
}
}
void do_gpregs_set(struct unw_frame_info *info, void *arg)
{
- struct pt_regs *pt;
struct regset_getset *dst = arg;
- elf_greg_t tmp[16];
- unsigned int i, index;
if (unw_unwind_to_user(info) < 0)
return;
+ if (!dst->count)
+ return;
/* Skip r0 */
- if (dst->count > 0 && dst->pos < ELF_GR_OFFSET(1)) {
+ if (dst->pos < ELF_GR_OFFSET(1)) {
dst->ret = user_regset_copyin_ignore(&dst->pos, &dst->count,
&dst->u.set.kbuf,
&dst->u.set.ubuf,
0, ELF_GR_OFFSET(1));
- if (dst->ret || dst->count == 0)
- return;
- }
-
- /* gr1-gr15 */
- if (dst->count > 0 && dst->pos < ELF_GR_OFFSET(16)) {
- i = dst->pos;
- index = (dst->pos - ELF_GR_OFFSET(1)) / sizeof(elf_greg_t);
- dst->ret = user_regset_copyin(&dst->pos, &dst->count,
- &dst->u.set.kbuf, &dst->u.set.ubuf, tmp,
- ELF_GR_OFFSET(1), ELF_GR_OFFSET(16));
if (dst->ret)
return;
- for ( ; i < dst->pos; i += sizeof(elf_greg_t), index++)
- if (access_elf_reg(dst->target, info, i,
- &tmp[index], 1) < 0) {
- dst->ret = -EIO;
- return;
- }
- if (dst->count == 0)
- return;
}
- /* gr16-gr31 */
- if (dst->count > 0 && dst->pos < ELF_NAT_OFFSET) {
- pt = task_pt_regs(dst->target);
- dst->ret = user_regset_copyin(&dst->pos, &dst->count,
- &dst->u.set.kbuf, &dst->u.set.ubuf, &pt->r16,
- ELF_GR_OFFSET(16), ELF_NAT_OFFSET);
- if (dst->ret || dst->count == 0)
- return;
- }
+ while (dst->count && dst->pos < ELF_AR_END_OFFSET) {
+ unsigned int n, from, to;
+ elf_greg_t tmp[16];
- /* nat, pr, b0 - b7 */
- if (dst->count > 0 && dst->pos < ELF_CR_IIP_OFFSET) {
- i = dst->pos;
- index = (dst->pos - ELF_NAT_OFFSET) / sizeof(elf_greg_t);
+ from = dst->pos;
+ to = from + sizeof(tmp);
+ if (to > ELF_AR_END_OFFSET)
+ to = ELF_AR_END_OFFSET;
+ /* get up to 16 values */
dst->ret = user_regset_copyin(&dst->pos, &dst->count,
&dst->u.set.kbuf, &dst->u.set.ubuf, tmp,
- ELF_NAT_OFFSET, ELF_CR_IIP_OFFSET);
+ from, to);
if (dst->ret)
return;
- for (; i < dst->pos; i += sizeof(elf_greg_t), index++)
- if (access_elf_reg(dst->target, info, i,
- &tmp[index], 1) < 0) {
- dst->ret = -EIO;
- return;
- }
- if (dst->count == 0)
- return;
- }
-
- /* ip cfm psr ar.rsc ar.bsp ar.bspstore ar.rnat
- * ar.ccv ar.unat ar.fpsr ar.pfs ar.lc ar.ec ar.csd ar.ssd
- */
- if (dst->count > 0 && dst->pos < (ELF_AR_END_OFFSET)) {
- i = dst->pos;
- index = (dst->pos - ELF_CR_IIP_OFFSET) / sizeof(elf_greg_t);
- dst->ret = user_regset_copyin(&dst->pos, &dst->count,
- &dst->u.set.kbuf, &dst->u.set.ubuf, tmp,
- ELF_CR_IIP_OFFSET, ELF_AR_END_OFFSET);
- if (dst->ret)
- return;
- for ( ; i < dst->pos; i += sizeof(elf_greg_t), index++)
- if (access_elf_reg(dst->target, info, i,
- &tmp[index], 1) < 0) {
+ /* now copy them into registers */
+ for (n = 0; from < dst->pos; from += sizeof(elf_greg_t), n++)
+ if (access_elf_reg(dst->target, info, from,
+ &tmp[n], 1) < 0) {
dst->ret = -EIO;
return;
}
@@ -1690,60 +1574,36 @@
void do_fpregs_get(struct unw_frame_info *info, void *arg)
{
- struct regset_getset *dst = arg;
- struct task_struct *task = dst->target;
- elf_fpreg_t tmp[30];
- int index, min_copy, i;
+ struct task_struct *task = info->task;
+ struct regset_membuf *dst = arg;
+ struct membuf to = dst->to;
+ elf_fpreg_t reg;
+ unsigned int n;
if (unw_unwind_to_user(info) < 0)
return;
/* Skip pos 0 and 1 */
- if (dst->count > 0 && dst->pos < ELF_FP_OFFSET(2)) {
- dst->ret = user_regset_copyout_zero(&dst->pos, &dst->count,
- &dst->u.get.kbuf,
- &dst->u.get.ubuf,
- 0, ELF_FP_OFFSET(2));
- if (dst->count == 0 || dst->ret)
- return;
- }
+ membuf_zero(&to, 2 * sizeof(elf_fpreg_t));
/* fr2-fr31 */
- if (dst->count > 0 && dst->pos < ELF_FP_OFFSET(32)) {
- index = (dst->pos - ELF_FP_OFFSET(2)) / sizeof(elf_fpreg_t);
-
- min_copy = min(((unsigned int)ELF_FP_OFFSET(32)),
- dst->pos + dst->count);
- for (i = dst->pos; i < min_copy; i += sizeof(elf_fpreg_t),
- index++)
- if (unw_get_fr(info, i / sizeof(elf_fpreg_t),
- &tmp[index])) {
- dst->ret = -EIO;
- return;
- }
- dst->ret = user_regset_copyout(&dst->pos, &dst->count,
- &dst->u.get.kbuf, &dst->u.get.ubuf, tmp,
- ELF_FP_OFFSET(2), ELF_FP_OFFSET(32));
- if (dst->count == 0 || dst->ret)
+ for (n = 2; to.left && n < 32; n++) {
+ if (unw_get_fr(info, n, ®)) {
+ dst->ret = -EIO;
return;
+ }
+ membuf_write(&to, ®, sizeof(reg));
}
/* fph */
- if (dst->count > 0) {
- ia64_flush_fph(dst->target);
- if (task->thread.flags & IA64_THREAD_FPH_VALID)
- dst->ret = user_regset_copyout(
- &dst->pos, &dst->count,
- &dst->u.get.kbuf, &dst->u.get.ubuf,
- &dst->target->thread.fph,
- ELF_FP_OFFSET(32), -1);
- else
- /* Zero fill instead. */
- dst->ret = user_regset_copyout_zero(
- &dst->pos, &dst->count,
- &dst->u.get.kbuf, &dst->u.get.ubuf,
- ELF_FP_OFFSET(32), -1);
- }
+ if (!to.left)
+ return;
+
+ ia64_flush_fph(task);
+ if (task->thread.flags & IA64_THREAD_FPH_VALID)
+ membuf_write(&to, &task->thread.fph, 96 * sizeof(reg));
+ else
+ membuf_zero(&to, 96 * sizeof(reg));
}
void do_fpregs_set(struct unw_frame_info *info, void *arg)
@@ -1819,6 +1679,20 @@
}
}
+static void
+unwind_and_call(void (*call)(struct unw_frame_info *, void *),
+ struct task_struct *target, void *data)
+{
+ if (target == current)
+ unw_init_running(call, data);
+ else {
+ struct unw_frame_info info;
+ memset(&info, 0, sizeof(info));
+ unw_init_from_blocked_task(&info, target);
+ (*call)(&info, data);
+ }
+}
+
static int
do_regset_call(void (*call)(struct unw_frame_info *, void *),
struct task_struct *target,
@@ -1830,27 +1704,18 @@
.pos = pos, .count = count,
.u.set = { .kbuf = kbuf, .ubuf = ubuf },
.ret = 0 };
-
- if (target == current)
- unw_init_running(call, &info);
- else {
- struct unw_frame_info ufi;
- memset(&ufi, 0, sizeof(ufi));
- unw_init_from_blocked_task(&ufi, target);
- (*call)(&ufi, &info);
- }
-
+ unwind_and_call(call, target, &info);
return info.ret;
}
static int
gpregs_get(struct task_struct *target,
const struct user_regset *regset,
- unsigned int pos, unsigned int count,
- void *kbuf, void __user *ubuf)
+ struct membuf to)
{
- return do_regset_call(do_gpregs_get, target, regset, pos, count,
- kbuf, ubuf);
+ struct regset_membuf info = {.to = to};
+ unwind_and_call(do_gpregs_get, target, &info);
+ return info.ret;
}
static int gpregs_set(struct task_struct *target,
@@ -1892,11 +1757,11 @@
static int fpregs_get(struct task_struct *target,
const struct user_regset *regset,
- unsigned int pos, unsigned int count,
- void *kbuf, void __user *ubuf)
+ struct membuf to)
{
- return do_regset_call(do_fpregs_get, target, regset, pos, count,
- kbuf, ubuf);
+ struct regset_membuf info = {.to = to};
+ unwind_and_call(do_fpregs_get, target, &info);
+ return info.ret;
}
static int fpregs_set(struct task_struct *target,
@@ -1913,7 +1778,6 @@
unsigned long *data, int write_access)
{
unsigned int pos = -1; /* an invalid value */
- int ret;
unsigned long *ptr, regnum;
if ((addr & 0x7) != 0) {
@@ -1945,14 +1809,39 @@
}
if (pos != -1) {
- if (write_access)
- ret = fpregs_set(child, NULL, pos,
- sizeof(unsigned long), data, NULL);
- else
- ret = fpregs_get(child, NULL, pos,
- sizeof(unsigned long), data, NULL);
- if (ret != 0)
- return -1;
+ unsigned reg = pos / sizeof(elf_fpreg_t);
+ int which_half = (pos / sizeof(unsigned long)) & 1;
+
+ if (reg < 32) { /* fr2-fr31 */
+ struct unw_frame_info info;
+ elf_fpreg_t fpreg;
+
+ memset(&info, 0, sizeof(info));
+ unw_init_from_blocked_task(&info, child);
+ if (unw_unwind_to_user(&info) < 0)
+ return 0;
+
+ if (unw_get_fr(&info, reg, &fpreg))
+ return -1;
+ if (write_access) {
+ fpreg.u.bits[which_half] = *data;
+ if (unw_set_fr(&info, reg, fpreg))
+ return -1;
+ } else {
+ *data = fpreg.u.bits[which_half];
+ }
+ } else { /* fph */
+ elf_fpreg_t *p = &child->thread.fph[reg - 32];
+ unsigned long *bits = &p->u.bits[which_half];
+
+ ia64_sync_fph(child);
+ if (write_access)
+ *bits = *data;
+ else if (child->thread.flags & IA64_THREAD_FPH_VALID)
+ *data = *bits;
+ else
+ *data = 0;
+ }
return 0;
}
@@ -2038,15 +1927,14 @@
}
if (pos != -1) {
- if (write_access)
- ret = gpregs_set(child, NULL, pos,
- sizeof(unsigned long), data, NULL);
- else
- ret = gpregs_get(child, NULL, pos,
- sizeof(unsigned long), data, NULL);
- if (ret != 0)
- return -1;
- return 0;
+ struct unw_frame_info info;
+
+ memset(&info, 0, sizeof(info));
+ unw_init_from_blocked_task(&info, child);
+ if (unw_unwind_to_user(&info) < 0)
+ return 0;
+
+ return access_elf_reg(child, &info, pos, data, write_access);
}
/* access debug registers */
@@ -2112,14 +2000,14 @@
.core_note_type = NT_PRSTATUS,
.n = ELF_NGREG,
.size = sizeof(elf_greg_t), .align = sizeof(elf_greg_t),
- .get = gpregs_get, .set = gpregs_set,
+ .regset_get = gpregs_get, .set = gpregs_set,
.writeback = gpregs_writeback
},
{
.core_note_type = NT_PRFPREG,
.n = ELF_NFPREG,
.size = sizeof(elf_fpreg_t), .align = sizeof(elf_fpreg_t),
- .get = fpregs_get, .set = fpregs_set, .active = fpregs_active
+ .regset_get = fpregs_get, .set = fpregs_set, .active = fpregs_active
},
};
diff --git a/arch/m68k/coldfire/stmark2.c b/arch/m68k/coldfire/stmark2.c
index a8d2b3d..8b5af9c 100644
--- a/arch/m68k/coldfire/stmark2.c
+++ b/arch/m68k/coldfire/stmark2.c
@@ -13,6 +13,7 @@
#include <linux/spi/spi.h>
#include <linux/spi/spi-fsl-dspi.h>
#include <linux/spi/flash.h>
+#include <linux/dma-mapping.h>
#include <asm/mcfsim.h>
/*
@@ -78,6 +79,8 @@
},
};
+static u64 stmark2_dspi_mask = DMA_BIT_MASK(32);
+
/* SPI controller, id = bus number */
static struct platform_device dspi_spi0_device = {
.name = "fsl-dspi",
@@ -86,6 +89,8 @@
.resource = dspi_spi0_resource,
.dev = {
.platform_data = &dspi_spi0_info,
+ .dma_mask = &stmark2_dspi_mask,
+ .coherent_dma_mask = DMA_BIT_MASK(32),
},
};
diff --git a/arch/m68k/configs/stmark2_defconfig b/arch/m68k/configs/stmark2_defconfig
index 2b746f5..d923064 100644
--- a/arch/m68k/configs/stmark2_defconfig
+++ b/arch/m68k/configs/stmark2_defconfig
@@ -1,38 +1,33 @@
CONFIG_LOCALVERSION="stmark2-001"
CONFIG_DEFAULT_HOSTNAME="stmark2"
CONFIG_SYSVIPC=y
-# CONFIG_FHANDLE is not set
CONFIG_LOG_BUF_SHIFT=14
CONFIG_NAMESPACES=y
CONFIG_BLK_DEV_INITRD=y
-# CONFIG_RD_BZIP2 is not set
-# CONFIG_RD_LZMA is not set
-# CONFIG_RD_XZ is not set
-# CONFIG_RD_LZO is not set
-# CONFIG_RD_LZ4 is not set
CONFIG_CC_OPTIMIZE_FOR_SIZE=y
+# CONFIG_FHANDLE is not set
# CONFIG_AIO is not set
# CONFIG_ADVISE_SYSCALLS is not set
# CONFIG_MEMBARRIER is not set
CONFIG_EMBEDDED=y
# CONFIG_VM_EVENT_COUNTERS is not set
# CONFIG_COMPAT_BRK is not set
-# CONFIG_BLK_DEV_BSG is not set
-CONFIG_BLK_CMDLINE_PARSER=y
-# CONFIG_MMU is not set
+CONFIG_COLDFIRE=y
CONFIG_M5441x=y
CONFIG_CLOCK_FREQ=240000000
CONFIG_STMARK2=y
+CONFIG_UBOOT=y
CONFIG_RAMBASE=0x40000000
CONFIG_RAMSIZE=0x8000000
CONFIG_VECTORBASE=0x40000000
CONFIG_KERNELBASE=0x40001000
+# CONFIG_BLK_DEV_BSG is not set
+CONFIG_BLK_CMDLINE_PARSER=y
CONFIG_BINFMT_FLAT=y
+CONFIG_BINFMT_ZFLAT=y
CONFIG_BINFMT_MISC=y
-# CONFIG_UEVENT_HELPER is not set
CONFIG_DEVTMPFS=y
CONFIG_DEVTMPFS_MOUNT=y
-CONFIG_FW_LOADER_USER_HELPER_FALLBACK=y
# CONFIG_ALLOW_DEV_COREDUMP is not set
CONFIG_MTD=y
CONFIG_MTD_CMDLINE_PARTS=y
@@ -65,24 +60,36 @@
CONFIG_GPIO_SYSFS=y
CONFIG_GPIO_GENERIC_PLATFORM=y
# CONFIG_HWMON is not set
-# CONFIG_RC_CORE is not set
# CONFIG_HID is not set
# CONFIG_USB_SUPPORT is not set
+CONFIG_MMC=y
+CONFIG_MMC_DEBUG=y
+CONFIG_MMC_SDHCI=y
+CONFIG_MMC_SDHCI_PLTFM=y
+CONFIG_MMC_SDHCI_ESDHC_MCF=y
+CONFIG_DMADEVICES=y
+CONFIG_MCF_EDMA=y
+CONFIG_EXT2_FS=y
+CONFIG_EXT2_FS_XATTR=y
+CONFIG_EXT2_FS_POSIX_ACL=y
+CONFIG_EXT2_FS_SECURITY=y
+CONFIG_EXT3_FS=y
+CONFIG_EXT3_FS_POSIX_ACL=y
+CONFIG_EXT3_FS_SECURITY=y
# CONFIG_FILE_LOCKING is not set
# CONFIG_DNOTIFY is not set
# CONFIG_INOTIFY_USER is not set
+CONFIG_OVERLAY_FS=y
CONFIG_FSCACHE=y
# CONFIG_PROC_SYSCTL is not set
+CONFIG_CRAMFS=y
+CONFIG_SQUASHFS=y
+CONFIG_ROMFS_FS=y
+CONFIG_CRYPTO_ANSI_CPRNG=y
+# CONFIG_CRYPTO_HW is not set
CONFIG_PRINTK_TIME=y
+# CONFIG_DEBUG_BUGVERBOSE is not set
# CONFIG_SECTION_MISMATCH_WARN_ONLY is not set
CONFIG_SLUB_DEBUG_ON=y
CONFIG_PANIC_ON_OOPS=y
# CONFIG_SCHED_DEBUG is not set
-# CONFIG_DEBUG_BUGVERBOSE is not set
-CONFIG_BOOTPARAM=y
-CONFIG_BOOTPARAM_STRING="console=ttyS0,115200 root=/dev/ram0 rw rootfstype=ramfs rdinit=/bin/init devtmpfs.mount=1"
-CONFIG_CRYPTO=y
-# CONFIG_CRYPTO_ECHAINIV is not set
-CONFIG_CRYPTO_ANSI_CPRNG=y
-# CONFIG_CRYPTO_HW is not set
-CONFIG_CRC16=y
diff --git a/arch/m68k/include/asm/adb_iop.h b/arch/m68k/include/asm/adb_iop.h
index 195d7fb..6aecd02 100644
--- a/arch/m68k/include/asm/adb_iop.h
+++ b/arch/m68k/include/asm/adb_iop.h
@@ -29,6 +29,7 @@
#define ADB_IOP_EXPLICIT 0x80 /* nonzero if explicit command */
#define ADB_IOP_AUTOPOLL 0x40 /* auto/SRQ polling enabled */
+#define ADB_IOP_SET_AUTOPOLL 0x20 /* set autopoll device list */
#define ADB_IOP_SRQ 0x04 /* SRQ detected */
#define ADB_IOP_TIMEOUT 0x02 /* nonzero if timeout */
diff --git a/arch/m68k/include/asm/cmpxchg.h b/arch/m68k/include/asm/cmpxchg.h
index 38e1d7a..3a3bdcf 100644
--- a/arch/m68k/include/asm/cmpxchg.h
+++ b/arch/m68k/include/asm/cmpxchg.h
@@ -129,14 +129,6 @@
#else
-/*
- * cmpxchg_local and cmpxchg64_local are atomic wrt current CPU. Always make
- * them available.
- */
-#define cmpxchg_local(ptr, o, n) \
- ((__typeof__(*(ptr)))__cmpxchg_local_generic((ptr), (unsigned long)(o),\
- (unsigned long)(n), sizeof(*(ptr))))
-
#include <asm-generic/cmpxchg.h>
#endif
diff --git a/arch/m68k/include/asm/io_no.h b/arch/m68k/include/asm/io_no.h
index 0498192..2c96e84 100644
--- a/arch/m68k/include/asm/io_no.h
+++ b/arch/m68k/include/asm/io_no.h
@@ -14,15 +14,15 @@
* that behavior here first before we include asm-generic/io.h.
*/
#define __raw_readb(addr) \
- ({ unsigned char __v = (*(volatile unsigned char *) (addr)); __v; })
+ ({ u8 __v = (*(__force volatile u8 *) (addr)); __v; })
#define __raw_readw(addr) \
- ({ unsigned short __v = (*(volatile unsigned short *) (addr)); __v; })
+ ({ u16 __v = (*(__force volatile u16 *) (addr)); __v; })
#define __raw_readl(addr) \
- ({ unsigned int __v = (*(volatile unsigned int *) (addr)); __v; })
+ ({ u32 __v = (*(__force volatile u32 *) (addr)); __v; })
-#define __raw_writeb(b, addr) (void)((*(volatile unsigned char *) (addr)) = (b))
-#define __raw_writew(b, addr) (void)((*(volatile unsigned short *) (addr)) = (b))
-#define __raw_writel(b, addr) (void)((*(volatile unsigned int *) (addr)) = (b))
+#define __raw_writeb(b, addr) (void)((*(__force volatile u8 *) (addr)) = (b))
+#define __raw_writew(b, addr) (void)((*(__force volatile u16 *) (addr)) = (b))
+#define __raw_writel(b, addr) (void)((*(__force volatile u32 *) (addr)) = (b))
#if defined(CONFIG_COLDFIRE)
/*
@@ -67,7 +67,7 @@
{
if (cf_internalio(addr))
return __raw_readw(addr);
- return __le16_to_cpu(__raw_readw(addr));
+ return swab16(__raw_readw(addr));
}
#define readl readl
@@ -75,7 +75,7 @@
{
if (cf_internalio(addr))
return __raw_readl(addr);
- return __le32_to_cpu(__raw_readl(addr));
+ return swab32(__raw_readl(addr));
}
#define writew writew
@@ -84,7 +84,7 @@
if (cf_internalio(addr))
__raw_writew(value, addr);
else
- __raw_writew(__cpu_to_le16(value), addr);
+ __raw_writew(swab16(value), addr);
}
#define writel writel
@@ -93,7 +93,7 @@
if (cf_internalio(addr))
__raw_writel(value, addr);
else
- __raw_writel(__cpu_to_le32(value), addr);
+ __raw_writel(swab32(value), addr);
}
#else
diff --git a/arch/m68k/include/asm/m53xxacr.h b/arch/m68k/include/asm/m53xxacr.h
index 9138a62..692f90e 100644
--- a/arch/m68k/include/asm/m53xxacr.h
+++ b/arch/m68k/include/asm/m53xxacr.h
@@ -89,9 +89,9 @@
* coherency though in all cases. And for copyback caches we will need
* to push cached data as well.
*/
-#define CACHE_INIT CACR_CINVA
-#define CACHE_INVALIDATE CACR_CINVA
-#define CACHE_INVALIDATED CACR_CINVA
+#define CACHE_INIT (CACHE_MODE + CACR_CINVA - CACR_EC)
+#define CACHE_INVALIDATE (CACHE_MODE + CACR_CINVA)
+#define CACHE_INVALIDATED (CACHE_MODE + CACR_CINVA)
#define ACR0_MODE ((CONFIG_RAMBASE & 0xff000000) + \
(0x000f0000) + \
diff --git a/arch/m68k/mm/mcfmmu.c b/arch/m68k/mm/mcfmmu.c
index 7068126..2b9cb4a 100644
--- a/arch/m68k/mm/mcfmmu.c
+++ b/arch/m68k/mm/mcfmmu.c
@@ -40,7 +40,6 @@
unsigned long address, size;
unsigned long next_pgtable, bootmem_end;
unsigned long max_zone_pfn[MAX_NR_ZONES] = { 0 };
- enum zone_type zone;
int i;
empty_zero_page = (void *) memblock_alloc(PAGE_SIZE, PAGE_SIZE);
@@ -215,11 +214,6 @@
/*
* Steal a context from a task that has one at the moment.
- * This is only used on 8xx and 4xx and we presently assume that
- * they don't do SMP. If they do then thicfpgalloc.hs will have to check
- * whether the MM we steal is in use.
- * We also assume that this is only used on systems that don't
- * use an MMU hash table - this is true for 8xx and 4xx.
* This isn't an LRU system, it just frees up each context in
* turn (sort-of pseudo-random replacement :). This would be the
* place to implement an LRU scheme if anyone was motivated to do it.
diff --git a/arch/mips/kernel/ptrace.c b/arch/mips/kernel/ptrace.c
index 2a61641..db7c5be 100644
--- a/arch/mips/kernel/ptrace.c
+++ b/arch/mips/kernel/ptrace.c
@@ -210,15 +210,13 @@
static int gpr32_get(struct task_struct *target,
const struct user_regset *regset,
- unsigned int pos, unsigned int count,
- void *kbuf, void __user *ubuf)
+ struct membuf to)
{
struct pt_regs *regs = task_pt_regs(target);
u32 uregs[ELF_NGREG] = {};
mips_dump_regs32(uregs, regs);
- return user_regset_copyout(&pos, &count, &kbuf, &ubuf, uregs, 0,
- sizeof(uregs));
+ return membuf_write(&to, uregs, sizeof(uregs));
}
static int gpr32_set(struct task_struct *target,
@@ -277,15 +275,13 @@
static int gpr64_get(struct task_struct *target,
const struct user_regset *regset,
- unsigned int pos, unsigned int count,
- void *kbuf, void __user *ubuf)
+ struct membuf to)
{
struct pt_regs *regs = task_pt_regs(target);
u64 uregs[ELF_NGREG] = {};
mips_dump_regs64(uregs, regs);
- return user_regset_copyout(&pos, &count, &kbuf, &ubuf, uregs, 0,
- sizeof(uregs));
+ return membuf_write(&to, uregs, sizeof(uregs));
}
static int gpr64_set(struct task_struct *target,
@@ -408,13 +404,11 @@
* !CONFIG_CPU_HAS_MSA variant. FP context's general register slots
* correspond 1:1 to buffer slots. Only general registers are copied.
*/
-static int fpr_get_fpa(struct task_struct *target,
- unsigned int *pos, unsigned int *count,
- void **kbuf, void __user **ubuf)
+static void fpr_get_fpa(struct task_struct *target,
+ struct membuf *to)
{
- return user_regset_copyout(pos, count, kbuf, ubuf,
- &target->thread.fpu,
- 0, NUM_FPU_REGS * sizeof(elf_fpreg_t));
+ membuf_write(to, &target->thread.fpu,
+ NUM_FPU_REGS * sizeof(elf_fpreg_t));
}
/*
@@ -423,25 +417,13 @@
* general register slots are copied to buffer slots. Only general
* registers are copied.
*/
-static int fpr_get_msa(struct task_struct *target,
- unsigned int *pos, unsigned int *count,
- void **kbuf, void __user **ubuf)
+static void fpr_get_msa(struct task_struct *target, struct membuf *to)
{
unsigned int i;
- u64 fpr_val;
- int err;
- BUILD_BUG_ON(sizeof(fpr_val) != sizeof(elf_fpreg_t));
- for (i = 0; i < NUM_FPU_REGS; i++) {
- fpr_val = get_fpr64(&target->thread.fpu.fpr[i], 0);
- err = user_regset_copyout(pos, count, kbuf, ubuf,
- &fpr_val, i * sizeof(elf_fpreg_t),
- (i + 1) * sizeof(elf_fpreg_t));
- if (err)
- return err;
- }
-
- return 0;
+ BUILD_BUG_ON(sizeof(u64) != sizeof(elf_fpreg_t));
+ for (i = 0; i < NUM_FPU_REGS; i++)
+ membuf_store(to, get_fpr64(&target->thread.fpu.fpr[i], 0));
}
/*
@@ -451,31 +433,16 @@
*/
static int fpr_get(struct task_struct *target,
const struct user_regset *regset,
- unsigned int pos, unsigned int count,
- void *kbuf, void __user *ubuf)
+ struct membuf to)
{
- const int fcr31_pos = NUM_FPU_REGS * sizeof(elf_fpreg_t);
- const int fir_pos = fcr31_pos + sizeof(u32);
- int err;
-
if (sizeof(target->thread.fpu.fpr[0]) == sizeof(elf_fpreg_t))
- err = fpr_get_fpa(target, &pos, &count, &kbuf, &ubuf);
+ fpr_get_fpa(target, &to);
else
- err = fpr_get_msa(target, &pos, &count, &kbuf, &ubuf);
- if (err)
- return err;
+ fpr_get_msa(target, &to);
- err = user_regset_copyout(&pos, &count, &kbuf, &ubuf,
- &target->thread.fpu.fcr31,
- fcr31_pos, fcr31_pos + sizeof(u32));
- if (err)
- return err;
-
- err = user_regset_copyout(&pos, &count, &kbuf, &ubuf,
- &boot_cpu_data.fpu_id,
- fir_pos, fir_pos + sizeof(u32));
-
- return err;
+ membuf_write(&to, &target->thread.fpu.fcr31, sizeof(u32));
+ membuf_write(&to, &boot_cpu_data.fpu_id, sizeof(u32));
+ return 0;
}
/*
@@ -576,14 +543,9 @@
/* Copy the FP mode setting to the supplied NT_MIPS_FP_MODE buffer. */
static int fp_mode_get(struct task_struct *target,
const struct user_regset *regset,
- unsigned int pos, unsigned int count,
- void *kbuf, void __user *ubuf)
+ struct membuf to)
{
- int fp_mode;
-
- fp_mode = mips_get_process_fp_mode(target);
- return user_regset_copyout(&pos, &count, &kbuf, &ubuf, &fp_mode, 0,
- sizeof(fp_mode));
+ return membuf_store(&to, (int)mips_get_process_fp_mode(target));
}
/*
@@ -630,13 +592,12 @@
unsigned int msacsr;
};
-static int copy_pad_fprs(struct task_struct *target,
+static void copy_pad_fprs(struct task_struct *target,
const struct user_regset *regset,
- unsigned int *ppos, unsigned int *pcount,
- void **pkbuf, void __user **pubuf,
+ struct membuf *to,
unsigned int live_sz)
{
- int i, j, start, start_pad, err;
+ int i, j;
unsigned long long fill = ~0ull;
unsigned int cp_sz, pad_sz;
@@ -644,28 +605,16 @@
pad_sz = regset->size - cp_sz;
WARN_ON(pad_sz % sizeof(fill));
- i = start = err = 0;
- for (; i < NUM_FPU_REGS; i++, start += regset->size) {
- err |= user_regset_copyout(ppos, pcount, pkbuf, pubuf,
- &target->thread.fpu.fpr[i],
- start, start + cp_sz);
-
- start_pad = start + cp_sz;
- for (j = 0; j < (pad_sz / sizeof(fill)); j++) {
- err |= user_regset_copyout(ppos, pcount, pkbuf, pubuf,
- &fill, start_pad,
- start_pad + sizeof(fill));
- start_pad += sizeof(fill);
- }
+ for (i = 0; i < NUM_FPU_REGS; i++) {
+ membuf_write(to, &target->thread.fpu.fpr[i], cp_sz);
+ for (j = 0; j < (pad_sz / sizeof(fill)); j++)
+ membuf_store(to, fill);
}
-
- return err;
}
static int msa_get(struct task_struct *target,
const struct user_regset *regset,
- unsigned int pos, unsigned int count,
- void *kbuf, void __user *ubuf)
+ struct membuf to)
{
const unsigned int wr_size = NUM_FPU_REGS * regset->size;
const struct msa_control_regs ctrl_regs = {
@@ -674,32 +623,23 @@
.msair = boot_cpu_data.msa_id,
.msacsr = target->thread.fpu.msacsr,
};
- int err;
if (!tsk_used_math(target)) {
/* The task hasn't used FP or MSA, fill with 0xff */
- err = copy_pad_fprs(target, regset, &pos, &count,
- &kbuf, &ubuf, 0);
+ copy_pad_fprs(target, regset, &to, 0);
} else if (!test_tsk_thread_flag(target, TIF_MSA_CTX_LIVE)) {
/* Copy scalar FP context, fill the rest with 0xff */
- err = copy_pad_fprs(target, regset, &pos, &count,
- &kbuf, &ubuf, 8);
+ copy_pad_fprs(target, regset, &to, 8);
} else if (sizeof(target->thread.fpu.fpr[0]) == regset->size) {
/* Trivially copy the vector registers */
- err = user_regset_copyout(&pos, &count, &kbuf, &ubuf,
- &target->thread.fpu.fpr,
- 0, wr_size);
+ membuf_write(&to, &target->thread.fpu.fpr, wr_size);
} else {
/* Copy as much context as possible, fill the rest with 0xff */
- err = copy_pad_fprs(target, regset, &pos, &count,
- &kbuf, &ubuf,
- sizeof(target->thread.fpu.fpr[0]));
+ copy_pad_fprs(target, regset, &to,
+ sizeof(target->thread.fpu.fpr[0]));
}
- err |= user_regset_copyout(&pos, &count, &kbuf, &ubuf,
- &ctrl_regs, wr_size,
- wr_size + sizeof(ctrl_regs));
- return err;
+ return membuf_write(&to, &ctrl_regs, sizeof(ctrl_regs));
}
static int msa_set(struct task_struct *target,
@@ -752,34 +692,20 @@
*/
static int dsp32_get(struct task_struct *target,
const struct user_regset *regset,
- unsigned int pos, unsigned int count,
- void *kbuf, void __user *ubuf)
+ struct membuf to)
{
- unsigned int start, num_regs, i;
u32 dspregs[NUM_DSP_REGS + 1];
+ unsigned int i;
- BUG_ON(count % sizeof(u32));
+ BUG_ON(to.left % sizeof(u32));
if (!cpu_has_dsp)
return -EIO;
- start = pos / sizeof(u32);
- num_regs = count / sizeof(u32);
-
- if (start + num_regs > NUM_DSP_REGS + 1)
- return -EIO;
-
- for (i = start; i < num_regs; i++)
- switch (i) {
- case 0 ... NUM_DSP_REGS - 1:
- dspregs[i] = target->thread.dsp.dspr[i];
- break;
- case NUM_DSP_REGS:
- dspregs[i] = target->thread.dsp.dspcontrol;
- break;
- }
- return user_regset_copyout(&pos, &count, &kbuf, &ubuf, dspregs, 0,
- sizeof(dspregs));
+ for (i = 0; i < NUM_DSP_REGS; i++)
+ dspregs[i] = target->thread.dsp.dspr[i];
+ dspregs[NUM_DSP_REGS] = target->thread.dsp.dspcontrol;
+ return membuf_write(&to, dspregs, sizeof(dspregs));
}
/*
@@ -832,34 +758,20 @@
*/
static int dsp64_get(struct task_struct *target,
const struct user_regset *regset,
- unsigned int pos, unsigned int count,
- void *kbuf, void __user *ubuf)
+ struct membuf to)
{
- unsigned int start, num_regs, i;
u64 dspregs[NUM_DSP_REGS + 1];
+ unsigned int i;
- BUG_ON(count % sizeof(u64));
+ BUG_ON(to.left % sizeof(u64));
if (!cpu_has_dsp)
return -EIO;
- start = pos / sizeof(u64);
- num_regs = count / sizeof(u64);
-
- if (start + num_regs > NUM_DSP_REGS + 1)
- return -EIO;
-
- for (i = start; i < num_regs; i++)
- switch (i) {
- case 0 ... NUM_DSP_REGS - 1:
- dspregs[i] = target->thread.dsp.dspr[i];
- break;
- case NUM_DSP_REGS:
- dspregs[i] = target->thread.dsp.dspcontrol;
- break;
- }
- return user_regset_copyout(&pos, &count, &kbuf, &ubuf, dspregs, 0,
- sizeof(dspregs));
+ for (i = 0; i < NUM_DSP_REGS; i++)
+ dspregs[i] = target->thread.dsp.dspr[i];
+ dspregs[NUM_DSP_REGS] = target->thread.dsp.dspcontrol;
+ return membuf_write(&to, dspregs, sizeof(dspregs));
}
/*
@@ -1018,7 +930,7 @@
.n = ELF_NGREG,
.size = sizeof(unsigned int),
.align = sizeof(unsigned int),
- .get = gpr32_get,
+ .regset_get = gpr32_get,
.set = gpr32_set,
},
[REGSET_DSP] = {
@@ -1026,7 +938,7 @@
.n = NUM_DSP_REGS + 1,
.size = sizeof(u32),
.align = sizeof(u32),
- .get = dsp32_get,
+ .regset_get = dsp32_get,
.set = dsp32_set,
.active = dsp_active,
},
@@ -1036,7 +948,7 @@
.n = ELF_NFPREG,
.size = sizeof(elf_fpreg_t),
.align = sizeof(elf_fpreg_t),
- .get = fpr_get,
+ .regset_get = fpr_get,
.set = fpr_set,
},
[REGSET_FP_MODE] = {
@@ -1044,7 +956,7 @@
.n = 1,
.size = sizeof(int),
.align = sizeof(int),
- .get = fp_mode_get,
+ .regset_get = fp_mode_get,
.set = fp_mode_set,
},
#endif
@@ -1054,7 +966,7 @@
.n = NUM_FPU_REGS + 1,
.size = 16,
.align = 16,
- .get = msa_get,
+ .regset_get = msa_get,
.set = msa_set,
},
#endif
@@ -1078,7 +990,7 @@
.n = ELF_NGREG,
.size = sizeof(unsigned long),
.align = sizeof(unsigned long),
- .get = gpr64_get,
+ .regset_get = gpr64_get,
.set = gpr64_set,
},
[REGSET_DSP] = {
@@ -1086,7 +998,7 @@
.n = NUM_DSP_REGS + 1,
.size = sizeof(u64),
.align = sizeof(u64),
- .get = dsp64_get,
+ .regset_get = dsp64_get,
.set = dsp64_set,
.active = dsp_active,
},
@@ -1096,7 +1008,7 @@
.n = 1,
.size = sizeof(int),
.align = sizeof(int),
- .get = fp_mode_get,
+ .regset_get = fp_mode_get,
.set = fp_mode_set,
},
[REGSET_FPR] = {
@@ -1104,7 +1016,7 @@
.n = ELF_NFPREG,
.size = sizeof(elf_fpreg_t),
.align = sizeof(elf_fpreg_t),
- .get = fpr_get,
+ .regset_get = fpr_get,
.set = fpr_set,
},
#endif
@@ -1114,7 +1026,7 @@
.n = NUM_FPU_REGS + 1,
.size = 16,
.align = 16,
- .get = msa_get,
+ .regset_get = msa_get,
.set = msa_set,
},
#endif
diff --git a/arch/nds32/kernel/ptrace.c b/arch/nds32/kernel/ptrace.c
index eaaf7a9..d0eda87 100644
--- a/arch/nds32/kernel/ptrace.c
+++ b/arch/nds32/kernel/ptrace.c
@@ -13,11 +13,10 @@
static int gpr_get(struct task_struct *target,
const struct user_regset *regset,
- unsigned int pos, unsigned int count,
- void *kbuf, void __user * ubuf)
+ struct membuf to)
{
- struct user_pt_regs *uregs = &task_pt_regs(target)->user_regs;
- return user_regset_copyout(&pos, &count, &kbuf, &ubuf, uregs, 0, -1);
+ return membuf_write(&to, &task_pt_regs(target)->user_regs,
+ sizeof(struct user_pt_regs));
}
static int gpr_set(struct task_struct *target, const struct user_regset *regset,
@@ -41,7 +40,7 @@
.n = sizeof(struct user_pt_regs) / sizeof(u32),
.size = sizeof(elf_greg_t),
.align = sizeof(elf_greg_t),
- .get = gpr_get,
+ .regset_get = gpr_get,
.set = gpr_set}
};
diff --git a/arch/nios2/kernel/ptrace.c b/arch/nios2/kernel/ptrace.c
index de97bcb..a6ea9e1 100644
--- a/arch/nios2/kernel/ptrace.c
+++ b/arch/nios2/kernel/ptrace.c
@@ -21,45 +21,24 @@
static int genregs_get(struct task_struct *target,
const struct user_regset *regset,
- unsigned int pos, unsigned int count,
- void *kbuf, void __user *ubuf)
+ struct membuf to)
{
const struct pt_regs *regs = task_pt_regs(target);
const struct switch_stack *sw = (struct switch_stack *)regs - 1;
- int ret = 0;
-#define REG_O_ZERO_RANGE(START, END) \
- if (!ret) \
- ret = user_regset_copyout_zero(&pos, &count, &kbuf, &ubuf, \
- START * 4, (END * 4) + 4);
-
-#define REG_O_ONE(PTR, LOC) \
- if (!ret) \
- ret = user_regset_copyout(&pos, &count, &kbuf, &ubuf, PTR, \
- LOC * 4, (LOC * 4) + 4);
-
-#define REG_O_RANGE(PTR, START, END) \
- if (!ret) \
- ret = user_regset_copyout(&pos, &count, &kbuf, &ubuf, PTR, \
- START * 4, (END * 4) + 4);
-
- REG_O_ZERO_RANGE(PTR_R0, PTR_R0);
- REG_O_RANGE(®s->r1, PTR_R1, PTR_R7);
- REG_O_RANGE(®s->r8, PTR_R8, PTR_R15);
- REG_O_RANGE(sw, PTR_R16, PTR_R23);
- REG_O_ZERO_RANGE(PTR_R24, PTR_R25); /* et and bt */
- REG_O_ONE(®s->gp, PTR_GP);
- REG_O_ONE(®s->sp, PTR_SP);
- REG_O_ONE(®s->fp, PTR_FP);
- REG_O_ONE(®s->ea, PTR_EA);
- REG_O_ZERO_RANGE(PTR_BA, PTR_BA);
- REG_O_ONE(®s->ra, PTR_RA);
- REG_O_ONE(®s->ea, PTR_PC); /* use ea for PC */
- if (!ret)
- ret = user_regset_copyout_zero(&pos, &count, &kbuf, &ubuf,
- PTR_STATUS * 4, -1);
-
- return ret;
+ membuf_zero(&to, 4); // R0
+ membuf_write(&to, ®s->r1, 7 * 4); // R1..R7
+ membuf_write(&to, ®s->r8, 8 * 4); // R8..R15
+ membuf_write(&to, sw, 8 * 4); // R16..R23
+ membuf_zero(&to, 2 * 4); /* et and bt */
+ membuf_store(&to, regs->gp);
+ membuf_store(&to, regs->sp);
+ membuf_store(&to, regs->fp);
+ membuf_store(&to, regs->ea);
+ membuf_zero(&to, 4); // PTR_BA
+ membuf_store(&to, regs->ra);
+ membuf_store(&to, regs->ea); /* use ea for PC */
+ return membuf_zero(&to, (NUM_PTRACE_REG - PTR_PC) * 4);
}
/*
@@ -121,7 +100,7 @@
.n = NUM_PTRACE_REG,
.size = sizeof(unsigned long),
.align = sizeof(unsigned long),
- .get = genregs_get,
+ .regset_get = genregs_get,
.set = genregs_set,
}
};
diff --git a/arch/openrisc/kernel/ptrace.c b/arch/openrisc/kernel/ptrace.c
index c8f47a6..4d60ae2 100644
--- a/arch/openrisc/kernel/ptrace.c
+++ b/arch/openrisc/kernel/ptrace.c
@@ -44,29 +44,15 @@
*/
static int genregs_get(struct task_struct *target,
const struct user_regset *regset,
- unsigned int pos, unsigned int count,
- void *kbuf, void __user * ubuf)
+ struct membuf to)
{
const struct pt_regs *regs = task_pt_regs(target);
- int ret;
/* r0 */
- ret = user_regset_copyout_zero(&pos, &count, &kbuf, &ubuf, 0, 4);
-
- if (!ret)
- ret = user_regset_copyout(&pos, &count, &kbuf, &ubuf,
- regs->gpr+1, 4, 4*32);
- if (!ret)
- ret = user_regset_copyout(&pos, &count, &kbuf, &ubuf,
- ®s->pc, 4*32, 4*33);
- if (!ret)
- ret = user_regset_copyout(&pos, &count, &kbuf, &ubuf,
- ®s->sr, 4*33, 4*34);
- if (!ret)
- ret = user_regset_copyout_zero(&pos, &count, &kbuf, &ubuf,
- 4*34, -1);
-
- return ret;
+ membuf_zero(&to, 4);
+ membuf_write(&to, regs->gpr + 1, 31 * 4);
+ membuf_store(&to, regs->pc);
+ return membuf_store(&to, regs->sr);
}
/*
@@ -114,7 +100,7 @@
.n = ELF_NGREG,
.size = sizeof(long),
.align = sizeof(long),
- .get = genregs_get,
+ .regset_get = genregs_get,
.set = genregs_set,
},
};
diff --git a/arch/parisc/kernel/ptrace.c b/arch/parisc/kernel/ptrace.c
index b51418a..2127974 100644
--- a/arch/parisc/kernel/ptrace.c
+++ b/arch/parisc/kernel/ptrace.c
@@ -391,31 +391,11 @@
static int fpr_get(struct task_struct *target,
const struct user_regset *regset,
- unsigned int pos, unsigned int count,
- void *kbuf, void __user *ubuf)
+ struct membuf to)
{
struct pt_regs *regs = task_regs(target);
- __u64 *k = kbuf;
- __u64 __user *u = ubuf;
- __u64 reg;
- pos /= sizeof(reg);
- count /= sizeof(reg);
-
- if (kbuf)
- for (; count > 0 && pos < ELF_NFPREG; --count)
- *k++ = regs->fr[pos++];
- else
- for (; count > 0 && pos < ELF_NFPREG; --count)
- if (__put_user(regs->fr[pos++], u++))
- return -EFAULT;
-
- kbuf = k;
- ubuf = u;
- pos *= sizeof(reg);
- count *= sizeof(reg);
- return user_regset_copyout_zero(&pos, &count, &kbuf, &ubuf,
- ELF_NFPREG * sizeof(reg), -1);
+ return membuf_write(&to, regs->fr, ELF_NFPREG * sizeof(__u64));
}
static int fpr_set(struct task_struct *target,
@@ -527,30 +507,14 @@
static int gpr_get(struct task_struct *target,
const struct user_regset *regset,
- unsigned int pos, unsigned int count,
- void *kbuf, void __user *ubuf)
+ struct membuf to)
{
struct pt_regs *regs = task_regs(target);
- unsigned long *k = kbuf;
- unsigned long __user *u = ubuf;
- unsigned long reg;
+ unsigned int pos;
- pos /= sizeof(reg);
- count /= sizeof(reg);
-
- if (kbuf)
- for (; count > 0 && pos < ELF_NGREG; --count)
- *k++ = get_reg(regs, pos++);
- else
- for (; count > 0 && pos < ELF_NGREG; --count)
- if (__put_user(get_reg(regs, pos++), u++))
- return -EFAULT;
- kbuf = k;
- ubuf = u;
- pos *= sizeof(reg);
- count *= sizeof(reg);
- return user_regset_copyout_zero(&pos, &count, &kbuf, &ubuf,
- ELF_NGREG * sizeof(reg), -1);
+ for (pos = 0; pos < ELF_NGREG; pos++)
+ membuf_store(&to, get_reg(regs, pos));
+ return 0;
}
static int gpr_set(struct task_struct *target,
@@ -588,12 +552,12 @@
[REGSET_GENERAL] = {
.core_note_type = NT_PRSTATUS, .n = ELF_NGREG,
.size = sizeof(long), .align = sizeof(long),
- .get = gpr_get, .set = gpr_set
+ .regset_get = gpr_get, .set = gpr_set
},
[REGSET_FP] = {
.core_note_type = NT_PRFPREG, .n = ELF_NFPREG,
.size = sizeof(__u64), .align = sizeof(__u64),
- .get = fpr_get, .set = fpr_set
+ .regset_get = fpr_get, .set = fpr_set
}
};
@@ -607,31 +571,15 @@
static int gpr32_get(struct task_struct *target,
const struct user_regset *regset,
- unsigned int pos, unsigned int count,
- void *kbuf, void __user *ubuf)
+ struct membuf to)
{
struct pt_regs *regs = task_regs(target);
- compat_ulong_t *k = kbuf;
- compat_ulong_t __user *u = ubuf;
- compat_ulong_t reg;
+ unsigned int pos;
- pos /= sizeof(reg);
- count /= sizeof(reg);
+ for (pos = 0; pos < ELF_NGREG; pos++)
+ membuf_store(&to, (compat_ulong_t)get_reg(regs, pos));
- if (kbuf)
- for (; count > 0 && pos < ELF_NGREG; --count)
- *k++ = get_reg(regs, pos++);
- else
- for (; count > 0 && pos < ELF_NGREG; --count)
- if (__put_user((compat_ulong_t) get_reg(regs, pos++), u++))
- return -EFAULT;
-
- kbuf = k;
- ubuf = u;
- pos *= sizeof(reg);
- count *= sizeof(reg);
- return user_regset_copyout_zero(&pos, &count, &kbuf, &ubuf,
- ELF_NGREG * sizeof(reg), -1);
+ return 0;
}
static int gpr32_set(struct task_struct *target,
@@ -672,12 +620,12 @@
[REGSET_GENERAL] = {
.core_note_type = NT_PRSTATUS, .n = ELF_NGREG,
.size = sizeof(compat_long_t), .align = sizeof(compat_long_t),
- .get = gpr32_get, .set = gpr32_set
+ .regset_get = gpr32_get, .set = gpr32_set
},
[REGSET_FP] = {
.core_note_type = NT_PRFPREG, .n = ELF_NFPREG,
.size = sizeof(__u64), .align = sizeof(__u64),
- .get = fpr_get, .set = fpr_set
+ .regset_get = fpr_get, .set = fpr_set
}
};
diff --git a/arch/powerpc/Kconfig b/arch/powerpc/Kconfig
index 29a0579..1f48bbf 100644
--- a/arch/powerpc/Kconfig
+++ b/arch/powerpc/Kconfig
@@ -131,6 +131,7 @@
select ARCH_HAS_PTE_DEVMAP if PPC_BOOK3S_64
select ARCH_HAS_PTE_SPECIAL
select ARCH_HAS_MEMBARRIER_CALLBACKS
+ select ARCH_HAS_MEMBARRIER_SYNC_CORE
select ARCH_HAS_SCALED_CPUTIME if VIRT_CPU_ACCOUNTING_NATIVE && PPC_BOOK3S_64
select ARCH_HAS_STRICT_KERNEL_RWX if (PPC32 && !HIBERNATION)
select ARCH_HAS_TICK_BROADCAST if GENERIC_CLOCKEVENTS_BROADCAST
@@ -145,6 +146,8 @@
select ARCH_SUPPORTS_ATOMIC_RMW
select ARCH_USE_BUILTIN_BSWAP
select ARCH_USE_CMPXCHG_LOCKREF if PPC64
+ select ARCH_USE_QUEUED_RWLOCKS if PPC_QUEUED_SPINLOCKS
+ select ARCH_USE_QUEUED_SPINLOCKS if PPC_QUEUED_SPINLOCKS
select ARCH_WANT_IPC_PARSE_VERSION
select ARCH_WEAK_RELEASE_ACQUIRE
select BINFMT_ELF
@@ -491,6 +494,19 @@
Say N if you are unsure.
+config PPC_QUEUED_SPINLOCKS
+ bool "Queued spinlocks"
+ depends on SMP
+ help
+ Say Y here to use queued spinlocks which give better scalability and
+ fairness on large SMP and NUMA systems without harming single threaded
+ performance.
+
+ This option is currently experimental, the code is more complex and
+ less tested so it defaults to "N" for the moment.
+
+ If unsure, say "N".
+
config ARCH_CPU_PROBE_RELEASE
def_bool y
depends on HOTPLUG_CPU
@@ -834,13 +850,16 @@
this in mind when choosing a value for this option.
config PPC_SUBPAGE_PROT
- bool "Support setting protections for 4k subpages"
+ bool "Support setting protections for 4k subpages (subpage_prot syscall)"
+ default n
depends on PPC_BOOK3S_64 && PPC_64K_PAGES
help
- This option adds support for a system call to allow user programs
+ This option adds support for system call to allow user programs
to set access permissions (read/write, readonly, or no access)
on the 4k subpages of each 64k page.
+ If unsure, say N here.
+
config PPC_COPRO_BASE
bool
@@ -860,12 +879,8 @@
Add support for handling denormalisation of single precision
values. Useful for bare metal only. If unsure say Y here.
-config CMDLINE_BOOL
- bool "Default bootloader kernel arguments"
-
config CMDLINE
- string "Initial kernel command string" if CMDLINE_BOOL
- default "console=ttyS0,9600 console=tty0 root=/dev/sda2" if CMDLINE_BOOL
+ string "Initial kernel command string"
default ""
help
On some platforms, there is currently no way for the boot loader to
@@ -1199,6 +1214,7 @@
config TASK_SIZE
hex "Size of user task space" if TASK_SIZE_BOOL
default "0x80000000" if PPC_8xx
+ default "0xb0000000" if PPC_BOOK3S_32 && STRICT_KERNEL_RWX
default "0xc0000000"
endmenu
diff --git a/arch/powerpc/Makefile b/arch/powerpc/Makefile
index f310c32..3e8da9c 100644
--- a/arch/powerpc/Makefile
+++ b/arch/powerpc/Makefile
@@ -246,7 +246,8 @@
# often slow when they are implemented at all
KBUILD_CFLAGS += $(call cc-option,-mno-string)
-cpu-as-$(CONFIG_4xx) += -Wa,-m405
+cpu-as-$(CONFIG_40x) += -Wa,-m405
+cpu-as-$(CONFIG_44x) += -Wa,-m440
cpu-as-$(CONFIG_ALTIVEC) += $(call as-option,-Wa$(comma)-maltivec)
cpu-as-$(CONFIG_E200) += -Wa,-me200
cpu-as-$(CONFIG_E500) += -Wa,-me500
diff --git a/arch/powerpc/boot/Makefile b/arch/powerpc/boot/Makefile
index 63d7456..44af715 100644
--- a/arch/powerpc/boot/Makefile
+++ b/arch/powerpc/boot/Makefile
@@ -70,10 +70,10 @@
DTC_FLAGS ?= -p 1024
$(obj)/4xx.o: BOOTCFLAGS += -mcpu=405
-$(obj)/ebony.o: BOOTCFLAGS += -mcpu=405
+$(obj)/ebony.o: BOOTCFLAGS += -mcpu=440
$(obj)/cuboot-hotfoot.o: BOOTCFLAGS += -mcpu=405
-$(obj)/cuboot-taishan.o: BOOTCFLAGS += -mcpu=405
-$(obj)/cuboot-katmai.o: BOOTCFLAGS += -mcpu=405
+$(obj)/cuboot-taishan.o: BOOTCFLAGS += -mcpu=440
+$(obj)/cuboot-katmai.o: BOOTCFLAGS += -mcpu=440
$(obj)/cuboot-acadia.o: BOOTCFLAGS += -mcpu=405
$(obj)/treeboot-iss4xx.o: BOOTCFLAGS += -mcpu=405
$(obj)/treeboot-currituck.o: BOOTCFLAGS += -mcpu=405
@@ -117,7 +117,7 @@
elf_util.c $(zlib-y) devtree.c stdlib.c \
oflib.c ofconsole.c cuboot.c
-src-wlib-$(CONFIG_PPC_MPC52XX) += mpc52xx-psc.c
+src-wlib-$(CONFIG_PPC_MPC52xx) += mpc52xx-psc.c
src-wlib-$(CONFIG_PPC64_BOOT_WRAPPER) += opal-calls.S opal.c
ifndef CONFIG_PPC64_BOOT_WRAPPER
src-wlib-y += crtsavres.S
diff --git a/arch/powerpc/boot/dts/akebono.dts b/arch/powerpc/boot/dts/akebono.dts
index cd9d660..df18f8d 100644
--- a/arch/powerpc/boot/dts/akebono.dts
+++ b/arch/powerpc/boot/dts/akebono.dts
@@ -248,7 +248,7 @@
};
};
- PCIE0: pciex@10100000000 {
+ PCIE0: pcie@10100000000 {
device_type = "pci";
#interrupt-cells = <1>;
#size-cells = <2>;
@@ -288,7 +288,7 @@
0x0 0x0 0x0 0x4 &MPIC 48 0x2 /* int D */>;
};
- PCIE1: pciex@20100000000 {
+ PCIE1: pcie@20100000000 {
device_type = "pci";
#interrupt-cells = <1>;
#size-cells = <2>;
@@ -328,7 +328,7 @@
0x0 0x0 0x0 0x4 &MPIC 56 0x2 /* int D */>;
};
- PCIE2: pciex@18100000000 {
+ PCIE2: pcie@18100000000 {
device_type = "pci";
#interrupt-cells = <1>;
#size-cells = <2>;
@@ -368,7 +368,7 @@
0x0 0x0 0x0 0x4 &MPIC 64 0x2 /* int D */>;
};
- PCIE3: pciex@28100000000 {
+ PCIE3: pcie@28100000000 {
device_type = "pci";
#interrupt-cells = <1>;
#size-cells = <2>;
diff --git a/arch/powerpc/boot/dts/bluestone.dts b/arch/powerpc/boot/dts/bluestone.dts
index cc965a1..aa1ae94 100644
--- a/arch/powerpc/boot/dts/bluestone.dts
+++ b/arch/powerpc/boot/dts/bluestone.dts
@@ -325,7 +325,7 @@
};
};
- PCIE0: pciex@d00000000 {
+ PCIE0: pcie@d00000000 {
device_type = "pci";
#interrupt-cells = <1>;
#size-cells = <2>;
diff --git a/arch/powerpc/boot/dts/canyonlands.dts b/arch/powerpc/boot/dts/canyonlands.dts
index 0d6ac92..c5fbb08 100644
--- a/arch/powerpc/boot/dts/canyonlands.dts
+++ b/arch/powerpc/boot/dts/canyonlands.dts
@@ -461,7 +461,7 @@
interrupt-map = < 0x0 0x0 0x0 0x0 &UIC1 0x0 0x8 >;
};
- PCIE0: pciex@d00000000 {
+ PCIE0: pcie@d00000000 {
device_type = "pci";
#interrupt-cells = <1>;
#size-cells = <2>;
@@ -503,7 +503,7 @@
0x0 0x0 0x0 0x4 &UIC3 0xf 0x4 /* swizzled int D */>;
};
- PCIE1: pciex@d20000000 {
+ PCIE1: pcie@d20000000 {
device_type = "pci";
#interrupt-cells = <1>;
#size-cells = <2>;
diff --git a/arch/powerpc/boot/dts/currituck.dts b/arch/powerpc/boot/dts/currituck.dts
index b6d87b9..aea8af8 100644
--- a/arch/powerpc/boot/dts/currituck.dts
+++ b/arch/powerpc/boot/dts/currituck.dts
@@ -122,7 +122,7 @@
};
};
- PCIE0: pciex@10100000000 { // 4xGBIF1
+ PCIE0: pcie@10100000000 { // 4xGBIF1
device_type = "pci";
#interrupt-cells = <1>;
#size-cells = <2>;
@@ -160,7 +160,7 @@
0x0 0x0 0x0 0x4 &MPIC 49 0x2 /* int D */>;
};
- PCIE1: pciex@30100000000 { // 4xGBIF0
+ PCIE1: pcie@30100000000 { // 4xGBIF0
device_type = "pci";
#interrupt-cells = <1>;
#size-cells = <2>;
@@ -197,7 +197,7 @@
0x0 0x0 0x0 0x4 &MPIC 41 0x2 /* int D */>;
};
- PCIE2: pciex@38100000000 { // 2xGBIF0
+ PCIE2: pcie@38100000000 { // 2xGBIF0
device_type = "pci";
#interrupt-cells = <1>;
#size-cells = <2>;
diff --git a/arch/powerpc/boot/dts/fsl/p4080ds.dts b/arch/powerpc/boot/dts/fsl/p4080ds.dts
index 65e2015..969b32c 100644
--- a/arch/powerpc/boot/dts/fsl/p4080ds.dts
+++ b/arch/powerpc/boot/dts/fsl/p4080ds.dts
@@ -125,11 +125,11 @@
i2c@118100 {
eeprom@51 {
- compatible = "atmel,24c256";
+ compatible = "atmel,spd";
reg = <0x51>;
};
eeprom@52 {
- compatible = "atmel,24c256";
+ compatible = "atmel,spd";
reg = <0x52>;
};
rtc@68 {
@@ -143,6 +143,45 @@
};
};
+ i2c@118000 {
+ zl2006@21 {
+ compatible = "zl2006";
+ reg = <0x21>;
+ };
+ zl2006@22 {
+ compatible = "zl2006";
+ reg = <0x22>;
+ };
+ zl2006@23 {
+ compatible = "zl2006";
+ reg = <0x23>;
+ };
+ zl2006@24 {
+ compatible = "zl2006";
+ reg = <0x24>;
+ };
+ eeprom@50 {
+ compatible = "atmel,24c64";
+ reg = <0x50>;
+ };
+ eeprom@55 {
+ compatible = "atmel,24c64";
+ reg = <0x55>;
+ };
+ eeprom@56 {
+ compatible = "atmel,24c64";
+ reg = <0x56>;
+ };
+ eeprom@57 {
+ compatible = "atmel,24c02";
+ reg = <0x57>;
+ };
+ };
+
+ i2c@119100 {
+ /* 0x6E: ICS9FG108 */
+ };
+
usb0: usb@210000 {
phy_type = "ulpi";
};
diff --git a/arch/powerpc/boot/dts/glacier.dts b/arch/powerpc/boot/dts/glacier.dts
index a7a802f..e84ff1a 100644
--- a/arch/powerpc/boot/dts/glacier.dts
+++ b/arch/powerpc/boot/dts/glacier.dts
@@ -489,7 +489,7 @@
interrupt-map = < 0x0 0x0 0x0 0x0 &UIC1 0x0 0x8 >;
};
- PCIE0: pciex@d00000000 {
+ PCIE0: pcie@d00000000 {
device_type = "pci";
#interrupt-cells = <1>;
#size-cells = <2>;
@@ -531,7 +531,7 @@
0x0 0x0 0x0 0x4 &UIC3 0xf 0x4 /* swizzled int D */>;
};
- PCIE1: pciex@d20000000 {
+ PCIE1: pcie@d20000000 {
device_type = "pci";
#interrupt-cells = <1>;
#size-cells = <2>;
diff --git a/arch/powerpc/boot/dts/haleakala.dts b/arch/powerpc/boot/dts/haleakala.dts
index cb16dad..f81ce87 100644
--- a/arch/powerpc/boot/dts/haleakala.dts
+++ b/arch/powerpc/boot/dts/haleakala.dts
@@ -237,7 +237,7 @@
};
};
- PCIE0: pciex@a0000000 {
+ PCIE0: pcie@a0000000 {
device_type = "pci";
#interrupt-cells = <1>;
#size-cells = <2>;
diff --git a/arch/powerpc/boot/dts/icon.dts b/arch/powerpc/boot/dts/icon.dts
index 2e6e3a7..fbaa60b 100644
--- a/arch/powerpc/boot/dts/icon.dts
+++ b/arch/powerpc/boot/dts/icon.dts
@@ -315,7 +315,7 @@
interrupt-map = <0x0 0x0 0x0 0x0 &UIC1 19 0x8>;
};
- PCIE0: pciex@d00000000 {
+ PCIE0: pcie@d00000000 {
device_type = "pci";
#interrupt-cells = <1>;
#size-cells = <2>;
@@ -356,7 +356,7 @@
0x0 0x0 0x0 0x4 &UIC3 0x3 0x4 /* swizzled int D */>;
};
- PCIE1: pciex@d20000000 {
+ PCIE1: pcie@d20000000 {
device_type = "pci";
#interrupt-cells = <1>;
#size-cells = <2>;
diff --git a/arch/powerpc/boot/dts/katmai.dts b/arch/powerpc/boot/dts/katmai.dts
index 02629e1..a8f3532 100644
--- a/arch/powerpc/boot/dts/katmai.dts
+++ b/arch/powerpc/boot/dts/katmai.dts
@@ -319,7 +319,7 @@
>;
};
- PCIE0: pciex@d00000000 {
+ PCIE0: pcie@d00000000 {
device_type = "pci";
#interrupt-cells = <1>;
#size-cells = <2>;
@@ -360,7 +360,7 @@
0x0 0x0 0x0 0x4 &UIC3 0x3 0x4 /* swizzled int D */>;
};
- PCIE1: pciex@d20000000 {
+ PCIE1: pcie@d20000000 {
device_type = "pci";
#interrupt-cells = <1>;
#size-cells = <2>;
@@ -401,7 +401,7 @@
0x0 0x0 0x0 0x4 &UIC3 0x7 0x4 /* swizzled int D */>;
};
- PCIE2: pciex@d40000000 {
+ PCIE2: pcie@d40000000 {
device_type = "pci";
#interrupt-cells = <1>;
#size-cells = <2>;
diff --git a/arch/powerpc/boot/dts/kilauea.dts b/arch/powerpc/boot/dts/kilauea.dts
index 2a34132..a709fb4 100644
--- a/arch/powerpc/boot/dts/kilauea.dts
+++ b/arch/powerpc/boot/dts/kilauea.dts
@@ -322,7 +322,7 @@
};
};
- PCIE0: pciex@a0000000 {
+ PCIE0: pcie@a0000000 {
device_type = "pci";
#interrupt-cells = <1>;
#size-cells = <2>;
@@ -363,7 +363,7 @@
0x0 0x0 0x0 0x4 &UIC2 0x3 0x4 /* swizzled int D */>;
};
- PCIE1: pciex@c0000000 {
+ PCIE1: pcie@c0000000 {
device_type = "pci";
#interrupt-cells = <1>;
#size-cells = <2>;
diff --git a/arch/powerpc/boot/dts/makalu.dts b/arch/powerpc/boot/dts/makalu.dts
index bf8fe16..c473cd9 100644
--- a/arch/powerpc/boot/dts/makalu.dts
+++ b/arch/powerpc/boot/dts/makalu.dts
@@ -268,7 +268,7 @@
};
};
- PCIE0: pciex@a0000000 {
+ PCIE0: pcie@a0000000 {
device_type = "pci";
#interrupt-cells = <1>;
#size-cells = <2>;
@@ -309,7 +309,7 @@
0x0 0x0 0x0 0x4 &UIC2 0x3 0x4 /* swizzled int D */>;
};
- PCIE1: pciex@c0000000 {
+ PCIE1: pcie@c0000000 {
device_type = "pci";
#interrupt-cells = <1>;
#size-cells = <2>;
diff --git a/arch/powerpc/boot/dts/redwood.dts b/arch/powerpc/boot/dts/redwood.dts
index f3e046f..f38035a 100644
--- a/arch/powerpc/boot/dts/redwood.dts
+++ b/arch/powerpc/boot/dts/redwood.dts
@@ -235,7 +235,7 @@
has-new-stacr-staopc;
};
};
- PCIE0: pciex@d00000000 {
+ PCIE0: pcie@d00000000 {
device_type = "pci";
#interrupt-cells = <1>;
#size-cells = <2>;
@@ -276,7 +276,7 @@
0x0 0x0 0x0 0x4 &UIC3 0x3 0x4 /* swizzled int D */>;
};
- PCIE1: pciex@d20000000 {
+ PCIE1: pcie@d20000000 {
device_type = "pci";
#interrupt-cells = <1>;
#size-cells = <2>;
@@ -317,7 +317,7 @@
0x0 0x0 0x0 0x4 &UIC3 0x7 0x4 /* swizzled int D */>;
};
- PCIE2: pciex@d40000000 {
+ PCIE2: pcie@d40000000 {
device_type = "pci";
#interrupt-cells = <1>;
#size-cells = <2>;
diff --git a/arch/powerpc/boot/main.c b/arch/powerpc/boot/main.c
index a9d2091..cae31a6 100644
--- a/arch/powerpc/boot/main.c
+++ b/arch/powerpc/boot/main.c
@@ -104,7 +104,7 @@
{
/* If we have an image attached to us, it overrides anything
* supplied by the loader. */
- if (_initrd_end > _initrd_start) {
+ if (&_initrd_end > &_initrd_start) {
printf("Attached initrd image at 0x%p-0x%p\n\r",
_initrd_start, _initrd_end);
initrd_addr = (unsigned long)_initrd_start;
@@ -152,7 +152,7 @@
unsigned long esm_blob_addr, esm_blob_size;
/* Do we have an ESM (Enter Secure Mode) blob? */
- if (_esm_blob_end <= _esm_blob_start)
+ if (&_esm_blob_end <= &_esm_blob_start)
return;
printf("Attached ESM blob at 0x%p-0x%p\n\r",
diff --git a/arch/powerpc/boot/ps3.c b/arch/powerpc/boot/ps3.c
index c52552a..6e4efbd 100644
--- a/arch/powerpc/boot/ps3.c
+++ b/arch/powerpc/boot/ps3.c
@@ -127,7 +127,7 @@
ps3_repository_read_rm_size(&rm_size);
dt_fixup_memory(0, rm_size);
- if (_initrd_end > _initrd_start) {
+ if (&_initrd_end > &_initrd_start) {
setprop_val(chosen, "linux,initrd-start", (u32)(_initrd_start));
setprop_val(chosen, "linux,initrd-end", (u32)(_initrd_end));
}
diff --git a/arch/powerpc/boot/serial.c b/arch/powerpc/boot/serial.c
index 0bfa7e8..9a19e59 100644
--- a/arch/powerpc/boot/serial.c
+++ b/arch/powerpc/boot/serial.c
@@ -128,7 +128,7 @@
dt_is_compatible(devp, "fsl,cpm2-smc-uart"))
rc = cpm_console_init(devp, &serial_cd);
#endif
-#ifdef CONFIG_PPC_MPC52XX
+#ifdef CONFIG_PPC_MPC52xx
else if (dt_is_compatible(devp, "fsl,mpc5200-psc-uart"))
rc = mpc5200_psc_console_init(devp, &serial_cd);
#endif
diff --git a/arch/powerpc/configs/44x/akebono_defconfig b/arch/powerpc/configs/44x/akebono_defconfig
index 7705a5c..3894ba8 100644
--- a/arch/powerpc/configs/44x/akebono_defconfig
+++ b/arch/powerpc/configs/44x/akebono_defconfig
@@ -19,8 +19,6 @@
CONFIG_HZ_100=y
CONFIG_IRQ_ALL_CPUS=y
# CONFIG_COMPACTION is not set
-CONFIG_CMDLINE_BOOL=y
-CONFIG_CMDLINE=""
# CONFIG_SUSPEND is not set
CONFIG_NET=y
CONFIG_PACKET=y
@@ -58,7 +56,6 @@
# CONFIG_NET_VENDOR_DEC is not set
# CONFIG_NET_VENDOR_DLINK is not set
# CONFIG_NET_VENDOR_EMULEX is not set
-# CONFIG_NET_VENDOR_EXAR is not set
CONFIG_IBM_EMAC=y
# CONFIG_NET_VENDOR_MARVELL is not set
# CONFIG_NET_VENDOR_MELLANOX is not set
diff --git a/arch/powerpc/configs/44x/arches_defconfig b/arch/powerpc/configs/44x/arches_defconfig
index 82c6f49..41d04e7 100644
--- a/arch/powerpc/configs/44x/arches_defconfig
+++ b/arch/powerpc/configs/44x/arches_defconfig
@@ -11,8 +11,6 @@
# CONFIG_BLK_DEV_BSG is not set
# CONFIG_EBONY is not set
CONFIG_ARCHES=y
-CONFIG_CMDLINE_BOOL=y
-CONFIG_CMDLINE=""
CONFIG_NET=y
CONFIG_PACKET=y
CONFIG_UNIX=y
diff --git a/arch/powerpc/configs/44x/bamboo_defconfig b/arch/powerpc/configs/44x/bamboo_defconfig
index 6792132..acbce71 100644
--- a/arch/powerpc/configs/44x/bamboo_defconfig
+++ b/arch/powerpc/configs/44x/bamboo_defconfig
@@ -9,8 +9,6 @@
# CONFIG_BLK_DEV_BSG is not set
CONFIG_BAMBOO=y
# CONFIG_EBONY is not set
-CONFIG_CMDLINE_BOOL=y
-CONFIG_CMDLINE=""
CONFIG_NET=y
CONFIG_PACKET=y
CONFIG_UNIX=y
diff --git a/arch/powerpc/configs/44x/bluestone_defconfig b/arch/powerpc/configs/44x/bluestone_defconfig
index 8006a57..37088f2 100644
--- a/arch/powerpc/configs/44x/bluestone_defconfig
+++ b/arch/powerpc/configs/44x/bluestone_defconfig
@@ -11,8 +11,6 @@
# CONFIG_COMPAT_BRK is not set
CONFIG_BLUESTONE=y
# CONFIG_EBONY is not set
-CONFIG_CMDLINE_BOOL=y
-CONFIG_CMDLINE=""
CONFIG_NET=y
CONFIG_PACKET=y
CONFIG_UNIX=y
diff --git a/arch/powerpc/configs/44x/canyonlands_defconfig b/arch/powerpc/configs/44x/canyonlands_defconfig
index ccc14eb..61776ad 100644
--- a/arch/powerpc/configs/44x/canyonlands_defconfig
+++ b/arch/powerpc/configs/44x/canyonlands_defconfig
@@ -11,8 +11,6 @@
# CONFIG_BLK_DEV_BSG is not set
# CONFIG_EBONY is not set
CONFIG_CANYONLANDS=y
-CONFIG_CMDLINE_BOOL=y
-CONFIG_CMDLINE=""
CONFIG_NET=y
CONFIG_PACKET=y
CONFIG_UNIX=y
diff --git a/arch/powerpc/configs/44x/currituck_defconfig b/arch/powerpc/configs/44x/currituck_defconfig
index be76e06..34c86b3 100644
--- a/arch/powerpc/configs/44x/currituck_defconfig
+++ b/arch/powerpc/configs/44x/currituck_defconfig
@@ -17,8 +17,6 @@
CONFIG_HZ_100=y
CONFIG_MATH_EMULATION=y
CONFIG_IRQ_ALL_CPUS=y
-CONFIG_CMDLINE_BOOL=y
-CONFIG_CMDLINE=""
# CONFIG_SUSPEND is not set
CONFIG_NET=y
CONFIG_PACKET=y
diff --git a/arch/powerpc/configs/44x/eiger_defconfig b/arch/powerpc/configs/44x/eiger_defconfig
index 1abaa63..509300f 100644
--- a/arch/powerpc/configs/44x/eiger_defconfig
+++ b/arch/powerpc/configs/44x/eiger_defconfig
@@ -10,8 +10,6 @@
CONFIG_MODULE_UNLOAD=y
# CONFIG_EBONY is not set
CONFIG_EIGER=y
-CONFIG_CMDLINE_BOOL=y
-CONFIG_CMDLINE=""
CONFIG_PCIEPORTBUS=y
# CONFIG_PCIEASPM is not set
CONFIG_NET=y
diff --git a/arch/powerpc/configs/44x/fsp2_defconfig b/arch/powerpc/configs/44x/fsp2_defconfig
index e67fc04..30845ce 100644
--- a/arch/powerpc/configs/44x/fsp2_defconfig
+++ b/arch/powerpc/configs/44x/fsp2_defconfig
@@ -28,7 +28,6 @@
CONFIG_SWIOTLB=y
CONFIG_KEXEC=y
CONFIG_CRASH_DUMP=y
-CONFIG_CMDLINE_BOOL=y
CONFIG_CMDLINE="ip=on rw"
# CONFIG_SUSPEND is not set
# CONFIG_PCI is not set
diff --git a/arch/powerpc/configs/44x/icon_defconfig b/arch/powerpc/configs/44x/icon_defconfig
index 7d7ff84..930948a 100644
--- a/arch/powerpc/configs/44x/icon_defconfig
+++ b/arch/powerpc/configs/44x/icon_defconfig
@@ -9,8 +9,6 @@
# CONFIG_EBONY is not set
CONFIG_ICON=y
CONFIG_HIGHMEM=y
-CONFIG_CMDLINE_BOOL=y
-CONFIG_CMDLINE=""
CONFIG_PCIEPORTBUS=y
# CONFIG_PCIEASPM is not set
CONFIG_NET=y
diff --git a/arch/powerpc/configs/44x/iss476-smp_defconfig b/arch/powerpc/configs/44x/iss476-smp_defconfig
index fb5c73a..2c3834e 100644
--- a/arch/powerpc/configs/44x/iss476-smp_defconfig
+++ b/arch/powerpc/configs/44x/iss476-smp_defconfig
@@ -17,7 +17,6 @@
CONFIG_HZ_100=y
CONFIG_MATH_EMULATION=y
CONFIG_IRQ_ALL_CPUS=y
-CONFIG_CMDLINE_BOOL=y
CONFIG_CMDLINE="root=/dev/issblk0"
# CONFIG_PCI is not set
CONFIG_ADVANCED_OPTIONS=y
diff --git a/arch/powerpc/configs/44x/katmai_defconfig b/arch/powerpc/configs/44x/katmai_defconfig
index c6dc144..1a0f1c3 100644
--- a/arch/powerpc/configs/44x/katmai_defconfig
+++ b/arch/powerpc/configs/44x/katmai_defconfig
@@ -9,8 +9,6 @@
# CONFIG_BLK_DEV_BSG is not set
# CONFIG_EBONY is not set
CONFIG_KATMAI=y
-CONFIG_CMDLINE_BOOL=y
-CONFIG_CMDLINE=""
CONFIG_NET=y
CONFIG_PACKET=y
CONFIG_UNIX=y
diff --git a/arch/powerpc/configs/44x/rainier_defconfig b/arch/powerpc/configs/44x/rainier_defconfig
index c83ad031..6dd67de 100644
--- a/arch/powerpc/configs/44x/rainier_defconfig
+++ b/arch/powerpc/configs/44x/rainier_defconfig
@@ -10,8 +10,6 @@
# CONFIG_EBONY is not set
CONFIG_RAINIER=y
CONFIG_MATH_EMULATION=y
-CONFIG_CMDLINE_BOOL=y
-CONFIG_CMDLINE=""
CONFIG_NET=y
CONFIG_PACKET=y
CONFIG_UNIX=y
diff --git a/arch/powerpc/configs/44x/redwood_defconfig b/arch/powerpc/configs/44x/redwood_defconfig
index 640fe1d..e28d764 100644
--- a/arch/powerpc/configs/44x/redwood_defconfig
+++ b/arch/powerpc/configs/44x/redwood_defconfig
@@ -10,8 +10,6 @@
CONFIG_MODULE_UNLOAD=y
# CONFIG_EBONY is not set
CONFIG_REDWOOD=y
-CONFIG_CMDLINE_BOOL=y
-CONFIG_CMDLINE=""
CONFIG_PCIEPORTBUS=y
# CONFIG_PCIEASPM is not set
CONFIG_NET=y
diff --git a/arch/powerpc/configs/44x/sam440ep_defconfig b/arch/powerpc/configs/44x/sam440ep_defconfig
index 22dc0da..ef09786 100644
--- a/arch/powerpc/configs/44x/sam440ep_defconfig
+++ b/arch/powerpc/configs/44x/sam440ep_defconfig
@@ -12,8 +12,6 @@
CONFIG_AMIGA_PARTITION=y
# CONFIG_EBONY is not set
CONFIG_SAM440EP=y
-CONFIG_CMDLINE_BOOL=y
-CONFIG_CMDLINE=""
CONFIG_NET=y
CONFIG_PACKET=y
CONFIG_UNIX=y
diff --git a/arch/powerpc/configs/44x/sequoia_defconfig b/arch/powerpc/configs/44x/sequoia_defconfig
index 2c0973d..b4984ea 100644
--- a/arch/powerpc/configs/44x/sequoia_defconfig
+++ b/arch/powerpc/configs/44x/sequoia_defconfig
@@ -11,8 +11,6 @@
# CONFIG_BLK_DEV_BSG is not set
# CONFIG_EBONY is not set
CONFIG_SEQUOIA=y
-CONFIG_CMDLINE_BOOL=y
-CONFIG_CMDLINE=""
CONFIG_NET=y
CONFIG_PACKET=y
CONFIG_UNIX=y
diff --git a/arch/powerpc/configs/44x/taishan_defconfig b/arch/powerpc/configs/44x/taishan_defconfig
index a2d355c..3ea5932 100644
--- a/arch/powerpc/configs/44x/taishan_defconfig
+++ b/arch/powerpc/configs/44x/taishan_defconfig
@@ -9,8 +9,6 @@
# CONFIG_BLK_DEV_BSG is not set
# CONFIG_EBONY is not set
CONFIG_TAISHAN=y
-CONFIG_CMDLINE_BOOL=y
-CONFIG_CMDLINE=""
CONFIG_NET=y
CONFIG_PACKET=y
CONFIG_UNIX=y
diff --git a/arch/powerpc/configs/44x/warp_defconfig b/arch/powerpc/configs/44x/warp_defconfig
index af66c69..47252c2 100644
--- a/arch/powerpc/configs/44x/warp_defconfig
+++ b/arch/powerpc/configs/44x/warp_defconfig
@@ -14,7 +14,6 @@
CONFIG_WARP=y
CONFIG_PPC4xx_GPIO=y
CONFIG_HZ_1000=y
-CONFIG_CMDLINE_BOOL=y
CONFIG_CMDLINE="ip=on"
# CONFIG_PCI is not set
CONFIG_NET=y
diff --git a/arch/powerpc/configs/85xx-hw.config b/arch/powerpc/configs/85xx-hw.config
index b507df6..524db76 100644
--- a/arch/powerpc/configs/85xx-hw.config
+++ b/arch/powerpc/configs/85xx-hw.config
@@ -67,7 +67,6 @@
CONFIG_MTD_CFI_INTELEXT=y
CONFIG_MTD_CFI=y
CONFIG_MTD_CMDLINE_PARTS=y
-CONFIG_MTD_M25P80=y
CONFIG_MTD_NAND_FSL_ELBC=y
CONFIG_MTD_NAND_FSL_IFC=y
CONFIG_MTD_RAW_NAND=y
diff --git a/arch/powerpc/configs/85xx/xes_mpc85xx_defconfig b/arch/powerpc/configs/85xx/xes_mpc85xx_defconfig
index d50aca6..3a6381a 100644
--- a/arch/powerpc/configs/85xx/xes_mpc85xx_defconfig
+++ b/arch/powerpc/configs/85xx/xes_mpc85xx_defconfig
@@ -51,9 +51,6 @@
CONFIG_IP_MROUTE=y
CONFIG_IP_PIMSM_V1=y
CONFIG_IP_PIMSM_V2=y
-# CONFIG_INET_XFRM_MODE_TRANSPORT is not set
-# CONFIG_INET_XFRM_MODE_TUNNEL is not set
-# CONFIG_INET_XFRM_MODE_BEET is not set
CONFIG_MTD=y
CONFIG_MTD_REDBOOT_PARTS=y
CONFIG_MTD_CMDLINE_PARTS=y
diff --git a/arch/powerpc/configs/86xx-hw.config b/arch/powerpc/configs/86xx-hw.config
index 151164c..0cb24b3 100644
--- a/arch/powerpc/configs/86xx-hw.config
+++ b/arch/powerpc/configs/86xx-hw.config
@@ -32,8 +32,6 @@
CONFIG_HZ_1000=y
CONFIG_I2C_MPC=y
CONFIG_I2C=y
-# CONFIG_INET_XFRM_MODE_TRANSPORT is not set
-# CONFIG_INET_XFRM_MODE_TUNNEL is not set
CONFIG_INPUT_FF_MEMLESS=m
# CONFIG_INPUT_KEYBOARD is not set
# CONFIG_INPUT_MOUSEDEV is not set
diff --git a/arch/powerpc/configs/fsl-emb-nonhw.config b/arch/powerpc/configs/fsl-emb-nonhw.config
index 3c7dad1..df37efe 100644
--- a/arch/powerpc/configs/fsl-emb-nonhw.config
+++ b/arch/powerpc/configs/fsl-emb-nonhw.config
@@ -56,7 +56,6 @@
CONFIG_INET_AH=y
CONFIG_INET_ESP=y
CONFIG_INET_IPCOMP=y
-# CONFIG_INET_XFRM_MODE_BEET is not set
CONFIG_INET=y
CONFIG_IP_ADVANCED_ROUTER=y
CONFIG_IP_MROUTE=y
diff --git a/arch/powerpc/configs/g5_defconfig b/arch/powerpc/configs/g5_defconfig
index a68c7f3..1c674c4 100644
--- a/arch/powerpc/configs/g5_defconfig
+++ b/arch/powerpc/configs/g5_defconfig
@@ -51,7 +51,6 @@
CONFIG_NF_CONNTRACK_IRC=m
CONFIG_NF_CONNTRACK_TFTP=m
CONFIG_NF_CT_NETLINK=m
-CONFIG_NF_CONNTRACK_IPV4=m
CONFIG_DEVTMPFS=y
CONFIG_DEVTMPFS_MOUNT=y
CONFIG_BLK_DEV_LOOP=y
diff --git a/arch/powerpc/configs/holly_defconfig b/arch/powerpc/configs/holly_defconfig
index 067f433..271daff 100644
--- a/arch/powerpc/configs/holly_defconfig
+++ b/arch/powerpc/configs/holly_defconfig
@@ -13,7 +13,6 @@
CONFIG_PPC_HOLLY=y
CONFIG_GEN_RTC=y
CONFIG_BINFMT_MISC=y
-CONFIG_CMDLINE_BOOL=y
CONFIG_CMDLINE="console=ttyS0,115200"
# CONFIG_SECCOMP is not set
CONFIG_NET=y
diff --git a/arch/powerpc/configs/linkstation_defconfig b/arch/powerpc/configs/linkstation_defconfig
index ea59f3d..d4be64f 100644
--- a/arch/powerpc/configs/linkstation_defconfig
+++ b/arch/powerpc/configs/linkstation_defconfig
@@ -37,7 +37,6 @@
CONFIG_NETFILTER_XT_MATCH_MAC=m
CONFIG_NETFILTER_XT_MATCH_PKTTYPE=m
CONFIG_NETFILTER_XT_MATCH_STATE=m
-CONFIG_NF_CONNTRACK_IPV4=m
CONFIG_IP_NF_IPTABLES=m
CONFIG_IP_NF_FILTER=m
CONFIG_IP_NF_TARGET_REJECT=m
diff --git a/arch/powerpc/configs/mpc512x_defconfig b/arch/powerpc/configs/mpc512x_defconfig
index e39346b..e75d3f3 100644
--- a/arch/powerpc/configs/mpc512x_defconfig
+++ b/arch/powerpc/configs/mpc512x_defconfig
@@ -47,7 +47,6 @@
CONFIG_BLK_DEV_RAM=y
CONFIG_BLK_DEV_RAM_COUNT=1
CONFIG_BLK_DEV_RAM_SIZE=8192
-CONFIG_BLK_DEV_RAM_DAX=y
CONFIG_EEPROM_AT24=y
CONFIG_EEPROM_AT25=y
CONFIG_SCSI=y
diff --git a/arch/powerpc/configs/mpc83xx_defconfig b/arch/powerpc/configs/mpc83xx_defconfig
index be12572..95d43f8 100644
--- a/arch/powerpc/configs/mpc83xx_defconfig
+++ b/arch/powerpc/configs/mpc83xx_defconfig
@@ -19,7 +19,6 @@
CONFIG_MPC836x_RDK=y
CONFIG_MPC837x_MDS=y
CONFIG_MPC837x_RDB=y
-CONFIG_SBC834x=y
CONFIG_ASP834x=y
CONFIG_QE_GPIO=y
CONFIG_MATH_EMULATION=y
diff --git a/arch/powerpc/configs/mvme5100_defconfig b/arch/powerpc/configs/mvme5100_defconfig
index 0a0d046..1fed6be 100644
--- a/arch/powerpc/configs/mvme5100_defconfig
+++ b/arch/powerpc/configs/mvme5100_defconfig
@@ -20,10 +20,9 @@
CONFIG_MVME5100=y
CONFIG_KVM_GUEST=y
CONFIG_HZ_100=y
+CONFIG_CMDLINE="console=ttyS0,9600 ip=dhcp root=/dev/nfs"
# CONFIG_CORE_DUMP_DEFAULT_ELF_HEADERS is not set
# CONFIG_COMPACTION is not set
-CONFIG_CMDLINE_BOOL=y
-CONFIG_CMDLINE="console=ttyS0,9600 ip=dhcp root=/dev/nfs"
CONFIG_NET=y
CONFIG_PACKET=y
CONFIG_UNIX=y
@@ -46,7 +45,6 @@
CONFIG_NETFILTER_XT_MATCH_MAC=m
CONFIG_NETFILTER_XT_MATCH_PKTTYPE=m
CONFIG_NETFILTER_XT_MATCH_STATE=m
-CONFIG_NF_CONNTRACK_IPV4=m
CONFIG_IP_NF_IPTABLES=m
CONFIG_IP_NF_FILTER=m
CONFIG_IP_NF_TARGET_REJECT=m
diff --git a/arch/powerpc/configs/pasemi_defconfig b/arch/powerpc/configs/pasemi_defconfig
index 08b7f4c..af9af03 100644
--- a/arch/powerpc/configs/pasemi_defconfig
+++ b/arch/powerpc/configs/pasemi_defconfig
@@ -58,7 +58,6 @@
CONFIG_EEPROM_LEGACY=y
CONFIG_BLK_DEV_SD=y
CONFIG_CHR_DEV_ST=y
-CONFIG_CHR_DEV_OSST=y
CONFIG_BLK_DEV_SR=y
CONFIG_CHR_DEV_SG=y
CONFIG_CHR_DEV_SCH=y
diff --git a/arch/powerpc/configs/pmac32_defconfig b/arch/powerpc/configs/pmac32_defconfig
index 05e325c..665a8d7 100644
--- a/arch/powerpc/configs/pmac32_defconfig
+++ b/arch/powerpc/configs/pmac32_defconfig
@@ -75,7 +75,6 @@
CONFIG_NETFILTER_XT_MATCH_TCPMSS=m
CONFIG_NETFILTER_XT_MATCH_TIME=m
CONFIG_NETFILTER_XT_MATCH_U32=m
-CONFIG_NF_CONNTRACK_IPV4=m
CONFIG_IP_NF_IPTABLES=m
CONFIG_IP_NF_MATCH_AH=m
CONFIG_IP_NF_MATCH_ECN=m
@@ -90,13 +89,6 @@
CONFIG_IP_NF_ARPFILTER=m
CONFIG_IP_NF_ARP_MANGLE=m
CONFIG_IP_DCCP=m
-CONFIG_IRDA=m
-CONFIG_IRLAN=m
-CONFIG_IRNET=m
-CONFIG_IRCOMM=m
-CONFIG_IRDA_CACHE_LAST_LSAP=y
-CONFIG_IRDA_FAST_RR=y
-CONFIG_IRTTY_SIR=m
CONFIG_BT=m
CONFIG_BT_RFCOMM=m
CONFIG_BT_RFCOMM_TTY=y
diff --git a/arch/powerpc/configs/powernv_defconfig b/arch/powerpc/configs/powernv_defconfig
index 2de9aadf..cf30fc2 100644
--- a/arch/powerpc/configs/powernv_defconfig
+++ b/arch/powerpc/configs/powernv_defconfig
@@ -64,7 +64,6 @@
CONFIG_TRANSPARENT_HUGEPAGE=y
CONFIG_DEFERRED_STRUCT_PAGE_INIT=y
CONFIG_PPC_64K_PAGES=y
-CONFIG_PPC_SUBPAGE_PROT=y
CONFIG_SCHED_SMT=y
CONFIG_PM=y
CONFIG_HOTPLUG_PCI=y
@@ -246,7 +245,6 @@
CONFIG_INFINIBAND_USER_MAD=m
CONFIG_INFINIBAND_USER_ACCESS=m
CONFIG_INFINIBAND_MTHCA=m
-CONFIG_INFINIBAND_CXGB3=m
CONFIG_INFINIBAND_CXGB4=m
CONFIG_MLX4_INFINIBAND=m
CONFIG_INFINIBAND_IPOIB=m
diff --git a/arch/powerpc/configs/ppc40x_defconfig b/arch/powerpc/configs/ppc40x_defconfig
index 25f6c91..7e48693 100644
--- a/arch/powerpc/configs/ppc40x_defconfig
+++ b/arch/powerpc/configs/ppc40x_defconfig
@@ -20,9 +20,6 @@
CONFIG_IP_PNP=y
CONFIG_IP_PNP_DHCP=y
CONFIG_IP_PNP_BOOTP=y
-# CONFIG_INET_XFRM_MODE_TRANSPORT is not set
-# CONFIG_INET_XFRM_MODE_TUNNEL is not set
-# CONFIG_INET_XFRM_MODE_BEET is not set
CONFIG_CONNECTOR=y
CONFIG_MTD=y
CONFIG_MTD_CMDLINE_PARTS=y
diff --git a/arch/powerpc/configs/ppc64_defconfig b/arch/powerpc/configs/ppc64_defconfig
index 8d7e3e9..4875965 100644
--- a/arch/powerpc/configs/ppc64_defconfig
+++ b/arch/powerpc/configs/ppc64_defconfig
@@ -268,7 +268,6 @@
CONFIG_INFINIBAND_USER_MAD=m
CONFIG_INFINIBAND_USER_ACCESS=m
CONFIG_INFINIBAND_MTHCA=m
-CONFIG_INFINIBAND_CXGB3=m
CONFIG_INFINIBAND_CXGB4=m
CONFIG_MLX4_INFINIBAND=m
CONFIG_INFINIBAND_IPOIB=m
diff --git a/arch/powerpc/configs/ppc6xx_defconfig b/arch/powerpc/configs/ppc6xx_defconfig
index feb5d47..5e6f92b 100644
--- a/arch/powerpc/configs/ppc6xx_defconfig
+++ b/arch/powerpc/configs/ppc6xx_defconfig
@@ -53,7 +53,6 @@
CONFIG_MPC836x_RDK=y
CONFIG_MPC837x_MDS=y
CONFIG_MPC837x_RDB=y
-CONFIG_SBC834x=y
CONFIG_ASP834x=y
CONFIG_PPC_86xx=y
CONFIG_MPC8641_HPCN=y
@@ -187,7 +186,6 @@
CONFIG_NETFILTER_XT_MATCH_TCPMSS=m
CONFIG_NETFILTER_XT_MATCH_TIME=m
CONFIG_NETFILTER_XT_MATCH_U32=m
-CONFIG_NF_CONNTRACK_IPV4=m
CONFIG_IP_NF_IPTABLES=m
CONFIG_IP_NF_MATCH_AH=m
CONFIG_IP_NF_MATCH_ECN=m
@@ -203,7 +201,6 @@
CONFIG_IP_NF_ARPTABLES=m
CONFIG_IP_NF_ARPFILTER=m
CONFIG_IP_NF_ARP_MANGLE=m
-CONFIG_NF_CONNTRACK_IPV6=m
CONFIG_IP6_NF_IPTABLES=m
CONFIG_IP6_NF_MATCH_AH=m
CONFIG_IP6_NF_MATCH_EUI64=m
@@ -241,7 +238,6 @@
CONFIG_BRIDGE_EBT_LOG=m
CONFIG_BRIDGE_EBT_NFLOG=m
CONFIG_IP_DCCP=m
-CONFIG_NET_DCCPPROBE=m
CONFIG_TIPC=m
CONFIG_ATM=m
CONFIG_ATM_CLIP=m
@@ -251,7 +247,6 @@
CONFIG_VLAN_8021Q=m
CONFIG_DECNET=m
CONFIG_DECNET_ROUTER=y
-CONFIG_IPX=m
CONFIG_ATALK=m
CONFIG_DEV_APPLETALK=m
CONFIG_IPDDP=m
@@ -297,26 +292,6 @@
CONFIG_NET_ACT_PEDIT=m
CONFIG_NET_ACT_SIMP=m
CONFIG_NET_ACT_SKBEDIT=m
-CONFIG_IRDA=m
-CONFIG_IRLAN=m
-CONFIG_IRNET=m
-CONFIG_IRCOMM=m
-CONFIG_IRDA_CACHE_LAST_LSAP=y
-CONFIG_IRDA_FAST_RR=y
-CONFIG_IRTTY_SIR=m
-CONFIG_KINGSUN_DONGLE=m
-CONFIG_KSDAZZLE_DONGLE=m
-CONFIG_KS959_DONGLE=m
-CONFIG_USB_IRDA=m
-CONFIG_SIGMATEL_FIR=m
-CONFIG_NSC_FIR=m
-CONFIG_WINBOND_FIR=m
-CONFIG_TOSHIBA_FIR=m
-CONFIG_SMC_IRCC_FIR=m
-CONFIG_ALI_FIR=m
-CONFIG_VLSI_FIR=m
-CONFIG_VIA_FIR=m
-CONFIG_MCS_FIR=m
CONFIG_BT=m
CONFIG_BT_RFCOMM=m
CONFIG_BT_RFCOMM_TTY=y
@@ -332,7 +307,6 @@
CONFIG_BT_HCIDTL1=m
CONFIG_BT_HCIBT3C=m
CONFIG_BT_HCIBLUECARD=m
-CONFIG_BT_HCIBTUART=m
CONFIG_BT_HCIVHCI=m
CONFIG_CFG80211=m
CONFIG_MAC80211=m
@@ -366,7 +340,6 @@
CONFIG_RAID_ATTRS=m
CONFIG_BLK_DEV_SD=y
CONFIG_CHR_DEV_ST=m
-CONFIG_CHR_DEV_OSST=m
CONFIG_BLK_DEV_SR=m
CONFIG_CHR_DEV_SG=y
CONFIG_CHR_DEV_SCH=m
@@ -663,7 +636,6 @@
CONFIG_I2C_PCA_PLATFORM=m
CONFIG_I2C_SIMTEC=m
CONFIG_I2C_PARPORT=m
-CONFIG_I2C_PARPORT_LIGHT=m
CONFIG_I2C_TINY_USB=m
CONFIG_I2C_PCA_ISA=m
CONFIG_I2C_STUB=m
@@ -676,7 +648,6 @@
CONFIG_W1_SLAVE_SMEM=m
CONFIG_W1_SLAVE_DS2433=m
CONFIG_W1_SLAVE_DS2433_CRC=y
-CONFIG_W1_SLAVE_DS2760=m
CONFIG_APM_POWER=m
CONFIG_BATTERY_PMU=m
CONFIG_HWMON=m
@@ -1065,15 +1036,6 @@
CONFIG_CIFS_XATTR=y
CONFIG_CIFS_POSIX=y
CONFIG_CIFS_DFS_UPCALL=y
-CONFIG_NCP_FS=m
-CONFIG_NCPFS_PACKET_SIGNING=y
-CONFIG_NCPFS_IOCTL_LOCKING=y
-CONFIG_NCPFS_STRONG=y
-CONFIG_NCPFS_NFS_NS=y
-CONFIG_NCPFS_OS2_NS=y
-CONFIG_NCPFS_SMALLDOS=y
-CONFIG_NCPFS_NLS=y
-CONFIG_NCPFS_EXTRAS=y
CONFIG_CODA_FS=m
CONFIG_9P_FS=m
CONFIG_NLS_DEFAULT="utf8"
@@ -1117,7 +1079,6 @@
CONFIG_DEBUG_INFO=y
CONFIG_UNUSED_SYMBOLS=y
CONFIG_HEADERS_INSTALL=y
-CONFIG_HEADERS_CHECK=y
CONFIG_MAGIC_SYSRQ=y
CONFIG_DEBUG_KERNEL=y
CONFIG_DEBUG_OBJECTS=y
diff --git a/arch/powerpc/configs/ps3_defconfig b/arch/powerpc/configs/ps3_defconfig
index 81b55c8..142f132 100644
--- a/arch/powerpc/configs/ps3_defconfig
+++ b/arch/powerpc/configs/ps3_defconfig
@@ -34,8 +34,6 @@
# CONFIG_SPARSEMEM_VMEMMAP is not set
# CONFIG_COMPACTION is not set
CONFIG_SCHED_SMT=y
-CONFIG_CMDLINE_BOOL=y
-CONFIG_CMDLINE=""
CONFIG_PM=y
CONFIG_PM_DEBUG=y
# CONFIG_SECCOMP is not set
diff --git a/arch/powerpc/configs/pseries_defconfig b/arch/powerpc/configs/pseries_defconfig
index dfa4a72..d5dece9 100644
--- a/arch/powerpc/configs/pseries_defconfig
+++ b/arch/powerpc/configs/pseries_defconfig
@@ -57,7 +57,6 @@
CONFIG_KSM=y
CONFIG_TRANSPARENT_HUGEPAGE=y
CONFIG_PPC_64K_PAGES=y
-CONFIG_PPC_SUBPAGE_PROT=y
CONFIG_SCHED_SMT=y
CONFIG_HOTPLUG_PCI=y
CONFIG_HOTPLUG_PCI_RPA=m
@@ -94,6 +93,7 @@
CONFIG_BLK_DEV_RAM=y
CONFIG_BLK_DEV_RAM_SIZE=65536
CONFIG_VIRTIO_BLK=m
+CONFIG_BLK_DEV_NVME=y
CONFIG_BLK_DEV_SD=y
CONFIG_CHR_DEV_ST=m
CONFIG_BLK_DEV_SR=y
@@ -224,7 +224,6 @@
CONFIG_INFINIBAND_USER_MAD=m
CONFIG_INFINIBAND_USER_ACCESS=m
CONFIG_INFINIBAND_MTHCA=m
-CONFIG_INFINIBAND_CXGB3=m
CONFIG_INFINIBAND_CXGB4=m
CONFIG_MLX4_INFINIBAND=m
CONFIG_INFINIBAND_IPOIB=m
diff --git a/arch/powerpc/configs/skiroot_defconfig b/arch/powerpc/configs/skiroot_defconfig
index ad6739a..b806a5d 100644
--- a/arch/powerpc/configs/skiroot_defconfig
+++ b/arch/powerpc/configs/skiroot_defconfig
@@ -45,7 +45,6 @@
CONFIG_NUMA=y
CONFIG_PPC_64K_PAGES=y
CONFIG_SCHED_SMT=y
-CONFIG_CMDLINE_BOOL=y
CONFIG_CMDLINE="console=tty0 console=hvc0 ipr.fast_reboot=1 quiet"
# CONFIG_SECCOMP is not set
# CONFIG_PPC_MEM_KEYS is not set
diff --git a/arch/powerpc/configs/storcenter_defconfig b/arch/powerpc/configs/storcenter_defconfig
index b964084..47dcfad 100644
--- a/arch/powerpc/configs/storcenter_defconfig
+++ b/arch/powerpc/configs/storcenter_defconfig
@@ -12,7 +12,6 @@
CONFIG_STORCENTER=y
CONFIG_HZ_100=y
CONFIG_BINFMT_MISC=y
-CONFIG_CMDLINE_BOOL=y
CONFIG_CMDLINE="console=ttyS0,115200"
# CONFIG_SECCOMP is not set
CONFIG_NET=y
diff --git a/arch/powerpc/crypto/crc32-vpmsum_core.S b/arch/powerpc/crypto/crc32-vpmsum_core.S
index c3524eb..a16a717 100644
--- a/arch/powerpc/crypto/crc32-vpmsum_core.S
+++ b/arch/powerpc/crypto/crc32-vpmsum_core.S
@@ -19,7 +19,7 @@
* We then use fixed point Barrett reduction to compute a mod n over GF(2)
* for n = CRC using POWER8 instructions. We use x = 32.
*
- * http://en.wikipedia.org/wiki/Barrett_reduction
+ * https://en.wikipedia.org/wiki/Barrett_reduction
*
* Copyright (C) 2015 Anton Blanchard <anton@au.ibm.com>, IBM
*/
diff --git a/arch/powerpc/include/asm/Kbuild b/arch/powerpc/include/asm/Kbuild
index 2d444d0..90cd5c5 100644
--- a/arch/powerpc/include/asm/Kbuild
+++ b/arch/powerpc/include/asm/Kbuild
@@ -7,5 +7,6 @@
generic-y += kvm_types.h
generic-y += local64.h
generic-y += mcs_spinlock.h
+generic-y += qrwlock.h
generic-y += vtime.h
generic-y += early_ioremap.h
diff --git a/arch/powerpc/include/asm/asm-prototypes.h b/arch/powerpc/include/asm/asm-prototypes.h
index 7d81e86..de14b1a 100644
--- a/arch/powerpc/include/asm/asm-prototypes.h
+++ b/arch/powerpc/include/asm/asm-prototypes.h
@@ -98,7 +98,7 @@
void __init machine_init(u64 dt_ptr);
#endif
long system_call_exception(long r3, long r4, long r5, long r6, long r7, long r8, unsigned long r0, struct pt_regs *regs);
-notrace unsigned long syscall_exit_prepare(unsigned long r3, struct pt_regs *regs);
+notrace unsigned long syscall_exit_prepare(unsigned long r3, struct pt_regs *regs, long scv);
notrace unsigned long interrupt_exit_user_prepare(struct pt_regs *regs, unsigned long msr);
notrace unsigned long interrupt_exit_kernel_prepare(struct pt_regs *regs, unsigned long msr);
@@ -144,13 +144,13 @@
void _kvmppc_save_tm_pr(struct kvm_vcpu *vcpu, u64 guest_msr);
/* Patch sites */
-extern s32 patch__call_flush_count_cache;
+extern s32 patch__call_flush_branch_caches;
extern s32 patch__flush_count_cache_return;
extern s32 patch__flush_link_stack_return;
extern s32 patch__call_kvm_flush_link_stack;
extern s32 patch__memset_nocache, patch__memcpy_nocache;
-extern long flush_count_cache;
+extern long flush_branch_caches;
extern long kvm_flush_link_stack;
#ifdef CONFIG_PPC_TRANSACTIONAL_MEM
diff --git a/arch/powerpc/include/asm/atomic.h b/arch/powerpc/include/asm/atomic.h
index 0311c3c..8a55eb8 100644
--- a/arch/powerpc/include/asm/atomic.h
+++ b/arch/powerpc/include/asm/atomic.h
@@ -191,6 +191,34 @@
#define atomic_xchg(v, new) (xchg(&((v)->counter), new))
#define atomic_xchg_relaxed(v, new) xchg_relaxed(&((v)->counter), (new))
+/*
+ * Don't want to override the generic atomic_try_cmpxchg_acquire, because
+ * we add a lock hint to the lwarx, which may not be wanted for the
+ * _acquire case (and is not used by the other _acquire variants so it
+ * would be a surprise).
+ */
+static __always_inline bool
+atomic_try_cmpxchg_lock(atomic_t *v, int *old, int new)
+{
+ int r, o = *old;
+
+ __asm__ __volatile__ (
+"1:\t" PPC_LWARX(%0,0,%2,1) " # atomic_try_cmpxchg_acquire \n"
+" cmpw 0,%0,%3 \n"
+" bne- 2f \n"
+" stwcx. %4,0,%2 \n"
+" bne- 1b \n"
+"\t" PPC_ACQUIRE_BARRIER " \n"
+"2: \n"
+ : "=&r" (r), "+m" (v->counter)
+ : "r" (&v->counter), "r" (o), "r" (new)
+ : "cr0", "memory");
+
+ if (unlikely(r != o))
+ *old = r;
+ return likely(r == o);
+}
+
/**
* atomic_fetch_add_unless - add unless the number is a given value
* @v: pointer of type atomic_t
diff --git a/arch/powerpc/include/asm/barrier.h b/arch/powerpc/include/asm/barrier.h
index 123adce..f53c423 100644
--- a/arch/powerpc/include/asm/barrier.h
+++ b/arch/powerpc/include/asm/barrier.h
@@ -7,6 +7,10 @@
#include <asm/asm-const.h>
+#ifndef __ASSEMBLY__
+#include <asm/ppc-opcode.h>
+#endif
+
/*
* Memory barrier.
* The sync instruction guarantees that all memory accesses initiated
@@ -76,6 +80,22 @@
___p1; \
})
+#ifdef CONFIG_PPC64
+#define smp_cond_load_relaxed(ptr, cond_expr) ({ \
+ typeof(ptr) __PTR = (ptr); \
+ __unqual_scalar_typeof(*ptr) VAL; \
+ VAL = READ_ONCE(*__PTR); \
+ if (unlikely(!(cond_expr))) { \
+ spin_begin(); \
+ do { \
+ VAL = READ_ONCE(*__PTR); \
+ } while (!(cond_expr)); \
+ spin_end(); \
+ } \
+ (typeof(*ptr))VAL; \
+})
+#endif
+
#ifdef CONFIG_PPC_BOOK3S_64
#define NOSPEC_BARRIER_SLOT nop
#elif defined(CONFIG_PPC_FSL_BOOK3E)
@@ -97,6 +117,15 @@
#define barrier_nospec()
#endif /* CONFIG_PPC_BARRIER_NOSPEC */
+/*
+ * pmem_wmb() ensures that all stores for which the modification
+ * are written to persistent storage by preceding dcbfps/dcbstps
+ * instructions have updated persistent storage before any data
+ * access or data transfer caused by subsequent instructions is
+ * initiated.
+ */
+#define pmem_wmb() __asm__ __volatile__(PPC_PHWSYNC ::: "memory")
+
#include <asm-generic/barrier.h>
#endif /* _ASM_POWERPC_BARRIER_H */
diff --git a/arch/powerpc/include/asm/book3s/32/pgtable.h b/arch/powerpc/include/asm/book3s/32/pgtable.h
index 2249124..36443cd 100644
--- a/arch/powerpc/include/asm/book3s/32/pgtable.h
+++ b/arch/powerpc/include/asm/book3s/32/pgtable.h
@@ -184,17 +184,7 @@
*/
#define VMALLOC_OFFSET (0x1000000) /* 16M */
-/*
- * With CONFIG_STRICT_KERNEL_RWX, kernel segments are set NX. But when modules
- * are used, NX cannot be set on VMALLOC space. So vmalloc VM space and linear
- * memory shall not share segments.
- */
-#if defined(CONFIG_STRICT_KERNEL_RWX) && defined(CONFIG_MODULES)
-#define VMALLOC_START ((ALIGN((long)high_memory, 256L << 20) + VMALLOC_OFFSET) & \
- ~(VMALLOC_OFFSET - 1))
-#else
#define VMALLOC_START ((((long)high_memory + VMALLOC_OFFSET) & ~(VMALLOC_OFFSET-1)))
-#endif
#ifdef CONFIG_KASAN_VMALLOC
#define VMALLOC_END ALIGN_DOWN(ioremap_bot, PAGE_SIZE << KASAN_SHADOW_SCALE_SHIFT)
@@ -202,6 +192,11 @@
#define VMALLOC_END ioremap_bot
#endif
+#ifdef CONFIG_STRICT_KERNEL_RWX
+#define MODULES_END ALIGN_DOWN(PAGE_OFFSET, SZ_256M)
+#define MODULES_VADDR (MODULES_END - SZ_256M)
+#endif
+
#ifndef __ASSEMBLY__
#include <linux/sched.h>
#include <linux/threads.h>
diff --git a/arch/powerpc/include/asm/book3s/64/hash-4k.h b/arch/powerpc/include/asm/book3s/64/hash-4k.h
index 3f9ae35..082b988 100644
--- a/arch/powerpc/include/asm/book3s/64/hash-4k.h
+++ b/arch/powerpc/include/asm/book3s/64/hash-4k.h
@@ -34,11 +34,11 @@
#define H_PUD_TABLE_SIZE (sizeof(pud_t) << H_PUD_INDEX_SIZE)
#define H_PGD_TABLE_SIZE (sizeof(pgd_t) << H_PGD_INDEX_SIZE)
-#define H_PAGE_F_GIX_SHIFT 53
-#define H_PAGE_F_SECOND _RPAGE_RPN44 /* HPTE is in 2ndary HPTEG */
-#define H_PAGE_F_GIX (_RPAGE_RPN43 | _RPAGE_RPN42 | _RPAGE_RPN41)
-#define H_PAGE_BUSY _RPAGE_RSV1 /* software: PTE & hash are busy */
-#define H_PAGE_HASHPTE _RPAGE_RSV2 /* software: PTE & hash are busy */
+#define H_PAGE_F_GIX_SHIFT _PAGE_PA_MAX
+#define H_PAGE_F_SECOND _RPAGE_PKEY_BIT0 /* HPTE is in 2ndary HPTEG */
+#define H_PAGE_F_GIX (_RPAGE_RPN43 | _RPAGE_RPN42 | _RPAGE_RPN41)
+#define H_PAGE_BUSY _RPAGE_RSV1
+#define H_PAGE_HASHPTE _RPAGE_PKEY_BIT4
/* PTE flags to conserve for HPTE identification */
#define _PAGE_HPTEFLAGS (H_PAGE_BUSY | H_PAGE_HASHPTE | \
@@ -57,11 +57,12 @@
#define H_PMD_FRAG_NR (PAGE_SIZE >> H_PMD_FRAG_SIZE_SHIFT)
/* memory key bits, only 8 keys supported */
-#define H_PTE_PKEY_BIT0 0
-#define H_PTE_PKEY_BIT1 0
-#define H_PTE_PKEY_BIT2 _RPAGE_RSV3
-#define H_PTE_PKEY_BIT3 _RPAGE_RSV4
-#define H_PTE_PKEY_BIT4 _RPAGE_RSV5
+#define H_PTE_PKEY_BIT4 0
+#define H_PTE_PKEY_BIT3 0
+#define H_PTE_PKEY_BIT2 _RPAGE_PKEY_BIT3
+#define H_PTE_PKEY_BIT1 _RPAGE_PKEY_BIT2
+#define H_PTE_PKEY_BIT0 _RPAGE_PKEY_BIT1
+
/*
* On all 4K setups, remap_4k_pfn() equates to remap_pfn_range()
diff --git a/arch/powerpc/include/asm/book3s/64/hash-64k.h b/arch/powerpc/include/asm/book3s/64/hash-64k.h
index 0729c03..f20de11 100644
--- a/arch/powerpc/include/asm/book3s/64/hash-64k.h
+++ b/arch/powerpc/include/asm/book3s/64/hash-64k.h
@@ -32,15 +32,15 @@
*/
#define H_PAGE_COMBO _RPAGE_RPN0 /* this is a combo 4k page */
#define H_PAGE_4K_PFN _RPAGE_RPN1 /* PFN is for a single 4k page */
-#define H_PAGE_BUSY _RPAGE_RPN44 /* software: PTE & hash are busy */
+#define H_PAGE_BUSY _RPAGE_RSV1 /* software: PTE & hash are busy */
#define H_PAGE_HASHPTE _RPAGE_RPN43 /* PTE has associated HPTE */
/* memory key bits. */
-#define H_PTE_PKEY_BIT0 _RPAGE_RSV1
-#define H_PTE_PKEY_BIT1 _RPAGE_RSV2
-#define H_PTE_PKEY_BIT2 _RPAGE_RSV3
-#define H_PTE_PKEY_BIT3 _RPAGE_RSV4
-#define H_PTE_PKEY_BIT4 _RPAGE_RSV5
+#define H_PTE_PKEY_BIT4 _RPAGE_PKEY_BIT4
+#define H_PTE_PKEY_BIT3 _RPAGE_PKEY_BIT3
+#define H_PTE_PKEY_BIT2 _RPAGE_PKEY_BIT2
+#define H_PTE_PKEY_BIT1 _RPAGE_PKEY_BIT1
+#define H_PTE_PKEY_BIT0 _RPAGE_PKEY_BIT0
/*
* We need to differentiate between explicit huge page and THP huge
diff --git a/arch/powerpc/include/asm/book3s/64/hash-pkey.h b/arch/powerpc/include/asm/book3s/64/hash-pkey.h
new file mode 100644
index 0000000..7950108
--- /dev/null
+++ b/arch/powerpc/include/asm/book3s/64/hash-pkey.h
@@ -0,0 +1,32 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#ifndef _ASM_POWERPC_BOOK3S_64_HASH_PKEY_H
+#define _ASM_POWERPC_BOOK3S_64_HASH_PKEY_H
+
+static inline u64 hash__vmflag_to_pte_pkey_bits(u64 vm_flags)
+{
+ return (((vm_flags & VM_PKEY_BIT0) ? H_PTE_PKEY_BIT0 : 0x0UL) |
+ ((vm_flags & VM_PKEY_BIT1) ? H_PTE_PKEY_BIT1 : 0x0UL) |
+ ((vm_flags & VM_PKEY_BIT2) ? H_PTE_PKEY_BIT2 : 0x0UL) |
+ ((vm_flags & VM_PKEY_BIT3) ? H_PTE_PKEY_BIT3 : 0x0UL) |
+ ((vm_flags & VM_PKEY_BIT4) ? H_PTE_PKEY_BIT4 : 0x0UL));
+}
+
+static inline u64 pte_to_hpte_pkey_bits(u64 pteflags)
+{
+ return (((pteflags & H_PTE_PKEY_BIT4) ? HPTE_R_KEY_BIT4 : 0x0UL) |
+ ((pteflags & H_PTE_PKEY_BIT3) ? HPTE_R_KEY_BIT3 : 0x0UL) |
+ ((pteflags & H_PTE_PKEY_BIT2) ? HPTE_R_KEY_BIT2 : 0x0UL) |
+ ((pteflags & H_PTE_PKEY_BIT1) ? HPTE_R_KEY_BIT1 : 0x0UL) |
+ ((pteflags & H_PTE_PKEY_BIT0) ? HPTE_R_KEY_BIT0 : 0x0UL));
+}
+
+static inline u16 hash__pte_to_pkey_bits(u64 pteflags)
+{
+ return (((pteflags & H_PTE_PKEY_BIT4) ? 0x10 : 0x0UL) |
+ ((pteflags & H_PTE_PKEY_BIT3) ? 0x8 : 0x0UL) |
+ ((pteflags & H_PTE_PKEY_BIT2) ? 0x4 : 0x0UL) |
+ ((pteflags & H_PTE_PKEY_BIT1) ? 0x2 : 0x0UL) |
+ ((pteflags & H_PTE_PKEY_BIT0) ? 0x1 : 0x0UL));
+}
+
+#endif
diff --git a/arch/powerpc/include/asm/book3s/64/kexec.h b/arch/powerpc/include/asm/book3s/64/kexec.h
new file mode 100644
index 0000000..6b5c3a2
--- /dev/null
+++ b/arch/powerpc/include/asm/book3s/64/kexec.h
@@ -0,0 +1,23 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+
+#ifndef _ASM_POWERPC_BOOK3S_64_KEXEC_H_
+#define _ASM_POWERPC_BOOK3S_64_KEXEC_H_
+
+
+#define reset_sprs reset_sprs
+static inline void reset_sprs(void)
+{
+ if (cpu_has_feature(CPU_FTR_ARCH_206)) {
+ mtspr(SPRN_AMR, 0);
+ mtspr(SPRN_UAMOR, 0);
+ }
+
+ if (cpu_has_feature(CPU_FTR_ARCH_207S)) {
+ mtspr(SPRN_IAMR, 0);
+ }
+
+ /* Do we need isync()? We are going via a kexec reset */
+ isync();
+}
+
+#endif
diff --git a/arch/powerpc/include/asm/book3s/64/mmu-hash.h b/arch/powerpc/include/asm/book3s/64/mmu-hash.h
index 3fa1b96..93d18da 100644
--- a/arch/powerpc/include/asm/book3s/64/mmu-hash.h
+++ b/arch/powerpc/include/asm/book3s/64/mmu-hash.h
@@ -86,8 +86,8 @@
#define HPTE_R_PP0 ASM_CONST(0x8000000000000000)
#define HPTE_R_TS ASM_CONST(0x4000000000000000)
#define HPTE_R_KEY_HI ASM_CONST(0x3000000000000000)
-#define HPTE_R_KEY_BIT0 ASM_CONST(0x2000000000000000)
-#define HPTE_R_KEY_BIT1 ASM_CONST(0x1000000000000000)
+#define HPTE_R_KEY_BIT4 ASM_CONST(0x2000000000000000)
+#define HPTE_R_KEY_BIT3 ASM_CONST(0x1000000000000000)
#define HPTE_R_RPN_SHIFT 12
#define HPTE_R_RPN ASM_CONST(0x0ffffffffffff000)
#define HPTE_R_RPN_3_0 ASM_CONST(0x01fffffffffff000)
@@ -103,8 +103,8 @@
#define HPTE_R_R ASM_CONST(0x0000000000000100)
#define HPTE_R_KEY_LO ASM_CONST(0x0000000000000e00)
#define HPTE_R_KEY_BIT2 ASM_CONST(0x0000000000000800)
-#define HPTE_R_KEY_BIT3 ASM_CONST(0x0000000000000400)
-#define HPTE_R_KEY_BIT4 ASM_CONST(0x0000000000000200)
+#define HPTE_R_KEY_BIT1 ASM_CONST(0x0000000000000400)
+#define HPTE_R_KEY_BIT0 ASM_CONST(0x0000000000000200)
#define HPTE_R_KEY (HPTE_R_KEY_LO | HPTE_R_KEY_HI)
#define HPTE_V_1TB_SEG ASM_CONST(0x4000000000000000)
@@ -793,7 +793,7 @@
}
/*
- * For kernel space, we use context ids as below
+ * For kernel space, we use context ids as
* below. Range is 512TB per context.
*
* 0x00001 - [ 0xc000000000000000 - 0xc001ffffffffffff]
diff --git a/arch/powerpc/include/asm/book3s/64/mmu.h b/arch/powerpc/include/asm/book3s/64/mmu.h
index 5393a53..55442d4 100644
--- a/arch/powerpc/include/asm/book3s/64/mmu.h
+++ b/arch/powerpc/include/asm/book3s/64/mmu.h
@@ -82,6 +82,11 @@
/* Base PID to allocate from */
extern unsigned int mmu_base_pid;
+/*
+ * memory block size used with radix translation.
+ */
+extern unsigned int __ro_after_init radix_mem_block_size;
+
#define PRTB_SIZE_SHIFT (mmu_pid_bits + 4)
#define PRTB_ENTRIES (1ul << mmu_pid_bits)
@@ -209,6 +214,12 @@
void mmu_early_init_devtree(void);
void hash__early_init_devtree(void);
void radix__early_init_devtree(void);
+#ifdef CONFIG_PPC_MEM_KEYS
+void pkey_early_init_devtree(void);
+#else
+static inline void pkey_early_init_devtree(void) {}
+#endif
+
extern void hash__early_init_mmu(void);
extern void radix__early_init_mmu(void);
static inline void __init early_init_mmu(void)
diff --git a/arch/powerpc/include/asm/book3s/64/pgalloc.h b/arch/powerpc/include/asm/book3s/64/pgalloc.h
index 69c5b05..e1af0b3 100644
--- a/arch/powerpc/include/asm/book3s/64/pgalloc.h
+++ b/arch/powerpc/include/asm/book3s/64/pgalloc.h
@@ -107,9 +107,23 @@
return pud;
}
+static inline void __pud_free(pud_t *pud)
+{
+ struct page *page = virt_to_page(pud);
+
+ /*
+ * Early pud pages allocated via memblock allocator
+ * can't be directly freed to slab
+ */
+ if (PageReserved(page))
+ free_reserved_page(page);
+ else
+ kmem_cache_free(PGT_CACHE(PUD_CACHE_INDEX), pud);
+}
+
static inline void pud_free(struct mm_struct *mm, pud_t *pud)
{
- kmem_cache_free(PGT_CACHE(PUD_CACHE_INDEX), pud);
+ return __pud_free(pud);
}
static inline void pud_populate(struct mm_struct *mm, pud_t *pud, pmd_t *pmd)
diff --git a/arch/powerpc/include/asm/book3s/64/pgtable.h b/arch/powerpc/include/asm/book3s/64/pgtable.h
index 25c3cb8..6de56c3b 100644
--- a/arch/powerpc/include/asm/book3s/64/pgtable.h
+++ b/arch/powerpc/include/asm/book3s/64/pgtable.h
@@ -20,9 +20,13 @@
#define _PAGE_RW (_PAGE_READ | _PAGE_WRITE)
#define _PAGE_RWX (_PAGE_READ | _PAGE_WRITE | _PAGE_EXEC)
#define _PAGE_PRIVILEGED 0x00008 /* kernel access only */
-#define _PAGE_SAO 0x00010 /* Strong access order */
+
+#define _PAGE_CACHE_CTL 0x00030 /* Bits for the folowing cache modes */
+ /* No bits set is normal cacheable memory */
+ /* 0x00010 unused, is SAO bit on radix POWER9 */
#define _PAGE_NON_IDEMPOTENT 0x00020 /* non idempotent memory */
#define _PAGE_TOLERANT 0x00030 /* tolerant memory, cache inhibited */
+
#define _PAGE_DIRTY 0x00080 /* C: page changed */
#define _PAGE_ACCESSED 0x00100 /* R: page referenced */
/*
@@ -32,11 +36,13 @@
#define _RPAGE_SW1 0x00800
#define _RPAGE_SW2 0x00400
#define _RPAGE_SW3 0x00200
-#define _RPAGE_RSV1 0x1000000000000000UL
-#define _RPAGE_RSV2 0x0800000000000000UL
-#define _RPAGE_RSV3 0x0400000000000000UL
-#define _RPAGE_RSV4 0x0200000000000000UL
-#define _RPAGE_RSV5 0x00040UL
+#define _RPAGE_RSV1 0x00040UL
+
+#define _RPAGE_PKEY_BIT4 0x1000000000000000UL
+#define _RPAGE_PKEY_BIT3 0x0800000000000000UL
+#define _RPAGE_PKEY_BIT2 0x0400000000000000UL
+#define _RPAGE_PKEY_BIT1 0x0200000000000000UL
+#define _RPAGE_PKEY_BIT0 0x0100000000000000UL
#define _PAGE_PTE 0x4000000000000000UL /* distinguishes PTEs from pointers */
#define _PAGE_PRESENT 0x8000000000000000UL /* pte contains a translation */
@@ -58,13 +64,12 @@
*/
#define _RPAGE_RPN0 0x01000
#define _RPAGE_RPN1 0x02000
-#define _RPAGE_RPN44 0x0100000000000000UL
#define _RPAGE_RPN43 0x0080000000000000UL
#define _RPAGE_RPN42 0x0040000000000000UL
#define _RPAGE_RPN41 0x0020000000000000UL
/* Max physical address bit as per radix table */
-#define _RPAGE_PA_MAX 57
+#define _RPAGE_PA_MAX 56
/*
* Max physical address bit we will use for now.
@@ -125,8 +130,6 @@
_PAGE_ACCESSED | _PAGE_SPECIAL | _PAGE_PTE | \
_PAGE_SOFT_DIRTY | _PAGE_DEVMAP)
-#define H_PTE_PKEY (H_PTE_PKEY_BIT0 | H_PTE_PKEY_BIT1 | H_PTE_PKEY_BIT2 | \
- H_PTE_PKEY_BIT3 | H_PTE_PKEY_BIT4)
/*
* We define 2 sets of base prot bits, one for basic pages (ie,
* cacheable kernel and user pages) and one for non cacheable
@@ -825,8 +828,6 @@
return hash__set_pte_at(mm, addr, ptep, pte, percpu);
}
-#define _PAGE_CACHE_CTL (_PAGE_SAO | _PAGE_NON_IDEMPOTENT | _PAGE_TOLERANT)
-
#define pgprot_noncached pgprot_noncached
static inline pgprot_t pgprot_noncached(pgprot_t prot)
{
diff --git a/arch/powerpc/include/asm/book3s/64/pkeys.h b/arch/powerpc/include/asm/book3s/64/pkeys.h
new file mode 100644
index 0000000..b7d9f42
--- /dev/null
+++ b/arch/powerpc/include/asm/book3s/64/pkeys.h
@@ -0,0 +1,27 @@
+/* SPDX-License-Identifier: GPL-2.0+ */
+
+#ifndef _ASM_POWERPC_BOOK3S_64_PKEYS_H
+#define _ASM_POWERPC_BOOK3S_64_PKEYS_H
+
+#include <asm/book3s/64/hash-pkey.h>
+
+extern u64 __ro_after_init default_uamor;
+
+static inline u64 vmflag_to_pte_pkey_bits(u64 vm_flags)
+{
+ if (!mmu_has_feature(MMU_FTR_PKEY))
+ return 0x0UL;
+
+ if (radix_enabled())
+ BUG();
+ return hash__vmflag_to_pte_pkey_bits(vm_flags);
+}
+
+static inline u16 pte_to_pkey_bits(u64 pteflags)
+{
+ if (radix_enabled())
+ BUG();
+ return hash__pte_to_pkey_bits(pteflags);
+}
+
+#endif /*_ASM_POWERPC_KEYS_H */
diff --git a/arch/powerpc/include/asm/book3s/64/radix-4k.h b/arch/powerpc/include/asm/book3s/64/radix-4k.h
index d5f5ab7..035ceec 100644
--- a/arch/powerpc/include/asm/book3s/64/radix-4k.h
+++ b/arch/powerpc/include/asm/book3s/64/radix-4k.h
@@ -11,7 +11,7 @@
#define RADIX_PGD_INDEX_SIZE 13 // size: 8B << 13 = 64KB, maps 2^13 x 512GB = 4PB
/*
- * One fragment per per page
+ * One fragment per page
*/
#define RADIX_PTE_FRAG_SIZE_SHIFT (RADIX_PTE_INDEX_SIZE + 3)
#define RADIX_PTE_FRAG_NR (PAGE_SIZE >> RADIX_PTE_FRAG_SIZE_SHIFT)
diff --git a/arch/powerpc/include/asm/book3s/64/tlbflush-radix.h b/arch/powerpc/include/asm/book3s/64/tlbflush-radix.h
index ca8db19..94439e0 100644
--- a/arch/powerpc/include/asm/book3s/64/tlbflush-radix.h
+++ b/arch/powerpc/include/asm/book3s/64/tlbflush-radix.h
@@ -2,10 +2,25 @@
#ifndef _ASM_POWERPC_TLBFLUSH_RADIX_H
#define _ASM_POWERPC_TLBFLUSH_RADIX_H
+#include <asm/hvcall.h>
+
struct vm_area_struct;
struct mm_struct;
struct mmu_gather;
+static inline u64 psize_to_rpti_pgsize(unsigned long psize)
+{
+ if (psize == MMU_PAGE_4K)
+ return H_RPTI_PAGE_4K;
+ if (psize == MMU_PAGE_64K)
+ return H_RPTI_PAGE_64K;
+ if (psize == MMU_PAGE_2M)
+ return H_RPTI_PAGE_2M;
+ if (psize == MMU_PAGE_1G)
+ return H_RPTI_PAGE_1G;
+ return H_RPTI_PAGE_ALL;
+}
+
static inline int mmu_get_ap(int psize)
{
return mmu_psize_defs[psize].ap;
diff --git a/arch/powerpc/include/asm/cacheflush.h b/arch/powerpc/include/asm/cacheflush.h
index de600b9..54764c6 100644
--- a/arch/powerpc/include/asm/cacheflush.h
+++ b/arch/powerpc/include/asm/cacheflush.h
@@ -6,6 +6,7 @@
#include <linux/mm.h>
#include <asm/cputable.h>
+#include <asm/cpu_has_feature.h>
#ifdef CONFIG_PPC_BOOK3S_64
/*
diff --git a/arch/powerpc/include/asm/cputable.h b/arch/powerpc/include/asm/cputable.h
index bac2252..fdddb82 100644
--- a/arch/powerpc/include/asm/cputable.h
+++ b/arch/powerpc/include/asm/cputable.h
@@ -191,14 +191,14 @@
#define CPU_FTR_SPURR LONG_ASM_CONST(0x0000000001000000)
#define CPU_FTR_DSCR LONG_ASM_CONST(0x0000000002000000)
#define CPU_FTR_VSX LONG_ASM_CONST(0x0000000004000000)
-#define CPU_FTR_SAO LONG_ASM_CONST(0x0000000008000000)
+// Free LONG_ASM_CONST(0x0000000008000000)
#define CPU_FTR_CP_USE_DCBTZ LONG_ASM_CONST(0x0000000010000000)
#define CPU_FTR_UNALIGNED_LD_STD LONG_ASM_CONST(0x0000000020000000)
#define CPU_FTR_ASYM_SMT LONG_ASM_CONST(0x0000000040000000)
#define CPU_FTR_STCX_CHECKS_ADDRESS LONG_ASM_CONST(0x0000000080000000)
#define CPU_FTR_POPCNTB LONG_ASM_CONST(0x0000000100000000)
#define CPU_FTR_POPCNTD LONG_ASM_CONST(0x0000000200000000)
-#define CPU_FTR_PKEY LONG_ASM_CONST(0x0000000400000000)
+/* LONG_ASM_CONST(0x0000000400000000) Free */
#define CPU_FTR_VMX_COPY LONG_ASM_CONST(0x0000000800000000)
#define CPU_FTR_TM LONG_ASM_CONST(0x0000001000000000)
#define CPU_FTR_CFAR LONG_ASM_CONST(0x0000002000000000)
@@ -214,6 +214,7 @@
#define CPU_FTR_P9_TLBIE_ERAT_BUG LONG_ASM_CONST(0x0001000000000000)
#define CPU_FTR_P9_RADIX_PREFETCH_BUG LONG_ASM_CONST(0x0002000000000000)
#define CPU_FTR_ARCH_31 LONG_ASM_CONST(0x0004000000000000)
+#define CPU_FTR_DAWR1 LONG_ASM_CONST(0x0008000000000000)
#ifndef __ASSEMBLY__
@@ -435,32 +436,32 @@
CPU_FTR_MMCRA | CPU_FTR_SMT | \
CPU_FTR_COHERENT_ICACHE | \
CPU_FTR_PURR | CPU_FTR_SPURR | CPU_FTR_REAL_LE | \
- CPU_FTR_DSCR | CPU_FTR_SAO | CPU_FTR_ASYM_SMT | \
+ CPU_FTR_DSCR | CPU_FTR_ASYM_SMT | \
CPU_FTR_STCX_CHECKS_ADDRESS | CPU_FTR_POPCNTB | CPU_FTR_POPCNTD | \
CPU_FTR_CFAR | CPU_FTR_HVMODE | \
- CPU_FTR_VMX_COPY | CPU_FTR_HAS_PPR | CPU_FTR_DABRX | CPU_FTR_PKEY)
+ CPU_FTR_VMX_COPY | CPU_FTR_HAS_PPR | CPU_FTR_DABRX )
#define CPU_FTRS_POWER8 (CPU_FTR_LWSYNC | \
CPU_FTR_PPCAS_ARCH_V2 | CPU_FTR_CTRL | CPU_FTR_ARCH_206 |\
CPU_FTR_MMCRA | CPU_FTR_SMT | \
CPU_FTR_COHERENT_ICACHE | \
CPU_FTR_PURR | CPU_FTR_SPURR | CPU_FTR_REAL_LE | \
- CPU_FTR_DSCR | CPU_FTR_SAO | \
+ CPU_FTR_DSCR | \
CPU_FTR_STCX_CHECKS_ADDRESS | CPU_FTR_POPCNTB | CPU_FTR_POPCNTD | \
CPU_FTR_CFAR | CPU_FTR_HVMODE | CPU_FTR_VMX_COPY | \
CPU_FTR_DBELL | CPU_FTR_HAS_PPR | CPU_FTR_DAWR | \
- CPU_FTR_ARCH_207S | CPU_FTR_TM_COMP | CPU_FTR_PKEY)
+ CPU_FTR_ARCH_207S | CPU_FTR_TM_COMP )
#define CPU_FTRS_POWER8E (CPU_FTRS_POWER8 | CPU_FTR_PMAO_BUG)
#define CPU_FTRS_POWER9 (CPU_FTR_LWSYNC | \
CPU_FTR_PPCAS_ARCH_V2 | CPU_FTR_CTRL | CPU_FTR_ARCH_206 |\
CPU_FTR_MMCRA | CPU_FTR_SMT | \
CPU_FTR_COHERENT_ICACHE | \
CPU_FTR_PURR | CPU_FTR_SPURR | CPU_FTR_REAL_LE | \
- CPU_FTR_DSCR | CPU_FTR_SAO | \
+ CPU_FTR_DSCR | \
CPU_FTR_STCX_CHECKS_ADDRESS | CPU_FTR_POPCNTB | CPU_FTR_POPCNTD | \
CPU_FTR_CFAR | CPU_FTR_HVMODE | CPU_FTR_VMX_COPY | \
CPU_FTR_DBELL | CPU_FTR_HAS_PPR | CPU_FTR_ARCH_207S | \
- CPU_FTR_TM_COMP | CPU_FTR_ARCH_300 | CPU_FTR_PKEY | \
- CPU_FTR_P9_TLBIE_STQ_BUG | CPU_FTR_P9_TLBIE_ERAT_BUG | CPU_FTR_P9_TIDR)
+ CPU_FTR_TM_COMP | CPU_FTR_ARCH_300 | CPU_FTR_P9_TLBIE_STQ_BUG | \
+ CPU_FTR_P9_TLBIE_ERAT_BUG | CPU_FTR_P9_TIDR)
#define CPU_FTRS_POWER9_DD2_0 (CPU_FTRS_POWER9 | CPU_FTR_P9_RADIX_PREFETCH_BUG)
#define CPU_FTRS_POWER9_DD2_1 (CPU_FTRS_POWER9 | \
CPU_FTR_P9_RADIX_PREFETCH_BUG | \
@@ -473,12 +474,12 @@
CPU_FTR_MMCRA | CPU_FTR_SMT | \
CPU_FTR_COHERENT_ICACHE | \
CPU_FTR_PURR | CPU_FTR_SPURR | CPU_FTR_REAL_LE | \
- CPU_FTR_DSCR | CPU_FTR_SAO | \
+ CPU_FTR_DSCR | \
CPU_FTR_STCX_CHECKS_ADDRESS | CPU_FTR_POPCNTB | CPU_FTR_POPCNTD | \
CPU_FTR_CFAR | CPU_FTR_HVMODE | CPU_FTR_VMX_COPY | \
CPU_FTR_DBELL | CPU_FTR_HAS_PPR | CPU_FTR_ARCH_207S | \
- CPU_FTR_TM_COMP | CPU_FTR_ARCH_300 | CPU_FTR_PKEY | \
- CPU_FTR_ARCH_31)
+ CPU_FTR_TM_COMP | CPU_FTR_ARCH_300 | CPU_FTR_ARCH_31 | \
+ CPU_FTR_DAWR | CPU_FTR_DAWR1)
#define CPU_FTRS_CELL (CPU_FTR_LWSYNC | \
CPU_FTR_PPCAS_ARCH_V2 | CPU_FTR_CTRL | \
CPU_FTR_ALTIVEC_COMP | CPU_FTR_MMCRA | CPU_FTR_SMT | \
@@ -628,9 +629,10 @@
/*
* Maximum number of hw breakpoint supported on powerpc. Number of
- * breakpoints supported by actual hw might be less than this.
+ * breakpoints supported by actual hw might be less than this, which
+ * is decided at run time in nr_wp_slots().
*/
-#define HBP_NUM_MAX 1
+#define HBP_NUM_MAX 2
#endif /* !__ASSEMBLY__ */
diff --git a/arch/powerpc/include/asm/cputime.h b/arch/powerpc/include/asm/cputime.h
index 0fccd5e..ed75d1c 100644
--- a/arch/powerpc/include/asm/cputime.h
+++ b/arch/powerpc/include/asm/cputime.h
@@ -36,6 +36,8 @@
return mulhdu((__force u64) ct, __cputime_usec_factor);
}
+#define cputime_to_nsecs(cputime) tb_to_ns((__force u64)cputime)
+
/*
* PPC64 uses PACA which is task independent for storing accounting data while
* PPC32 uses struct thread_info, therefore at task switch the accounting data
@@ -65,7 +67,7 @@
/*
* account_cpu_user_entry/exit runs "unreconciled", so can't trace,
- * can't use use get_paca()
+ * can't use get_paca()
*/
static notrace inline void account_cpu_user_entry(void)
{
diff --git a/arch/powerpc/include/asm/crashdump-ppc64.h b/arch/powerpc/include/asm/crashdump-ppc64.h
new file mode 100644
index 0000000..68d9717
--- /dev/null
+++ b/arch/powerpc/include/asm/crashdump-ppc64.h
@@ -0,0 +1,19 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+#ifndef _ASM_POWERPC_CRASHDUMP_PPC64_H
+#define _ASM_POWERPC_CRASHDUMP_PPC64_H
+
+/*
+ * Backup region - first 64KB of System RAM
+ *
+ * If ever the below macros are to be changed, please be judicious.
+ * The implicit assumptions are:
+ * - start, end & size are less than UINT32_MAX.
+ * - start & size are at least 8 byte aligned.
+ *
+ * For implementation details: arch/powerpc/purgatory/trampoline_64.S
+ */
+#define BACKUP_SRC_START 0
+#define BACKUP_SRC_END 0xffff
+#define BACKUP_SRC_SIZE (BACKUP_SRC_END - BACKUP_SRC_START + 1)
+
+#endif /* __ASM_POWERPC_CRASHDUMP_PPC64_H */
diff --git a/arch/powerpc/include/asm/dbell.h b/arch/powerpc/include/asm/dbell.h
index 4ce6808..3e9da22 100644
--- a/arch/powerpc/include/asm/dbell.h
+++ b/arch/powerpc/include/asm/dbell.h
@@ -11,8 +11,10 @@
#include <linux/smp.h>
#include <linux/threads.h>
+#include <asm/cputhreads.h>
#include <asm/ppc-opcode.h>
#include <asm/feature-fixups.h>
+#include <asm/kvm_ppc.h>
#define PPC_DBELL_MSG_BRDCAST (0x04000000)
#define PPC_DBELL_TYPE(x) (((x) & 0xf) << (63-36))
@@ -87,9 +89,6 @@
#endif /* CONFIG_PPC_BOOK3S */
-extern void doorbell_global_ipi(int cpu);
-extern void doorbell_core_ipi(int cpu);
-extern int doorbell_try_core_ipi(int cpu);
extern void doorbell_exception(struct pt_regs *regs);
static inline void ppc_msgsnd(enum ppc_dbell type, u32 flags, u32 tag)
@@ -100,4 +99,63 @@
_ppc_msgsnd(msg);
}
+#ifdef CONFIG_SMP
+
+/*
+ * Doorbells must only be used if CPU_FTR_DBELL is available.
+ * msgsnd is used in HV, and msgsndp is used in !HV.
+ *
+ * These should be used by platform code that is aware of restrictions.
+ * Other arch code should use ->cause_ipi.
+ *
+ * doorbell_global_ipi() sends a dbell to any target CPU.
+ * Must be used only by architectures that address msgsnd target
+ * by PIR/get_hard_smp_processor_id.
+ */
+static inline void doorbell_global_ipi(int cpu)
+{
+ u32 tag = get_hard_smp_processor_id(cpu);
+
+ kvmppc_set_host_ipi(cpu);
+ /* Order previous accesses vs. msgsnd, which is treated as a store */
+ ppc_msgsnd_sync();
+ ppc_msgsnd(PPC_DBELL_MSGTYPE, 0, tag);
+}
+
+/*
+ * doorbell_core_ipi() sends a dbell to a target CPU in the same core.
+ * Must be used only by architectures that address msgsnd target
+ * by TIR/cpu_thread_in_core.
+ */
+static inline void doorbell_core_ipi(int cpu)
+{
+ u32 tag = cpu_thread_in_core(cpu);
+
+ kvmppc_set_host_ipi(cpu);
+ /* Order previous accesses vs. msgsnd, which is treated as a store */
+ ppc_msgsnd_sync();
+ ppc_msgsnd(PPC_DBELL_MSGTYPE, 0, tag);
+}
+
+/*
+ * Attempt to cause a core doorbell if destination is on the same core.
+ * Returns 1 on success, 0 on failure.
+ */
+static inline int doorbell_try_core_ipi(int cpu)
+{
+ int this_cpu = get_cpu();
+ int ret = 0;
+
+ if (cpumask_test_cpu(cpu, cpu_sibling_mask(this_cpu))) {
+ doorbell_core_ipi(cpu);
+ ret = 1;
+ }
+
+ put_cpu();
+
+ return ret;
+}
+
+#endif /* CONFIG_SMP */
+
#endif /* _ASM_POWERPC_DBELL_H */
diff --git a/arch/powerpc/include/asm/device.h b/arch/powerpc/include/asm/device.h
index 45240221..d8a0729 100644
--- a/arch/powerpc/include/asm/device.h
+++ b/arch/powerpc/include/asm/device.h
@@ -44,6 +44,9 @@
#ifdef CONFIG_CXL_BASE
struct cxl_context *cxl_ctx;
#endif
+#ifdef CONFIG_PCI_IOV
+ void *iov_data;
+#endif
};
struct pdev_archdata {
diff --git a/arch/powerpc/include/asm/drmem.h b/arch/powerpc/include/asm/drmem.h
index 414d209..17ccc64 100644
--- a/arch/powerpc/include/asm/drmem.h
+++ b/arch/powerpc/include/asm/drmem.h
@@ -90,13 +90,14 @@
}
u64 drmem_lmb_memory_max(void);
-void __init walk_drmem_lmbs(struct device_node *dn,
- void (*func)(struct drmem_lmb *, const __be32 **));
+int walk_drmem_lmbs(struct device_node *dn, void *data,
+ int (*func)(struct drmem_lmb *, const __be32 **, void *));
int drmem_update_dt(void);
#ifdef CONFIG_PPC_PSERIES
-void __init walk_drmem_lmbs_early(unsigned long node,
- void (*func)(struct drmem_lmb *, const __be32 **));
+int __init
+walk_drmem_lmbs_early(unsigned long node, void *data,
+ int (*func)(struct drmem_lmb *, const __be32 **, void *));
#endif
static inline void invalidate_lmb_associativity_index(struct drmem_lmb *lmb)
diff --git a/arch/powerpc/include/asm/eeh.h b/arch/powerpc/include/asm/eeh.h
index 964a542..d5f369b 100644
--- a/arch/powerpc/include/asm/eeh.h
+++ b/arch/powerpc/include/asm/eeh.h
@@ -133,7 +133,6 @@
struct eeh_dev {
int mode; /* EEH mode */
- int class_code; /* Class code of the device */
int bdfn; /* bdfn of device (for cfg ops) */
struct pci_controller *controller;
int pe_config_addr; /* PE config address */
@@ -148,7 +147,10 @@
struct pci_dn *pdn; /* Associated PCI device node */
struct pci_dev *pdev; /* Associated PCI device */
bool in_error; /* Error flag for edev */
+
+ /* VF specific properties */
struct pci_dev *physfn; /* Associated SRIOV PF */
+ int vf_index; /* Index of this VF */
};
/* "fmt" must be a simple literal string */
@@ -217,18 +219,17 @@
int (*init)(void);
struct eeh_dev *(*probe)(struct pci_dev *pdev);
int (*set_option)(struct eeh_pe *pe, int option);
- int (*get_pe_addr)(struct eeh_pe *pe);
int (*get_state)(struct eeh_pe *pe, int *delay);
int (*reset)(struct eeh_pe *pe, int option);
int (*get_log)(struct eeh_pe *pe, int severity, char *drv_log, unsigned long len);
int (*configure_bridge)(struct eeh_pe *pe);
int (*err_inject)(struct eeh_pe *pe, int type, int func,
unsigned long addr, unsigned long mask);
- int (*read_config)(struct pci_dn *pdn, int where, int size, u32 *val);
- int (*write_config)(struct pci_dn *pdn, int where, int size, u32 val);
+ int (*read_config)(struct eeh_dev *edev, int where, int size, u32 *val);
+ int (*write_config)(struct eeh_dev *edev, int where, int size, u32 val);
int (*next_error)(struct eeh_pe **pe);
- int (*restore_config)(struct pci_dn *pdn);
- int (*notify_resume)(struct pci_dn *pdn);
+ int (*restore_config)(struct eeh_dev *edev);
+ int (*notify_resume)(struct eeh_dev *edev);
};
extern int eeh_subsystem_flags;
@@ -282,8 +283,8 @@
struct eeh_pe *eeh_pe_next(struct eeh_pe *pe, struct eeh_pe *root);
struct eeh_pe *eeh_pe_get(struct pci_controller *phb,
int pe_no, int config_addr);
-int eeh_add_to_parent_pe(struct eeh_dev *edev);
-int eeh_rmv_from_parent_pe(struct eeh_dev *edev);
+int eeh_pe_tree_insert(struct eeh_dev *edev, struct eeh_pe *new_pe_parent);
+int eeh_pe_tree_remove(struct eeh_dev *edev);
void eeh_pe_update_time_stamp(struct eeh_pe *pe);
void *eeh_pe_traverse(struct eeh_pe *root,
eeh_pe_traverse_func fn, void *flag);
@@ -293,8 +294,6 @@
const char *eeh_pe_loc_get(struct eeh_pe *pe);
struct pci_bus *eeh_pe_bus_get(struct eeh_pe *pe);
-struct eeh_dev *eeh_dev_init(struct pci_dn *pdn);
-void eeh_dev_phb_init_dynamic(struct pci_controller *phb);
void eeh_show_enabled(void);
int __init eeh_ops_register(struct eeh_ops *ops);
int __exit eeh_ops_unregister(const char *name);
@@ -314,7 +313,6 @@
int eeh_pe_configure(struct eeh_pe *pe);
int eeh_pe_inject_err(struct eeh_pe *pe, int type, int func,
unsigned long addr, unsigned long mask);
-int eeh_restore_vf_config(struct pci_dn *pdn);
/**
* EEH_POSSIBLE_ERROR() -- test for possible MMIO failure.
@@ -340,11 +338,6 @@
static inline void eeh_show_enabled(void) { }
-static inline void *eeh_dev_init(struct pci_dn *pdn, void *data)
-{
- return NULL;
-}
-
static inline void eeh_dev_phb_init_dynamic(struct pci_controller *phb) { }
static inline int eeh_check_failure(const volatile void __iomem *token)
@@ -362,6 +355,7 @@
#define EEH_POSSIBLE_ERROR(val, type) (0)
#define EEH_IO_ERROR_VALUE(size) (-1UL)
+static inline int eeh_phb_pe_create(struct pci_controller *phb) { return 0; }
#endif /* CONFIG_EEH */
#if defined(CONFIG_PPC_PSERIES) && defined(CONFIG_EEH)
diff --git a/arch/powerpc/include/asm/epapr_hcalls.h b/arch/powerpc/include/asm/epapr_hcalls.h
index d3a7e36..c99ba08 100644
--- a/arch/powerpc/include/asm/epapr_hcalls.h
+++ b/arch/powerpc/include/asm/epapr_hcalls.h
@@ -37,7 +37,7 @@
* SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
-/* A "hypercall" is an "sc 1" instruction. This header file file provides C
+/* A "hypercall" is an "sc 1" instruction. This header file provides C
* wrapper functions for the ePAPR hypervisor interface. It is inteded
* for use by Linux device drivers and other operating systems.
*
@@ -246,7 +246,7 @@
* ev_int_eoi - signal the end of interrupt processing
* @interrupt: the interrupt number
*
- * This function signals the end of processing for the the specified
+ * This function signals the end of processing for the specified
* interrupt, which must be the interrupt currently in service. By
* definition, this is also the highest-priority interrupt.
*
diff --git a/arch/powerpc/include/asm/exception-64e.h b/arch/powerpc/include/asm/exception-64e.h
index 54a98ef..40cdcb2 100644
--- a/arch/powerpc/include/asm/exception-64e.h
+++ b/arch/powerpc/include/asm/exception-64e.h
@@ -66,14 +66,7 @@
#define EX_TLB_SRR0 (10 * 8)
#define EX_TLB_SRR1 (11 * 8)
#define EX_TLB_R7 (12 * 8)
-#ifdef CONFIG_BOOK3E_MMU_TLB_STATS
-#define EX_TLB_R8 (13 * 8)
-#define EX_TLB_R9 (14 * 8)
-#define EX_TLB_LR (15 * 8)
-#define EX_TLB_SIZE (16 * 8)
-#else
#define EX_TLB_SIZE (13 * 8)
-#endif
#define START_EXCEPTION(label) \
.globl exc_##label##_book3e; \
@@ -110,8 +103,7 @@
std r11,EX_TLB_R12(r12); \
mtspr SPRN_SPRG_TLB_EXFRAME,r14; \
std r15,EX_TLB_SRR1(r12); \
- std r16,EX_TLB_SRR0(r12); \
- TLB_MISS_PROLOG_STATS
+ std r16,EX_TLB_SRR0(r12);
/* And these are the matching epilogs that restores things
*
@@ -143,7 +135,6 @@
mtspr SPRN_SRR0,r15; \
ld r15,EX_TLB_R15(r12); \
mtspr SPRN_SRR1,r16; \
- TLB_MISS_RESTORE_STATS \
ld r16,EX_TLB_R16(r12); \
ld r12,EX_TLB_R12(r12); \
@@ -158,53 +149,15 @@
addi r11,r13,PACA_EXTLB; \
TLB_MISS_RESTORE(r11)
-#ifdef CONFIG_BOOK3E_MMU_TLB_STATS
-#define TLB_MISS_PROLOG_STATS \
- mflr r10; \
- std r8,EX_TLB_R8(r12); \
- std r9,EX_TLB_R9(r12); \
- std r10,EX_TLB_LR(r12);
-#define TLB_MISS_RESTORE_STATS \
- ld r16,EX_TLB_LR(r12); \
- ld r9,EX_TLB_R9(r12); \
- ld r8,EX_TLB_R8(r12); \
- mtlr r16;
-#define TLB_MISS_STATS_D(name) \
- addi r9,r13,MMSTAT_DSTATS+name; \
- bl tlb_stat_inc;
-#define TLB_MISS_STATS_I(name) \
- addi r9,r13,MMSTAT_ISTATS+name; \
- bl tlb_stat_inc;
-#define TLB_MISS_STATS_X(name) \
- ld r8,PACA_EXTLB+EX_TLB_ESR(r13); \
- cmpdi cr2,r8,-1; \
- beq cr2,61f; \
- addi r9,r13,MMSTAT_DSTATS+name; \
- b 62f; \
-61: addi r9,r13,MMSTAT_ISTATS+name; \
-62: bl tlb_stat_inc;
-#define TLB_MISS_STATS_SAVE_INFO \
- std r14,EX_TLB_ESR(r12); /* save ESR */
-#define TLB_MISS_STATS_SAVE_INFO_BOLTED \
- std r14,PACA_EXTLB+EX_TLB_ESR(r13); /* save ESR */
-#else
-#define TLB_MISS_PROLOG_STATS
-#define TLB_MISS_RESTORE_STATS
-#define TLB_MISS_PROLOG_STATS_BOLTED
-#define TLB_MISS_RESTORE_STATS_BOLTED
-#define TLB_MISS_STATS_D(name)
-#define TLB_MISS_STATS_I(name)
-#define TLB_MISS_STATS_X(name)
-#define TLB_MISS_STATS_Y(name)
-#define TLB_MISS_STATS_SAVE_INFO
-#define TLB_MISS_STATS_SAVE_INFO_BOLTED
-#endif
-
#define SET_IVOR(vector_number, vector_offset) \
LOAD_REG_ADDR(r3,interrupt_base_book3e);\
ori r3,r3,vector_offset@l; \
mtspr SPRN_IVOR##vector_number,r3;
-
+/*
+ * powerpc relies on return from interrupt/syscall being context synchronising
+ * (which rfi is) to support ARCH_HAS_MEMBARRIER_SYNC_CORE without additional
+ * synchronisation instructions.
+ */
#define RFI_TO_KERNEL \
rfi
diff --git a/arch/powerpc/include/asm/exception-64s.h b/arch/powerpc/include/asm/exception-64s.h
index 47bd4ea..ebe95aa 100644
--- a/arch/powerpc/include/asm/exception-64s.h
+++ b/arch/powerpc/include/asm/exception-64s.h
@@ -68,6 +68,14 @@
*
* The nop instructions allow us to insert one or more instructions to flush the
* L1-D cache when returning to userspace or a guest.
+ *
+ * powerpc relies on return from interrupt/syscall being context synchronising
+ * (which hrfid, rfid, and rfscv are) to support ARCH_HAS_MEMBARRIER_SYNC_CORE
+ * without additional synchronisation instructions.
+ *
+ * soft-masked interrupt replay does not include a context-synchronising rfid,
+ * but those always return to kernel, the sync is only required when returning
+ * to user.
*/
#define RFI_FLUSH_SLOT \
RFI_FLUSH_FIXUP_SECTION; \
@@ -123,6 +131,12 @@
hrfid; \
b hrfi_flush_fallback
+#define RFSCV_TO_USER \
+ STF_EXIT_BARRIER_SLOT; \
+ RFI_FLUSH_SLOT; \
+ RFSCV; \
+ b rfscv_flush_fallback
+
#endif /* __ASSEMBLY__ */
#endif /* _ASM_POWERPC_EXCEPTION_H */
diff --git a/arch/powerpc/include/asm/firmware.h b/arch/powerpc/include/asm/firmware.h
index 6003c2e..0b295bd 100644
--- a/arch/powerpc/include/asm/firmware.h
+++ b/arch/powerpc/include/asm/firmware.h
@@ -52,6 +52,7 @@
#define FW_FEATURE_PAPR_SCM ASM_CONST(0x0000002000000000)
#define FW_FEATURE_ULTRAVISOR ASM_CONST(0x0000004000000000)
#define FW_FEATURE_STUFF_TCE ASM_CONST(0x0000008000000000)
+#define FW_FEATURE_RPT_INVALIDATE ASM_CONST(0x0000010000000000)
#ifndef __ASSEMBLY__
@@ -71,7 +72,8 @@
FW_FEATURE_TYPE1_AFFINITY | FW_FEATURE_PRRN |
FW_FEATURE_HPT_RESIZE | FW_FEATURE_DRMEM_V2 |
FW_FEATURE_DRC_INFO | FW_FEATURE_BLOCK_REMOVE |
- FW_FEATURE_PAPR_SCM | FW_FEATURE_ULTRAVISOR,
+ FW_FEATURE_PAPR_SCM | FW_FEATURE_ULTRAVISOR |
+ FW_FEATURE_RPT_INVALIDATE,
FW_FEATURE_PSERIES_ALWAYS = 0,
FW_FEATURE_POWERNV_POSSIBLE = FW_FEATURE_OPAL | FW_FEATURE_ULTRAVISOR,
FW_FEATURE_POWERNV_ALWAYS = 0,
@@ -132,6 +134,12 @@
extern unsigned int __start___fw_ftr_fixup, __stop___fw_ftr_fixup;
+#if defined(CONFIG_PPC_PSERIES) || defined(CONFIG_KVM_GUEST)
+bool is_kvm_guest(void);
+#else
+static inline bool is_kvm_guest(void) { return false; }
+#endif
+
#ifdef CONFIG_PPC_PSERIES
void pseries_probe_fw_features(void);
#else
diff --git a/arch/powerpc/include/asm/fixmap.h b/arch/powerpc/include/asm/fixmap.h
index 29188810..925cf89 100644
--- a/arch/powerpc/include/asm/fixmap.h
+++ b/arch/powerpc/include/asm/fixmap.h
@@ -52,7 +52,7 @@
FIX_HOLE,
/* reserve the top 128K for early debugging purposes */
FIX_EARLY_DEBUG_TOP = FIX_HOLE,
- FIX_EARLY_DEBUG_BASE = FIX_EARLY_DEBUG_TOP+((128*1024)/PAGE_SIZE)-1,
+ FIX_EARLY_DEBUG_BASE = FIX_EARLY_DEBUG_TOP+(ALIGN(SZ_128, PAGE_SIZE)/PAGE_SIZE)-1,
#ifdef CONFIG_HIGHMEM
FIX_KMAP_BEGIN, /* reserved pte's for temporary kernel mappings */
FIX_KMAP_END = FIX_KMAP_BEGIN+(KM_TYPE_NR*NR_CPUS)-1,
diff --git a/arch/powerpc/include/asm/hardirq.h b/arch/powerpc/include/asm/hardirq.h
index f1e9067..f133b59 100644
--- a/arch/powerpc/include/asm/hardirq.h
+++ b/arch/powerpc/include/asm/hardirq.h
@@ -13,7 +13,6 @@
unsigned int pmu_irqs;
unsigned int mce_exceptions;
unsigned int spurious_irqs;
- unsigned int hmi_exceptions;
unsigned int sreset_irqs;
#ifdef CONFIG_PPC_WATCHDOG
unsigned int soft_nmi_irqs;
diff --git a/arch/powerpc/include/asm/head-64.h b/arch/powerpc/include/asm/head-64.h
index 2dabcf6..4cb9efa 100644
--- a/arch/powerpc/include/asm/head-64.h
+++ b/arch/powerpc/include/asm/head-64.h
@@ -128,7 +128,7 @@
.if ((start) % (size) != 0); \
.error "Fixed section exception vector misalignment"; \
.endif; \
- .if ((size) != 0x20) && ((size) != 0x80) && ((size) != 0x100); \
+ .if ((size) != 0x20) && ((size) != 0x80) && ((size) != 0x100) && ((size) != 0x1000); \
.error "Fixed section exception vector bad size"; \
.endif; \
.if (start) < sname##_start; \
diff --git a/arch/powerpc/include/asm/hugetlb.h b/arch/powerpc/include/asm/hugetlb.h
index 551a9d4..013165e 100644
--- a/arch/powerpc/include/asm/hugetlb.h
+++ b/arch/powerpc/include/asm/hugetlb.h
@@ -57,6 +57,7 @@
unsigned long addr, pte_t *ptep,
pte_t pte, int dirty);
+void gigantic_hugetlb_cma_reserve(void) __init;
#include <asm-generic/hugetlb.h>
#else /* ! CONFIG_HUGETLB_PAGE */
@@ -71,6 +72,12 @@
{
return NULL;
}
+
+
+static inline void __init gigantic_hugetlb_cma_reserve(void)
+{
+}
+
#endif /* CONFIG_HUGETLB_PAGE */
#endif /* _ASM_POWERPC_HUGETLB_H */
diff --git a/arch/powerpc/include/asm/hvcall.h b/arch/powerpc/include/asm/hvcall.h
index e90c073..fbb3770 100644
--- a/arch/powerpc/include/asm/hvcall.h
+++ b/arch/powerpc/include/asm/hvcall.h
@@ -237,7 +237,7 @@
#define H_CREATE_RPT 0x1A4
#define H_REMOVE_RPT 0x1A8
#define H_REGISTER_RPAGES 0x1AC
-#define H_DISABLE_AND_GETC 0x1B0
+#define H_DISABLE_AND_GET 0x1B0
#define H_ERROR_DATA 0x1B4
#define H_GET_HCA_INFO 0x1B8
#define H_GET_PERF_COUNT 0x1BC
@@ -305,7 +305,8 @@
#define H_SCM_UNBIND_ALL 0x3FC
#define H_SCM_HEALTH 0x400
#define H_SCM_PERFORMANCE_STATS 0x418
-#define MAX_HCALL_OPCODE H_SCM_PERFORMANCE_STATS
+#define H_RPT_INVALIDATE 0x448
+#define MAX_HCALL_OPCODE H_RPT_INVALIDATE
/* Scope args for H_SCM_UNBIND_ALL */
#define H_UNBIND_SCOPE_ALL (0x1)
@@ -354,9 +355,10 @@
/* Values for 2nd argument to H_SET_MODE */
#define H_SET_MODE_RESOURCE_SET_CIABR 1
-#define H_SET_MODE_RESOURCE_SET_DAWR 2
+#define H_SET_MODE_RESOURCE_SET_DAWR0 2
#define H_SET_MODE_RESOURCE_ADDR_TRANS_MODE 3
#define H_SET_MODE_RESOURCE_LE 4
+#define H_SET_MODE_RESOURCE_SET_DAWR1 5
/* Values for argument to H_SIGNAL_SYS_RESET */
#define H_SIGNAL_SYS_RESET_ALL -1
@@ -389,6 +391,37 @@
#define PROC_TABLE_RADIX 0x04
#define PROC_TABLE_GTSE 0x01
+/*
+ * Defines for
+ * H_RPT_INVALIDATE - Invalidate RPT translation lookaside information.
+ */
+
+/* Type of translation to invalidate (type) */
+#define H_RPTI_TYPE_NESTED 0x0001 /* Invalidate nested guest partition-scope */
+#define H_RPTI_TYPE_TLB 0x0002 /* Invalidate TLB */
+#define H_RPTI_TYPE_PWC 0x0004 /* Invalidate Page Walk Cache */
+/* Invalidate Process Table Entries if H_RPTI_TYPE_NESTED is clear */
+#define H_RPTI_TYPE_PRT 0x0008
+/* Invalidate Partition Table Entries if H_RPTI_TYPE_NESTED is set */
+#define H_RPTI_TYPE_PAT 0x0008
+#define H_RPTI_TYPE_ALL (H_RPTI_TYPE_TLB | H_RPTI_TYPE_PWC | \
+ H_RPTI_TYPE_PRT)
+#define H_RPTI_TYPE_NESTED_ALL (H_RPTI_TYPE_TLB | H_RPTI_TYPE_PWC | \
+ H_RPTI_TYPE_PAT)
+
+/* Invalidation targets (target) */
+#define H_RPTI_TARGET_CMMU 0x01 /* All virtual processors in the partition */
+#define H_RPTI_TARGET_CMMU_LOCAL 0x02 /* Current virtual processor */
+/* All nest/accelerator agents in use by the partition */
+#define H_RPTI_TARGET_NMMU 0x04
+
+/* Page size mask (page sizes) */
+#define H_RPTI_PAGE_4K 0x01
+#define H_RPTI_PAGE_64K 0x02
+#define H_RPTI_PAGE_2M 0x04
+#define H_RPTI_PAGE_1G 0x08
+#define H_RPTI_PAGE_ALL (-1UL)
+
#ifndef __ASSEMBLY__
#include <linux/types.h>
diff --git a/arch/powerpc/include/asm/hw_breakpoint.h b/arch/powerpc/include/asm/hw_breakpoint.h
index cb42479..db206a7 100644
--- a/arch/powerpc/include/asm/hw_breakpoint.h
+++ b/arch/powerpc/include/asm/hw_breakpoint.h
@@ -9,6 +9,8 @@
#ifndef _PPC_BOOK3S_64_HW_BREAKPOINT_H
#define _PPC_BOOK3S_64_HW_BREAKPOINT_H
+#include <asm/cpu_has_feature.h>
+
#ifdef __KERNEL__
struct arch_hw_breakpoint {
unsigned long address;
@@ -17,7 +19,7 @@
u16 hw_len; /* length programmed in hw */
};
-/* Note: Don't change the the first 6 bits below as they are in the same order
+/* Note: Don't change the first 6 bits below as they are in the same order
* as the dabr and dabrx.
*/
#define HW_BRK_TYPE_READ 0x01
@@ -46,7 +48,7 @@
static inline int nr_wp_slots(void)
{
- return HBP_NUM_MAX;
+ return cpu_has_feature(CPU_FTR_DAWR1) ? 2 : 1;
}
#ifdef CONFIG_HAVE_HW_BREAKPOINT
diff --git a/arch/powerpc/include/asm/hydra.h b/arch/powerpc/include/asm/hydra.h
index b3b0f2d..ae02eb5 100644
--- a/arch/powerpc/include/asm/hydra.h
+++ b/arch/powerpc/include/asm/hydra.h
@@ -10,7 +10,7 @@
*
* © Copyright 1995 Apple Computer, Inc. All rights reserved.
*
- * It's available online from http://www.cpu.lu/~mlan/ftp/MacTech.pdf
+ * It's available online from https://www.cpu.lu/~mlan/ftp/MacTech.pdf
* You can obtain paper copies of this book from computer bookstores or by
* writing Morgan Kaufmann Publishers, Inc., 340 Pine Street, Sixth Floor, San
* Francisco, CA 94104. Reference ISBN 1-55860-393-X.
diff --git a/arch/powerpc/include/asm/imc-pmu.h b/arch/powerpc/include/asm/imc-pmu.h
index 4da4fcb..4f89799 100644
--- a/arch/powerpc/include/asm/imc-pmu.h
+++ b/arch/powerpc/include/asm/imc-pmu.h
@@ -99,6 +99,11 @@
*/
#define IMC_TRACE_RECORD_TB1_MASK 0x3ffffffffffULL
+/*
+ * Bit 0:1 in third DW of IMC trace record
+ * specifies the MSR[HV PR] values.
+ */
+#define IMC_TRACE_RECORD_VAL_HVPR(x) ((x) >> 62)
/*
* Device tree parser code detects IMC pmu support and
diff --git a/arch/powerpc/include/asm/inst.h b/arch/powerpc/include/asm/inst.h
index 45f3ec8..cc73c12 100644
--- a/arch/powerpc/include/asm/inst.h
+++ b/arch/powerpc/include/asm/inst.h
@@ -122,6 +122,25 @@
#endif
}
+#define PPC_INST_STR_LEN sizeof("00000000 00000000")
+
+static inline char *__ppc_inst_as_str(char str[PPC_INST_STR_LEN], struct ppc_inst x)
+{
+ if (ppc_inst_prefixed(x))
+ sprintf(str, "%08x %08x", ppc_inst_val(x), ppc_inst_suffix(x));
+ else
+ sprintf(str, "%08x", ppc_inst_val(x));
+
+ return str;
+}
+
+#define ppc_inst_as_str(x) \
+({ \
+ char __str[PPC_INST_STR_LEN]; \
+ __ppc_inst_as_str(__str, x); \
+ __str; \
+})
+
int probe_user_read_inst(struct ppc_inst *inst,
struct ppc_inst __user *nip);
diff --git a/arch/powerpc/include/asm/kasan.h b/arch/powerpc/include/asm/kasan.h
index be85c70..d635b96 100644
--- a/arch/powerpc/include/asm/kasan.h
+++ b/arch/powerpc/include/asm/kasan.h
@@ -27,10 +27,12 @@
#ifdef CONFIG_KASAN
void kasan_early_init(void);
+void kasan_mmu_init(void);
void kasan_init(void);
void kasan_late_init(void);
#else
static inline void kasan_init(void) { }
+static inline void kasan_mmu_init(void) { }
static inline void kasan_late_init(void) { }
#endif
diff --git a/arch/powerpc/include/asm/kexec.h b/arch/powerpc/include/asm/kexec.h
index c684768..55d6ede 100644
--- a/arch/powerpc/include/asm/kexec.h
+++ b/arch/powerpc/include/asm/kexec.h
@@ -100,15 +100,26 @@
#ifdef CONFIG_KEXEC_FILE
extern const struct kexec_file_ops kexec_elf64_ops;
-#ifdef CONFIG_IMA_KEXEC
#define ARCH_HAS_KIMAGE_ARCH
struct kimage_arch {
+ struct crash_mem *exclude_ranges;
+
+ unsigned long backup_start;
+ void *backup_buf;
+
+ unsigned long elfcorehdr_addr;
+ unsigned long elf_headers_sz;
+ void *elf_headers;
+
+#ifdef CONFIG_IMA_KEXEC
phys_addr_t ima_buffer_addr;
size_t ima_buffer_size;
-};
#endif
+};
+char *setup_kdump_cmdline(struct kimage *image, char *cmdline,
+ unsigned long cmdline_len);
int setup_purgatory(struct kimage *image, const void *slave_code,
const void *fdt, unsigned long kernel_load_addr,
unsigned long fdt_load_addr);
@@ -116,6 +127,20 @@
unsigned long initrd_load_addr, unsigned long initrd_len,
const char *cmdline);
int delete_fdt_mem_rsv(void *fdt, unsigned long start, unsigned long size);
+
+#ifdef CONFIG_PPC64
+struct kexec_buf;
+
+int load_crashdump_segments_ppc64(struct kimage *image,
+ struct kexec_buf *kbuf);
+int setup_purgatory_ppc64(struct kimage *image, const void *slave_code,
+ const void *fdt, unsigned long kernel_load_addr,
+ unsigned long fdt_load_addr);
+int setup_new_fdt_ppc64(const struct kimage *image, void *fdt,
+ unsigned long initrd_load_addr,
+ unsigned long initrd_len, const char *cmdline);
+#endif /* CONFIG_PPC64 */
+
#endif /* CONFIG_KEXEC_FILE */
#else /* !CONFIG_KEXEC_CORE */
@@ -150,6 +175,18 @@
}
#endif /* CONFIG_KEXEC_CORE */
+
+#ifdef CONFIG_PPC_BOOK3S_64
+#include <asm/book3s/64/kexec.h>
+#endif
+
+#ifndef reset_sprs
+#define reset_sprs reset_sprs
+static inline void reset_sprs(void)
+{
+}
+#endif
+
#endif /* ! __ASSEMBLY__ */
#endif /* __KERNEL__ */
#endif /* _ASM_POWERPC_KEXEC_H */
diff --git a/arch/powerpc/include/asm/kexec_ranges.h b/arch/powerpc/include/asm/kexec_ranges.h
new file mode 100644
index 0000000..7a90000
--- /dev/null
+++ b/arch/powerpc/include/asm/kexec_ranges.h
@@ -0,0 +1,25 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+#ifndef _ASM_POWERPC_KEXEC_RANGES_H
+#define _ASM_POWERPC_KEXEC_RANGES_H
+
+#define MEM_RANGE_CHUNK_SZ 2048 /* Memory ranges size chunk */
+
+void sort_memory_ranges(struct crash_mem *mrngs, bool merge);
+struct crash_mem *realloc_mem_ranges(struct crash_mem **mem_ranges);
+int add_mem_range(struct crash_mem **mem_ranges, u64 base, u64 size);
+int add_tce_mem_ranges(struct crash_mem **mem_ranges);
+int add_initrd_mem_range(struct crash_mem **mem_ranges);
+#ifdef CONFIG_PPC_BOOK3S_64
+int add_htab_mem_range(struct crash_mem **mem_ranges);
+#else
+static inline int add_htab_mem_range(struct crash_mem **mem_ranges)
+{
+ return 0;
+}
+#endif
+int add_kernel_mem_range(struct crash_mem **mem_ranges);
+int add_rtas_mem_range(struct crash_mem **mem_ranges);
+int add_opal_mem_range(struct crash_mem **mem_ranges);
+int add_reserved_mem_ranges(struct crash_mem **mem_ranges);
+
+#endif /* _ASM_POWERPC_KEXEC_RANGES_H */
diff --git a/arch/powerpc/include/asm/kvm_book3s_asm.h b/arch/powerpc/include/asm/kvm_book3s_asm.h
index 45704f2..078f464 100644
--- a/arch/powerpc/include/asm/kvm_book3s_asm.h
+++ b/arch/powerpc/include/asm/kvm_book3s_asm.h
@@ -119,7 +119,7 @@
void __iomem *xive_tima_virt;
u32 saved_xirr;
u64 dabr;
- u64 host_mmcr[7]; /* MMCR 0,1,A, SIAR, SDAR, MMCR2, SIER */
+ u64 host_mmcr[10]; /* MMCR 0,1,A, SIAR, SDAR, MMCR2, SIER, MMCR3, SIER2/3 */
u32 host_pmc[8];
u64 host_purr;
u64 host_spurr;
diff --git a/arch/powerpc/include/asm/kvm_booke.h b/arch/powerpc/include/asm/kvm_booke.h
index 310ba48..0c3401b 100644
--- a/arch/powerpc/include/asm/kvm_booke.h
+++ b/arch/powerpc/include/asm/kvm_booke.h
@@ -89,10 +89,12 @@
return vcpu->arch.regs.nip;
}
+#ifdef CONFIG_BOOKE
static inline ulong kvmppc_get_fault_dar(struct kvm_vcpu *vcpu)
{
return vcpu->arch.fault_dear;
}
+#endif
static inline bool kvmppc_supports_magic_page(struct kvm_vcpu *vcpu)
{
diff --git a/arch/powerpc/include/asm/kvm_host.h b/arch/powerpc/include/asm/kvm_host.h
index 7e2d061..e020d26 100644
--- a/arch/powerpc/include/asm/kvm_host.h
+++ b/arch/powerpc/include/asm/kvm_host.h
@@ -637,12 +637,14 @@
u32 ccr1;
u32 dbsr;
- u64 mmcr[5];
+ u64 mmcr[4]; /* MMCR0, MMCR1, MMCR2, MMCR3 */
+ u64 mmcra;
+ u64 mmcrs;
u32 pmc[8];
u32 spmc[2];
u64 siar;
u64 sdar;
- u64 sier;
+ u64 sier[3];
#ifdef CONFIG_PPC_TRANSACTIONAL_MEM
u64 tfhar;
u64 texasr;
diff --git a/arch/powerpc/include/asm/kvm_para.h b/arch/powerpc/include/asm/kvm_para.h
index 9c1f6b4..7446120 100644
--- a/arch/powerpc/include/asm/kvm_para.h
+++ b/arch/powerpc/include/asm/kvm_para.h
@@ -8,35 +8,15 @@
#ifndef __POWERPC_KVM_PARA_H__
#define __POWERPC_KVM_PARA_H__
+#include <asm/firmware.h>
+
#include <uapi/asm/kvm_para.h>
-#ifdef CONFIG_KVM_GUEST
-
-#include <linux/of.h>
-
static inline int kvm_para_available(void)
{
- struct device_node *hyper_node;
-
- hyper_node = of_find_node_by_path("/hypervisor");
- if (!hyper_node)
- return 0;
-
- if (!of_device_is_compatible(hyper_node, "linux,kvm"))
- return 0;
-
- return 1;
+ return IS_ENABLED(CONFIG_KVM_GUEST) && is_kvm_guest();
}
-#else
-
-static inline int kvm_para_available(void)
-{
- return 0;
-}
-
-#endif
-
static inline unsigned int kvm_arch_para_features(void)
{
unsigned long r;
diff --git a/arch/powerpc/include/asm/machdep.h b/arch/powerpc/include/asm/machdep.h
index 7bcb644..a90b892 100644
--- a/arch/powerpc/include/asm/machdep.h
+++ b/arch/powerpc/include/asm/machdep.h
@@ -131,7 +131,7 @@
unsigned long dabrx);
/* Set DAWR for this platform, leave empty for default implementation */
- int (*set_dawr)(unsigned long dawr,
+ int (*set_dawr)(int nr, unsigned long dawr,
unsigned long dawrx);
#ifdef CONFIG_PPC32 /* XXX for now */
diff --git a/arch/powerpc/include/asm/mce.h b/arch/powerpc/include/asm/mce.h
index 376a395..adf2cda 100644
--- a/arch/powerpc/include/asm/mce.h
+++ b/arch/powerpc/include/asm/mce.h
@@ -86,6 +86,7 @@
enum MCE_UserErrorType {
MCE_USER_ERROR_INDETERMINATE = 0,
MCE_USER_ERROR_TLBIE = 1,
+ MCE_USER_ERROR_SCV = 2,
};
enum MCE_RaErrorType {
@@ -220,6 +221,8 @@
unsigned long addr_to_pfn(struct pt_regs *regs, unsigned long addr);
extern void mce_common_process_ue(struct pt_regs *regs,
struct mce_error_info *mce_err);
+int mce_register_notifier(struct notifier_block *nb);
+int mce_unregister_notifier(struct notifier_block *nb);
#ifdef CONFIG_PPC_BOOK3S_64
void flush_and_reload_slb(void);
#endif /* CONFIG_PPC_BOOK3S_64 */
diff --git a/arch/powerpc/include/asm/mman.h b/arch/powerpc/include/asm/mman.h
index d610c2e..7c07728 100644
--- a/arch/powerpc/include/asm/mman.h
+++ b/arch/powerpc/include/asm/mman.h
@@ -13,42 +13,20 @@
#include <linux/pkeys.h>
#include <asm/cpu_has_feature.h>
-/*
- * This file is included by linux/mman.h, so we can't use cacl_vm_prot_bits()
- * here. How important is the optimization?
- */
+#ifdef CONFIG_PPC_MEM_KEYS
static inline unsigned long arch_calc_vm_prot_bits(unsigned long prot,
unsigned long pkey)
{
-#ifdef CONFIG_PPC_MEM_KEYS
- return (((prot & PROT_SAO) ? VM_SAO : 0) | pkey_to_vmflag_bits(pkey));
-#else
- return ((prot & PROT_SAO) ? VM_SAO : 0);
-#endif
+ return pkey_to_vmflag_bits(pkey);
}
#define arch_calc_vm_prot_bits(prot, pkey) arch_calc_vm_prot_bits(prot, pkey)
static inline pgprot_t arch_vm_get_page_prot(unsigned long vm_flags)
{
-#ifdef CONFIG_PPC_MEM_KEYS
- return (vm_flags & VM_SAO) ?
- __pgprot(_PAGE_SAO | vmflag_to_pte_pkey_bits(vm_flags)) :
- __pgprot(0 | vmflag_to_pte_pkey_bits(vm_flags));
-#else
- return (vm_flags & VM_SAO) ? __pgprot(_PAGE_SAO) : __pgprot(0);
-#endif
+ return __pgprot(vmflag_to_pte_pkey_bits(vm_flags));
}
#define arch_vm_get_page_prot(vm_flags) arch_vm_get_page_prot(vm_flags)
-
-static inline bool arch_validate_prot(unsigned long prot, unsigned long addr)
-{
- if (prot & ~(PROT_READ | PROT_WRITE | PROT_EXEC | PROT_SEM | PROT_SAO))
- return false;
- if ((prot & PROT_SAO) && !cpu_has_feature(CPU_FTR_SAO))
- return false;
- return true;
-}
-#define arch_validate_prot arch_validate_prot
+#endif
#endif /* CONFIG_PPC64 */
#endif /* _ASM_POWERPC_MMAN_H */
diff --git a/arch/powerpc/include/asm/mmu.h b/arch/powerpc/include/asm/mmu.h
index f4ac25d..255a183 100644
--- a/arch/powerpc/include/asm/mmu.h
+++ b/arch/powerpc/include/asm/mmu.h
@@ -29,6 +29,19 @@
*/
/*
+ * Support for KUEP feature.
+ */
+#define MMU_FTR_KUEP ASM_CONST(0x00000400)
+
+/*
+ * Support for memory protection keys.
+ */
+#define MMU_FTR_PKEY ASM_CONST(0x00000800)
+
+/* Guest Translation Shootdown Enable */
+#define MMU_FTR_GTSE ASM_CONST(0x00001000)
+
+/*
* Support for 68 bit VA space. We added that from ISA 2.05
*/
#define MMU_FTR_68_BIT_VA ASM_CONST(0x00002000)
@@ -173,10 +186,18 @@
#endif
#ifdef CONFIG_PPC_RADIX_MMU
MMU_FTR_TYPE_RADIX |
+ MMU_FTR_GTSE |
#ifdef CONFIG_PPC_KUAP
MMU_FTR_RADIX_KUAP |
#endif /* CONFIG_PPC_KUAP */
#endif /* CONFIG_PPC_RADIX_MMU */
+#ifdef CONFIG_PPC_MEM_KEYS
+ MMU_FTR_PKEY |
+#endif
+#ifdef CONFIG_PPC_KUEP
+ MMU_FTR_KUEP |
+#endif /* CONFIG_PPC_KUAP */
+
0,
};
@@ -356,6 +377,8 @@
phys_addr_t first_memblock_size);
static inline void mmu_early_init_devtree(void) { }
+static inline void pkey_early_init_devtree(void) {}
+
extern void *abatron_pteptrs[2];
#endif /* __ASSEMBLY__ */
#endif
diff --git a/arch/powerpc/include/asm/mmu_context.h b/arch/powerpc/include/asm/mmu_context.h
index 1a474f6b..7f3658a 100644
--- a/arch/powerpc/include/asm/mmu_context.h
+++ b/arch/powerpc/include/asm/mmu_context.h
@@ -218,8 +218,6 @@
static inline void dec_mm_active_cpus(struct mm_struct *mm) { }
static inline void mm_context_add_copro(struct mm_struct *mm) { }
static inline void mm_context_remove_copro(struct mm_struct *mm) { }
-static inline void mm_context_add_vas_windows(struct mm_struct *mm) { }
-static inline void mm_context_remove_vas_windows(struct mm_struct *mm) { }
#endif
diff --git a/arch/powerpc/include/asm/nohash/32/pgtable.h b/arch/powerpc/include/asm/nohash/32/pgtable.h
index b0afbdd..b9e134d 100644
--- a/arch/powerpc/include/asm/nohash/32/pgtable.h
+++ b/arch/powerpc/include/asm/nohash/32/pgtable.h
@@ -249,6 +249,18 @@
return old;
}
+
+#ifdef CONFIG_PPC_16K_PAGES
+#define __HAVE_ARCH_PTEP_GET
+static inline pte_t ptep_get(pte_t *ptep)
+{
+ pte_basic_t val = READ_ONCE(ptep->pte);
+ pte_t pte = {val, val, val, val};
+
+ return pte;
+}
+#endif /* CONFIG_PPC_16K_PAGES */
+
#else
static inline pte_basic_t pte_update(struct mm_struct *mm, unsigned long addr, pte_t *p,
unsigned long clr, unsigned long set, int huge)
@@ -284,16 +296,6 @@
return __pte(pte_update(mm, addr, ptep, ~0, 0, 0));
}
-#if defined(CONFIG_PPC_8xx) && defined(CONFIG_PPC_16K_PAGES)
-#define __HAVE_ARCH_PTEP_GET
-static inline pte_t ptep_get(pte_t *ptep)
-{
- pte_t pte = {READ_ONCE(ptep->pte), 0, 0, 0};
-
- return pte;
-}
-#endif
-
#define __HAVE_ARCH_PTEP_SET_WRPROTECT
static inline void ptep_set_wrprotect(struct mm_struct *mm, unsigned long addr,
pte_t *ptep)
diff --git a/arch/powerpc/include/asm/nohash/64/pgtable.h b/arch/powerpc/include/asm/nohash/64/pgtable.h
index 6cb8aa3..59ee9fa 100644
--- a/arch/powerpc/include/asm/nohash/64/pgtable.h
+++ b/arch/powerpc/include/asm/nohash/64/pgtable.h
@@ -82,8 +82,6 @@
*/
#include <asm/nohash/pte-book3e.h>
-#define _PAGE_SAO 0
-
#define PTE_RPN_MASK (~((1UL << PTE_RPN_SHIFT) - 1))
/*
diff --git a/arch/powerpc/include/asm/paca.h b/arch/powerpc/include/asm/paca.h
index 84b2564..9454d29 100644
--- a/arch/powerpc/include/asm/paca.h
+++ b/arch/powerpc/include/asm/paca.h
@@ -225,6 +225,7 @@
u16 in_mce;
u8 hmi_event_available; /* HMI event is available */
u8 hmi_p9_special_emu; /* HMI P9 special emulation */
+ u32 hmi_irqs; /* HMI irq stat */
#endif
u8 ftrace_enabled; /* Hard disable ftrace */
diff --git a/arch/powerpc/include/asm/page.h b/arch/powerpc/include/asm/page.h
index a63fe6f..2546872 100644
--- a/arch/powerpc/include/asm/page.h
+++ b/arch/powerpc/include/asm/page.h
@@ -255,8 +255,10 @@
*/
#ifdef CONFIG_PPC_BOOK3E_64
#define is_kernel_addr(x) ((x) >= 0x8000000000000000ul)
-#else
+#elif defined(CONFIG_PPC_BOOK3S_64)
#define is_kernel_addr(x) ((x) >= PAGE_OFFSET)
+#else
+#define is_kernel_addr(x) ((x) >= TASK_SIZE)
#endif
#ifndef CONFIG_PPC_BOOK3S_64
diff --git a/arch/powerpc/include/asm/paravirt.h b/arch/powerpc/include/asm/paravirt.h
new file mode 100644
index 0000000..9362c94
--- /dev/null
+++ b/arch/powerpc/include/asm/paravirt.h
@@ -0,0 +1,87 @@
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+#ifndef _ASM_POWERPC_PARAVIRT_H
+#define _ASM_POWERPC_PARAVIRT_H
+
+#include <linux/jump_label.h>
+#include <asm/smp.h>
+#ifdef CONFIG_PPC64
+#include <asm/paca.h>
+#include <asm/hvcall.h>
+#endif
+
+#ifdef CONFIG_PPC_SPLPAR
+DECLARE_STATIC_KEY_FALSE(shared_processor);
+
+static inline bool is_shared_processor(void)
+{
+ return static_branch_unlikely(&shared_processor);
+}
+
+/* If bit 0 is set, the cpu has been preempted */
+static inline u32 yield_count_of(int cpu)
+{
+ __be32 yield_count = READ_ONCE(lppaca_of(cpu).yield_count);
+ return be32_to_cpu(yield_count);
+}
+
+static inline void yield_to_preempted(int cpu, u32 yield_count)
+{
+ plpar_hcall_norets(H_CONFER, get_hard_smp_processor_id(cpu), yield_count);
+}
+
+static inline void prod_cpu(int cpu)
+{
+ plpar_hcall_norets(H_PROD, get_hard_smp_processor_id(cpu));
+}
+
+static inline void yield_to_any(void)
+{
+ plpar_hcall_norets(H_CONFER, -1, 0);
+}
+#else
+static inline bool is_shared_processor(void)
+{
+ return false;
+}
+
+static inline u32 yield_count_of(int cpu)
+{
+ return 0;
+}
+
+extern void ___bad_yield_to_preempted(void);
+static inline void yield_to_preempted(int cpu, u32 yield_count)
+{
+ ___bad_yield_to_preempted(); /* This would be a bug */
+}
+
+extern void ___bad_yield_to_any(void);
+static inline void yield_to_any(void)
+{
+ ___bad_yield_to_any(); /* This would be a bug */
+}
+
+extern void ___bad_prod_cpu(void);
+static inline void prod_cpu(int cpu)
+{
+ ___bad_prod_cpu(); /* This would be a bug */
+}
+
+#endif
+
+#define vcpu_is_preempted vcpu_is_preempted
+static inline bool vcpu_is_preempted(int cpu)
+{
+ if (!is_shared_processor())
+ return false;
+ if (yield_count_of(cpu) & 1)
+ return true;
+ return false;
+}
+
+static inline bool pv_is_native_spin_unlock(void)
+{
+ return !is_shared_processor();
+}
+
+#endif /* _ASM_POWERPC_PARAVIRT_H */
diff --git a/arch/powerpc/include/asm/pci-bridge.h b/arch/powerpc/include/asm/pci-bridge.h
index b92e81b..d2a2a14 100644
--- a/arch/powerpc/include/asm/pci-bridge.h
+++ b/arch/powerpc/include/asm/pci-bridge.h
@@ -202,7 +202,6 @@
#define IODA_INVALID_PE 0xFFFFFFFF
unsigned int pe_number;
#ifdef CONFIG_PCI_IOV
- int vf_index; /* VF index in the PF */
u16 vfs_expanded; /* number of VFs IOV BAR expanded */
u16 num_vfs; /* number of VFs enabled*/
unsigned int *pe_num_map; /* PE# for the first VF PE or array */
diff --git a/arch/powerpc/include/asm/percpu.h b/arch/powerpc/include/asm/percpu.h
index dce863a..8e5b7d0b8 100644
--- a/arch/powerpc/include/asm/percpu.h
+++ b/arch/powerpc/include/asm/percpu.h
@@ -10,8 +10,6 @@
#ifdef CONFIG_SMP
-#include <asm/paca.h>
-
#define __my_cpu_offset local_paca->data_offset
#endif /* CONFIG_SMP */
@@ -19,4 +17,6 @@
#include <asm-generic/percpu.h>
+#include <asm/paca.h>
+
#endif /* _ASM_POWERPC_PERCPU_H_ */
diff --git a/arch/powerpc/include/asm/perf_event.h b/arch/powerpc/include/asm/perf_event.h
index eed3954..1e8b2e1 100644
--- a/arch/powerpc/include/asm/perf_event.h
+++ b/arch/powerpc/include/asm/perf_event.h
@@ -12,6 +12,8 @@
#ifdef CONFIG_PPC_PERF_CTRS
#include <asm/perf_event_server.h>
+#else
+static inline bool is_sier_available(void) { return false; }
#endif
#ifdef CONFIG_FSL_EMB_PERF_EVENT
diff --git a/arch/powerpc/include/asm/perf_event_server.h b/arch/powerpc/include/asm/perf_event_server.h
index 3e9703f..86c9eb06 100644
--- a/arch/powerpc/include/asm/perf_event_server.h
+++ b/arch/powerpc/include/asm/perf_event_server.h
@@ -17,6 +17,13 @@
struct perf_event;
+struct mmcr_regs {
+ unsigned long mmcr0;
+ unsigned long mmcr1;
+ unsigned long mmcr2;
+ unsigned long mmcra;
+ unsigned long mmcr3;
+};
/*
* This struct provides the constants and functions needed to
* describe the PMU on a particular POWER-family CPU.
@@ -28,7 +35,7 @@
unsigned long add_fields;
unsigned long test_adder;
int (*compute_mmcr)(u64 events[], int n_ev,
- unsigned int hwc[], unsigned long mmcr[],
+ unsigned int hwc[], struct mmcr_regs *mmcr,
struct perf_event *pevents[]);
int (*get_constraint)(u64 event_id, unsigned long *mskp,
unsigned long *valp);
@@ -41,13 +48,13 @@
unsigned long group_constraint_val;
u64 (*bhrb_filter_map)(u64 branch_sample_type);
void (*config_bhrb)(u64 pmu_bhrb_filter);
- void (*disable_pmc)(unsigned int pmc, unsigned long mmcr[]);
+ void (*disable_pmc)(unsigned int pmc, struct mmcr_regs *mmcr);
int (*limited_pmc_event)(u64 event_id);
u32 flags;
const struct attribute_group **attr_groups;
int n_generic;
int *generic_events;
- int (*cache_events)[PERF_COUNT_HW_CACHE_MAX]
+ u64 (*cache_events)[PERF_COUNT_HW_CACHE_MAX]
[PERF_COUNT_HW_CACHE_OP_MAX]
[PERF_COUNT_HW_CACHE_RESULT_MAX];
@@ -69,6 +76,7 @@
#define PPMU_HAS_SIER 0x00000040 /* Has SIER */
#define PPMU_ARCH_207S 0x00000080 /* PMC is architecture v2.07S */
#define PPMU_NO_SIAR 0x00000100 /* Do not use SIAR */
+#define PPMU_ARCH_31 0x00000200 /* Has MMCR3, SIER2 and SIER3 */
/*
* Values for flags to get_alternatives()
diff --git a/arch/powerpc/include/asm/pkeys.h b/arch/powerpc/include/asm/pkeys.h
index 2fe6cae..a795104 100644
--- a/arch/powerpc/include/asm/pkeys.h
+++ b/arch/powerpc/include/asm/pkeys.h
@@ -11,9 +11,7 @@
#include <linux/jump_label.h>
#include <asm/firmware.h>
-DECLARE_STATIC_KEY_TRUE(pkey_disabled);
-extern int pkeys_total; /* total pkeys as per device tree */
-extern u32 initial_allocation_mask; /* bits set for the initially allocated keys */
+extern int num_pkey;
extern u32 reserved_allocation_mask; /* bits set for reserved keys */
#define ARCH_VM_PKEY_FLAGS (VM_PKEY_BIT0 | VM_PKEY_BIT1 | VM_PKEY_BIT2 | \
@@ -25,48 +23,28 @@
PKEY_DISABLE_WRITE | \
PKEY_DISABLE_EXECUTE)
+#ifdef CONFIG_PPC_BOOK3S_64
+#include <asm/book3s/64/pkeys.h>
+#else
+#error "Not supported"
+#endif
+
+
static inline u64 pkey_to_vmflag_bits(u16 pkey)
{
return (((u64)pkey << VM_PKEY_SHIFT) & ARCH_VM_PKEY_FLAGS);
}
-static inline u64 vmflag_to_pte_pkey_bits(u64 vm_flags)
-{
- if (static_branch_likely(&pkey_disabled))
- return 0x0UL;
-
- return (((vm_flags & VM_PKEY_BIT0) ? H_PTE_PKEY_BIT4 : 0x0UL) |
- ((vm_flags & VM_PKEY_BIT1) ? H_PTE_PKEY_BIT3 : 0x0UL) |
- ((vm_flags & VM_PKEY_BIT2) ? H_PTE_PKEY_BIT2 : 0x0UL) |
- ((vm_flags & VM_PKEY_BIT3) ? H_PTE_PKEY_BIT1 : 0x0UL) |
- ((vm_flags & VM_PKEY_BIT4) ? H_PTE_PKEY_BIT0 : 0x0UL));
-}
-
static inline int vma_pkey(struct vm_area_struct *vma)
{
- if (static_branch_likely(&pkey_disabled))
+ if (!mmu_has_feature(MMU_FTR_PKEY))
return 0;
return (vma->vm_flags & ARCH_VM_PKEY_FLAGS) >> VM_PKEY_SHIFT;
}
-#define arch_max_pkey() pkeys_total
-
-static inline u64 pte_to_hpte_pkey_bits(u64 pteflags)
+static inline int arch_max_pkey(void)
{
- return (((pteflags & H_PTE_PKEY_BIT0) ? HPTE_R_KEY_BIT0 : 0x0UL) |
- ((pteflags & H_PTE_PKEY_BIT1) ? HPTE_R_KEY_BIT1 : 0x0UL) |
- ((pteflags & H_PTE_PKEY_BIT2) ? HPTE_R_KEY_BIT2 : 0x0UL) |
- ((pteflags & H_PTE_PKEY_BIT3) ? HPTE_R_KEY_BIT3 : 0x0UL) |
- ((pteflags & H_PTE_PKEY_BIT4) ? HPTE_R_KEY_BIT4 : 0x0UL));
-}
-
-static inline u16 pte_to_pkey_bits(u64 pteflags)
-{
- return (((pteflags & H_PTE_PKEY_BIT0) ? 0x10 : 0x0UL) |
- ((pteflags & H_PTE_PKEY_BIT1) ? 0x8 : 0x0UL) |
- ((pteflags & H_PTE_PKEY_BIT2) ? 0x4 : 0x0UL) |
- ((pteflags & H_PTE_PKEY_BIT3) ? 0x2 : 0x0UL) |
- ((pteflags & H_PTE_PKEY_BIT4) ? 0x1 : 0x0UL));
+ return num_pkey;
}
#define pkey_alloc_mask(pkey) (0x1 << pkey)
@@ -114,9 +92,8 @@
u32 all_pkeys_mask = (u32)(~(0x0));
int ret;
- if (static_branch_likely(&pkey_disabled))
+ if (!mmu_has_feature(MMU_FTR_PKEY))
return -1;
-
/*
* Are we out of pkeys? We must handle this specially because ffz()
* behavior is undefined if there are no zeros.
@@ -132,7 +109,7 @@
static inline int mm_pkey_free(struct mm_struct *mm, int pkey)
{
- if (static_branch_likely(&pkey_disabled))
+ if (!mmu_has_feature(MMU_FTR_PKEY))
return -1;
if (!mm_pkey_is_allocated(mm, pkey))
@@ -147,21 +124,13 @@
* Try to dedicate one of the protection keys to be used as an
* execute-only protection key.
*/
-extern int __execute_only_pkey(struct mm_struct *mm);
-static inline int execute_only_pkey(struct mm_struct *mm)
-{
- if (static_branch_likely(&pkey_disabled))
- return -1;
-
- return __execute_only_pkey(mm);
-}
-
+extern int execute_only_pkey(struct mm_struct *mm);
extern int __arch_override_mprotect_pkey(struct vm_area_struct *vma,
int prot, int pkey);
static inline int arch_override_mprotect_pkey(struct vm_area_struct *vma,
int prot, int pkey)
{
- if (static_branch_likely(&pkey_disabled))
+ if (!mmu_has_feature(MMU_FTR_PKEY))
return 0;
/*
@@ -179,7 +148,7 @@
static inline int arch_set_user_pkey_access(struct task_struct *tsk, int pkey,
unsigned long init_val)
{
- if (static_branch_likely(&pkey_disabled))
+ if (!mmu_has_feature(MMU_FTR_PKEY))
return -EINVAL;
/*
@@ -196,7 +165,7 @@
static inline bool arch_pkeys_enabled(void)
{
- return !static_branch_likely(&pkey_disabled);
+ return mmu_has_feature(MMU_FTR_PKEY);
}
extern void pkey_mm_init(struct mm_struct *mm);
diff --git a/arch/powerpc/include/asm/plpar_wrappers.h b/arch/powerpc/include/asm/plpar_wrappers.h
index 4497c8a..ece84a4 100644
--- a/arch/powerpc/include/asm/plpar_wrappers.h
+++ b/arch/powerpc/include/asm/plpar_wrappers.h
@@ -312,7 +312,12 @@
static inline long plpar_set_watchpoint0(unsigned long dawr0, unsigned long dawrx0)
{
- return plpar_set_mode(0, H_SET_MODE_RESOURCE_SET_DAWR, dawr0, dawrx0);
+ return plpar_set_mode(0, H_SET_MODE_RESOURCE_SET_DAWR0, dawr0, dawrx0);
+}
+
+static inline long plpar_set_watchpoint1(unsigned long dawr1, unsigned long dawrx1)
+{
+ return plpar_set_mode(0, H_SET_MODE_RESOURCE_SET_DAWR1, dawr1, dawrx1);
}
static inline long plpar_signal_sys_reset(long cpu)
@@ -334,6 +339,51 @@
return rc;
}
+/*
+ * Wrapper to H_RPT_INVALIDATE hcall that handles return values appropriately
+ *
+ * - Returns H_SUCCESS on success
+ * - For H_BUSY return value, we retry the hcall.
+ * - For any other hcall failures, attempt a full flush once before
+ * resorting to BUG().
+ *
+ * Note: This hcall is expected to fail only very rarely. The correct
+ * error recovery of killing the process/guest will be eventually
+ * needed.
+ */
+static inline long pseries_rpt_invalidate(u32 pid, u64 target, u64 type,
+ u64 page_sizes, u64 start, u64 end)
+{
+ long rc;
+ unsigned long all;
+
+ while (true) {
+ rc = plpar_hcall_norets(H_RPT_INVALIDATE, pid, target, type,
+ page_sizes, start, end);
+ if (rc == H_BUSY) {
+ cpu_relax();
+ continue;
+ } else if (rc == H_SUCCESS)
+ return rc;
+
+ /* Flush request failed, try with a full flush once */
+ if (type & H_RPTI_TYPE_NESTED)
+ all = H_RPTI_TYPE_NESTED | H_RPTI_TYPE_NESTED_ALL;
+ else
+ all = H_RPTI_TYPE_ALL;
+retry:
+ rc = plpar_hcall_norets(H_RPT_INVALIDATE, pid, target,
+ all, page_sizes, 0, -1UL);
+ if (rc == H_BUSY) {
+ cpu_relax();
+ goto retry;
+ } else if (rc == H_SUCCESS)
+ return rc;
+
+ BUG();
+ }
+}
+
#else /* !CONFIG_PPC_PSERIES */
static inline long plpar_set_ciabr(unsigned long ciabr)
@@ -346,6 +396,13 @@
{
return 0;
}
+
+static inline long pseries_rpt_invalidate(u32 pid, u64 target, u64 type,
+ u64 page_sizes, u64 start, u64 end)
+{
+ return 0;
+}
+
#endif /* CONFIG_PPC_PSERIES */
#endif /* _ASM_POWERPC_PLPAR_WRAPPERS_H */
diff --git a/arch/powerpc/include/asm/pnv-ocxl.h b/arch/powerpc/include/asm/pnv-ocxl.h
index 7de8264..ee79d2c 100644
--- a/arch/powerpc/include/asm/pnv-ocxl.h
+++ b/arch/powerpc/include/asm/pnv-ocxl.h
@@ -9,28 +9,26 @@
#define PNV_OCXL_TL_BITS_PER_RATE 4
#define PNV_OCXL_TL_RATE_BUF_SIZE ((PNV_OCXL_TL_MAX_TEMPLATE+1) * PNV_OCXL_TL_BITS_PER_RATE / 8)
-extern int pnv_ocxl_get_actag(struct pci_dev *dev, u16 *base, u16 *enabled,
- u16 *supported);
-extern int pnv_ocxl_get_pasid_count(struct pci_dev *dev, int *count);
+int pnv_ocxl_get_actag(struct pci_dev *dev, u16 *base, u16 *enabled, u16 *supported);
+int pnv_ocxl_get_pasid_count(struct pci_dev *dev, int *count);
-extern int pnv_ocxl_get_tl_cap(struct pci_dev *dev, long *cap,
+int pnv_ocxl_get_tl_cap(struct pci_dev *dev, long *cap,
char *rate_buf, int rate_buf_size);
-extern int pnv_ocxl_set_tl_conf(struct pci_dev *dev, long cap,
- uint64_t rate_buf_phys, int rate_buf_size);
+int pnv_ocxl_set_tl_conf(struct pci_dev *dev, long cap,
+ uint64_t rate_buf_phys, int rate_buf_size);
-extern int pnv_ocxl_get_xsl_irq(struct pci_dev *dev, int *hwirq);
-extern void pnv_ocxl_unmap_xsl_regs(void __iomem *dsisr, void __iomem *dar,
- void __iomem *tfc, void __iomem *pe_handle);
-extern int pnv_ocxl_map_xsl_regs(struct pci_dev *dev, void __iomem **dsisr,
- void __iomem **dar, void __iomem **tfc,
- void __iomem **pe_handle);
+int pnv_ocxl_get_xsl_irq(struct pci_dev *dev, int *hwirq);
+void pnv_ocxl_unmap_xsl_regs(void __iomem *dsisr, void __iomem *dar,
+ void __iomem *tfc, void __iomem *pe_handle);
+int pnv_ocxl_map_xsl_regs(struct pci_dev *dev, void __iomem **dsisr,
+ void __iomem **dar, void __iomem **tfc,
+ void __iomem **pe_handle);
-extern int pnv_ocxl_spa_setup(struct pci_dev *dev, void *spa_mem, int PE_mask,
- void **platform_data);
-extern void pnv_ocxl_spa_release(void *platform_data);
-extern int pnv_ocxl_spa_remove_pe_from_cache(void *platform_data, int pe_handle);
+int pnv_ocxl_spa_setup(struct pci_dev *dev, void *spa_mem, int PE_mask, void **platform_data);
+void pnv_ocxl_spa_release(void *platform_data);
+int pnv_ocxl_spa_remove_pe_from_cache(void *platform_data, int pe_handle);
-extern int pnv_ocxl_alloc_xive_irq(u32 *irq, u64 *trigger_addr);
-extern void pnv_ocxl_free_xive_irq(u32 irq);
+int pnv_ocxl_alloc_xive_irq(u32 *irq, u64 *trigger_addr);
+void pnv_ocxl_free_xive_irq(u32 irq);
#endif /* _ASM_PNV_OCXL_H */
diff --git a/arch/powerpc/include/asm/ppc-opcode.h b/arch/powerpc/include/asm/ppc-opcode.h
index 2a39c71..a6e3700 100644
--- a/arch/powerpc/include/asm/ppc-opcode.h
+++ b/arch/powerpc/include/asm/ppc-opcode.h
@@ -76,6 +76,19 @@
#define __REGA0_R30 30
#define __REGA0_R31 31
+#define IMM_L(i) ((uintptr_t)(i) & 0xffff)
+#define IMM_DS(i) ((uintptr_t)(i) & 0xfffc)
+
+/*
+ * 16-bit immediate helper macros: HA() is for use with sign-extending instrs
+ * (e.g. LD, ADDI). If the bottom 16 bits is "-ve", add another bit into the
+ * top half to negate the effect (i.e. 0xffff + 1 = 0x(1)0000).
+ */
+#define IMM_H(i) ((uintptr_t)(i)>>16)
+#define IMM_HA(i) (((uintptr_t)(i)>>16) + \
+ (((uintptr_t)(i) & 0x8000) >> 15))
+
+
/* opcode and xopcode for instructions */
#define OP_TRAP 3
#define OP_TRAP_64 2
@@ -195,56 +208,28 @@
#define OP_LQ 56
/* sorted alphabetically */
-#define PPC_INST_BHRBE 0x7c00025c
-#define PPC_INST_CLRBHRB 0x7c00035c
+#define PPC_INST_BCCTR_FLUSH 0x4c400420
#define PPC_INST_COPY 0x7c20060c
-#define PPC_INST_CP_ABORT 0x7c00068c
-#define PPC_INST_DARN 0x7c0005e6
#define PPC_INST_DCBA 0x7c0005ec
#define PPC_INST_DCBA_MASK 0xfc0007fe
-#define PPC_INST_DCBAL 0x7c2005ec
-#define PPC_INST_DCBZL 0x7c2007ec
-#define PPC_INST_ICBT 0x7c00002c
-#define PPC_INST_ICSWX 0x7c00032d
-#define PPC_INST_ICSWEPX 0x7c00076d
#define PPC_INST_ISEL 0x7c00001e
#define PPC_INST_ISEL_MASK 0xfc00003e
-#define PPC_INST_LDARX 0x7c0000a8
-#define PPC_INST_STDCX 0x7c0001ad
-#define PPC_INST_LQARX 0x7c000228
-#define PPC_INST_STQCX 0x7c00016d
#define PPC_INST_LSWI 0x7c0004aa
#define PPC_INST_LSWX 0x7c00042a
-#define PPC_INST_LWARX 0x7c000028
-#define PPC_INST_STWCX 0x7c00012d
#define PPC_INST_LWSYNC 0x7c2004ac
#define PPC_INST_SYNC 0x7c0004ac
#define PPC_INST_SYNC_MASK 0xfc0007fe
#define PPC_INST_ISYNC 0x4c00012c
-#define PPC_INST_LXVD2X 0x7c000698
#define PPC_INST_MCRXR 0x7c000400
#define PPC_INST_MCRXR_MASK 0xfc0007fe
#define PPC_INST_MFSPR_PVR 0x7c1f42a6
#define PPC_INST_MFSPR_PVR_MASK 0xfc1ffffe
-#define PPC_INST_MFTMR 0x7c0002dc
-#define PPC_INST_MSGSND 0x7c00019c
-#define PPC_INST_MSGCLR 0x7c0001dc
-#define PPC_INST_MSGSYNC 0x7c0006ec
-#define PPC_INST_MSGSNDP 0x7c00011c
-#define PPC_INST_MSGCLRP 0x7c00015c
#define PPC_INST_MTMSRD 0x7c000164
-#define PPC_INST_MTTMR 0x7c0003dc
#define PPC_INST_NOP 0x60000000
-#define PPC_INST_PASTE 0x7c20070d
#define PPC_INST_POPCNTB 0x7c0000f4
#define PPC_INST_POPCNTB_MASK 0xfc0007fe
-#define PPC_INST_POPCNTD 0x7c0003f4
-#define PPC_INST_POPCNTW 0x7c0002f4
#define PPC_INST_RFEBB 0x4c000124
-#define PPC_INST_RFCI 0x4c000066
-#define PPC_INST_RFDI 0x4c00004e
#define PPC_INST_RFID 0x4c000024
-#define PPC_INST_RFMCI 0x4c00004c
#define PPC_INST_MFSPR 0x7c0002a6
#define PPC_INST_MFSPR_DSCR 0x7c1102a6
#define PPC_INST_MFSPR_DSCR_MASK 0xfc1ffffe
@@ -254,131 +239,43 @@
#define PPC_INST_MFSPR_DSCR_USER_MASK 0xfc1ffffe
#define PPC_INST_MTSPR_DSCR_USER 0x7c0303a6
#define PPC_INST_MTSPR_DSCR_USER_MASK 0xfc1ffffe
-#define PPC_INST_MFVSRD 0x7c000066
-#define PPC_INST_MTVSRD 0x7c000166
#define PPC_INST_SC 0x44000002
-#define PPC_INST_SLBFEE 0x7c0007a7
-#define PPC_INST_SLBIA 0x7c0003e4
-
#define PPC_INST_STRING 0x7c00042a
#define PPC_INST_STRING_MASK 0xfc0007fe
#define PPC_INST_STRING_GEN_MASK 0xfc00067e
-
#define PPC_INST_STSWI 0x7c0005aa
#define PPC_INST_STSWX 0x7c00052a
-#define PPC_INST_STXVD2X 0x7c000798
-#define PPC_INST_TLBIE 0x7c000264
-#define PPC_INST_TLBIEL 0x7c000224
-#define PPC_INST_TLBILX 0x7c000024
-#define PPC_INST_WAIT 0x7c00007c
-#define PPC_INST_TLBIVAX 0x7c000624
-#define PPC_INST_TLBSRX_DOT 0x7c0006a5
-#define PPC_INST_VPMSUMW 0x10000488
-#define PPC_INST_VPMSUMD 0x100004c8
-#define PPC_INST_VPERMXOR 0x1000002d
-#define PPC_INST_XXLOR 0xf0000490
-#define PPC_INST_XXSWAPD 0xf0000250
-#define PPC_INST_XVCPSGNDP 0xf0000780
#define PPC_INST_TRECHKPT 0x7c0007dd
#define PPC_INST_TRECLAIM 0x7c00075d
-#define PPC_INST_TABORT 0x7c00071d
#define PPC_INST_TSR 0x7c0005dd
-
-#define PPC_INST_NAP 0x4c000364
-#define PPC_INST_SLEEP 0x4c0003a4
-#define PPC_INST_WINKLE 0x4c0003e4
-
-#define PPC_INST_STOP 0x4c0002e4
-
-/* A2 specific instructions */
-#define PPC_INST_ERATWE 0x7c0001a6
-#define PPC_INST_ERATRE 0x7c000166
-#define PPC_INST_ERATILX 0x7c000066
-#define PPC_INST_ERATIVAX 0x7c000666
-#define PPC_INST_ERATSX 0x7c000126
-#define PPC_INST_ERATSX_DOT 0x7c000127
-
-/* Misc instructions for BPF compiler */
-#define PPC_INST_LBZ 0x88000000
#define PPC_INST_LD 0xe8000000
-#define PPC_INST_LDX 0x7c00002a
-#define PPC_INST_LHZ 0xa0000000
-#define PPC_INST_LWZ 0x80000000
-#define PPC_INST_LHBRX 0x7c00062c
-#define PPC_INST_LDBRX 0x7c000428
-#define PPC_INST_STB 0x98000000
-#define PPC_INST_STH 0xb0000000
#define PPC_INST_STD 0xf8000000
-#define PPC_INST_STDX 0x7c00012a
-#define PPC_INST_STDU 0xf8000001
-#define PPC_INST_STW 0x90000000
-#define PPC_INST_STWU 0x94000000
#define PPC_INST_MFLR 0x7c0802a6
-#define PPC_INST_MTLR 0x7c0803a6
#define PPC_INST_MTCTR 0x7c0903a6
-#define PPC_INST_CMPWI 0x2c000000
-#define PPC_INST_CMPDI 0x2c200000
-#define PPC_INST_CMPW 0x7c000000
-#define PPC_INST_CMPD 0x7c200000
-#define PPC_INST_CMPLW 0x7c000040
-#define PPC_INST_CMPLD 0x7c200040
-#define PPC_INST_CMPLWI 0x28000000
-#define PPC_INST_CMPLDI 0x28200000
#define PPC_INST_ADDI 0x38000000
#define PPC_INST_ADDIS 0x3c000000
#define PPC_INST_ADD 0x7c000214
-#define PPC_INST_ADDC 0x7c000014
-#define PPC_INST_SUB 0x7c000050
#define PPC_INST_BLR 0x4e800020
-#define PPC_INST_BLRL 0x4e800021
#define PPC_INST_BCTR 0x4e800420
-#define PPC_INST_MULLD 0x7c0001d2
-#define PPC_INST_MULLW 0x7c0001d6
-#define PPC_INST_MULHWU 0x7c000016
-#define PPC_INST_MULLI 0x1c000000
-#define PPC_INST_MADDHD 0x10000030
-#define PPC_INST_MADDHDU 0x10000031
-#define PPC_INST_MADDLD 0x10000033
-#define PPC_INST_DIVWU 0x7c000396
+#define PPC_INST_BCTRL 0x4e800421
#define PPC_INST_DIVD 0x7c0003d2
-#define PPC_INST_DIVDU 0x7c000392
-#define PPC_INST_RLWINM 0x54000000
-#define PPC_INST_RLWINM_DOT 0x54000001
-#define PPC_INST_RLWIMI 0x50000000
-#define PPC_INST_RLDICL 0x78000000
#define PPC_INST_RLDICR 0x78000004
-#define PPC_INST_SLW 0x7c000030
-#define PPC_INST_SLD 0x7c000036
-#define PPC_INST_SRW 0x7c000430
-#define PPC_INST_SRAW 0x7c000630
-#define PPC_INST_SRAWI 0x7c000670
-#define PPC_INST_SRD 0x7c000436
-#define PPC_INST_SRAD 0x7c000634
-#define PPC_INST_SRADI 0x7c000674
-#define PPC_INST_AND 0x7c000038
-#define PPC_INST_ANDDOT 0x7c000039
-#define PPC_INST_OR 0x7c000378
-#define PPC_INST_XOR 0x7c000278
-#define PPC_INST_ANDI 0x70000000
#define PPC_INST_ORI 0x60000000
#define PPC_INST_ORIS 0x64000000
-#define PPC_INST_XORI 0x68000000
-#define PPC_INST_XORIS 0x6c000000
-#define PPC_INST_NEG 0x7c0000d0
-#define PPC_INST_EXTSW 0x7c0007b4
#define PPC_INST_BRANCH 0x48000000
#define PPC_INST_BRANCH_COND 0x40800000
-#define PPC_INST_LBZCIX 0x7c0006aa
-#define PPC_INST_STBCIX 0x7c0007aa
-#define PPC_INST_LWZX 0x7c00002e
-#define PPC_INST_LFSX 0x7c00042e
-#define PPC_INST_STFSX 0x7c00052e
-#define PPC_INST_LFDX 0x7c0004ae
-#define PPC_INST_STFDX 0x7c0005ae
-#define PPC_INST_LVX 0x7c0000ce
-#define PPC_INST_STVX 0x7c0001ce
-#define PPC_INST_VCMPEQUD 0x100000c7
-#define PPC_INST_VCMPEQUB 0x10000006
+
+/* Prefixes */
+#define PPC_INST_LFS 0xc0000000
+#define PPC_INST_STFS 0xd0000000
+#define PPC_INST_LFD 0xc8000000
+#define PPC_INST_STFD 0xd8000000
+#define PPC_PREFIX_MLS 0x06000000
+#define PPC_PREFIX_8LS 0x04000000
+
+/* Prefixed instructions */
+#define PPC_INST_PLD 0xe4000000
+#define PPC_INST_PSTD 0xf4000000
/* macros to insert fields into opcodes */
#define ___PPC_RA(a) (((a) & 0x1f) << 16)
@@ -411,6 +308,7 @@
#define __PPC_CT(t) (((t) & 0x0f) << 21)
#define __PPC_SPR(r) ((((r) & 0x1f) << 16) | ((((r) >> 5) & 0x1f) << 11))
#define __PPC_RC21 (0x1 << 10)
+#define __PPC_PRFX_R(r) (((r) & 0x1) << 20)
/*
* Both low and high 16 bits are added as SIGNED additions, so if low 16 bits
@@ -431,175 +329,278 @@
#define __PPC_EH(eh) 0
#endif
-/* Deal with instructions that older assemblers aren't aware of */
-#define PPC_CP_ABORT stringify_in_c(.long PPC_INST_CP_ABORT)
-#define PPC_COPY(a, b) stringify_in_c(.long PPC_INST_COPY | \
- ___PPC_RA(a) | ___PPC_RB(b))
-#define PPC_DARN(t, l) stringify_in_c(.long PPC_INST_DARN | \
- ___PPC_RT(t) | \
- (((l) & 0x3) << 16))
-#define PPC_DCBAL(a, b) stringify_in_c(.long PPC_INST_DCBAL | \
- __PPC_RA(a) | __PPC_RB(b))
-#define PPC_DCBZL(a, b) stringify_in_c(.long PPC_INST_DCBZL | \
- __PPC_RA(a) | __PPC_RB(b))
-#define PPC_LQARX(t, a, b, eh) stringify_in_c(.long PPC_INST_LQARX | \
- ___PPC_RT(t) | ___PPC_RA(a) | \
- ___PPC_RB(b) | __PPC_EH(eh))
-#define PPC_LDARX(t, a, b, eh) stringify_in_c(.long PPC_INST_LDARX | \
- ___PPC_RT(t) | ___PPC_RA(a) | \
- ___PPC_RB(b) | __PPC_EH(eh))
-#define PPC_LWARX(t, a, b, eh) stringify_in_c(.long PPC_INST_LWARX | \
- ___PPC_RT(t) | ___PPC_RA(a) | \
- ___PPC_RB(b) | __PPC_EH(eh))
-#define PPC_STQCX(t, a, b) stringify_in_c(.long PPC_INST_STQCX | \
- ___PPC_RT(t) | ___PPC_RA(a) | \
- ___PPC_RB(b))
-#define PPC_MADDHD(t, a, b, c) stringify_in_c(.long PPC_INST_MADDHD | \
- ___PPC_RT(t) | ___PPC_RA(a) | \
- ___PPC_RB(b) | ___PPC_RC(c))
-#define PPC_MADDHDU(t, a, b, c) stringify_in_c(.long PPC_INST_MADDHDU | \
- ___PPC_RT(t) | ___PPC_RA(a) | \
- ___PPC_RB(b) | ___PPC_RC(c))
-#define PPC_MADDLD(t, a, b, c) stringify_in_c(.long PPC_INST_MADDLD | \
- ___PPC_RT(t) | ___PPC_RA(a) | \
- ___PPC_RB(b) | ___PPC_RC(c))
-#define PPC_MSGSND(b) stringify_in_c(.long PPC_INST_MSGSND | \
- ___PPC_RB(b))
-#define PPC_MSGSYNC stringify_in_c(.long PPC_INST_MSGSYNC)
-#define PPC_MSGCLR(b) stringify_in_c(.long PPC_INST_MSGCLR | \
- ___PPC_RB(b))
-#define PPC_MSGSNDP(b) stringify_in_c(.long PPC_INST_MSGSNDP | \
- ___PPC_RB(b))
-#define PPC_MSGCLRP(b) stringify_in_c(.long PPC_INST_MSGCLRP | \
- ___PPC_RB(b))
-#define PPC_PASTE(a, b) stringify_in_c(.long PPC_INST_PASTE | \
- ___PPC_RA(a) | ___PPC_RB(b))
-#define PPC_POPCNTB(a, s) stringify_in_c(.long PPC_INST_POPCNTB | \
- __PPC_RA(a) | __PPC_RS(s))
-#define PPC_POPCNTD(a, s) stringify_in_c(.long PPC_INST_POPCNTD | \
- __PPC_RA(a) | __PPC_RS(s))
-#define PPC_POPCNTW(a, s) stringify_in_c(.long PPC_INST_POPCNTW | \
- __PPC_RA(a) | __PPC_RS(s))
-#define PPC_RFCI stringify_in_c(.long PPC_INST_RFCI)
-#define PPC_RFDI stringify_in_c(.long PPC_INST_RFDI)
-#define PPC_RFMCI stringify_in_c(.long PPC_INST_RFMCI)
-#define PPC_TLBILX(t, a, b) stringify_in_c(.long PPC_INST_TLBILX | \
- __PPC_T_TLB(t) | __PPC_RA0(a) | __PPC_RB(b))
-#define PPC_TLBILX_ALL(a, b) PPC_TLBILX(0, a, b)
-#define PPC_TLBILX_PID(a, b) PPC_TLBILX(1, a, b)
-#define PPC_TLBILX_VA(a, b) PPC_TLBILX(3, a, b)
-#define PPC_WAIT(w) stringify_in_c(.long PPC_INST_WAIT | \
- __PPC_WC(w))
-#define PPC_TLBIE(lp,a) stringify_in_c(.long PPC_INST_TLBIE | \
- ___PPC_RB(a) | ___PPC_RS(lp))
-#define PPC_TLBIE_5(rb,rs,ric,prs,r) \
- stringify_in_c(.long PPC_INST_TLBIE | \
- ___PPC_RB(rb) | ___PPC_RS(rs) | \
- ___PPC_RIC(ric) | ___PPC_PRS(prs) | \
- ___PPC_R(r))
-#define PPC_TLBIEL(rb,rs,ric,prs,r) \
- stringify_in_c(.long PPC_INST_TLBIEL | \
- ___PPC_RB(rb) | ___PPC_RS(rs) | \
- ___PPC_RIC(ric) | ___PPC_PRS(prs) | \
- ___PPC_R(r))
-#define PPC_TLBSRX_DOT(a,b) stringify_in_c(.long PPC_INST_TLBSRX_DOT | \
- __PPC_RA0(a) | __PPC_RB(b))
-#define PPC_TLBIVAX(a,b) stringify_in_c(.long PPC_INST_TLBIVAX | \
- __PPC_RA0(a) | __PPC_RB(b))
-
-#define PPC_ERATWE(s, a, w) stringify_in_c(.long PPC_INST_ERATWE | \
- __PPC_RS(s) | __PPC_RA(a) | __PPC_WS(w))
-#define PPC_ERATRE(s, a, w) stringify_in_c(.long PPC_INST_ERATRE | \
- __PPC_RS(s) | __PPC_RA(a) | __PPC_WS(w))
-#define PPC_ERATILX(t, a, b) stringify_in_c(.long PPC_INST_ERATILX | \
- __PPC_T_TLB(t) | __PPC_RA0(a) | \
- __PPC_RB(b))
-#define PPC_ERATIVAX(s, a, b) stringify_in_c(.long PPC_INST_ERATIVAX | \
- __PPC_RS(s) | __PPC_RA0(a) | __PPC_RB(b))
-#define PPC_ERATSX(t, a, w) stringify_in_c(.long PPC_INST_ERATSX | \
- __PPC_RS(t) | __PPC_RA0(a) | __PPC_RB(b))
-#define PPC_ERATSX_DOT(t, a, w) stringify_in_c(.long PPC_INST_ERATSX_DOT | \
- __PPC_RS(t) | __PPC_RA0(a) | __PPC_RB(b))
-#define PPC_SLBFEE_DOT(t, b) stringify_in_c(.long PPC_INST_SLBFEE | \
- __PPC_RT(t) | __PPC_RB(b))
-#define __PPC_SLBFEE_DOT(t, b) stringify_in_c(.long PPC_INST_SLBFEE | \
- ___PPC_RT(t) | ___PPC_RB(b))
-#define PPC_ICBT(c,a,b) stringify_in_c(.long PPC_INST_ICBT | \
- __PPC_CT(c) | __PPC_RA0(a) | __PPC_RB(b))
-/* PASemi instructions */
-#define LBZCIX(t,a,b) stringify_in_c(.long PPC_INST_LBZCIX | \
- __PPC_RT(t) | __PPC_RA(a) | __PPC_RB(b))
-#define STBCIX(s,a,b) stringify_in_c(.long PPC_INST_STBCIX | \
- __PPC_RS(s) | __PPC_RA(a) | __PPC_RB(b))
-
+/* Base instruction encoding */
+#define PPC_RAW_CP_ABORT (0x7c00068c)
+#define PPC_RAW_COPY(a, b) (PPC_INST_COPY | ___PPC_RA(a) | ___PPC_RB(b))
+#define PPC_RAW_DARN(t, l) (0x7c0005e6 | ___PPC_RT(t) | (((l) & 0x3) << 16))
+#define PPC_RAW_DCBAL(a, b) (0x7c2005ec | __PPC_RA(a) | __PPC_RB(b))
+#define PPC_RAW_DCBZL(a, b) (0x7c2007ec | __PPC_RA(a) | __PPC_RB(b))
+#define PPC_RAW_LQARX(t, a, b, eh) (0x7c000228 | ___PPC_RT(t) | ___PPC_RA(a) | ___PPC_RB(b) | __PPC_EH(eh))
+#define PPC_RAW_LDARX(t, a, b, eh) (0x7c0000a8 | ___PPC_RT(t) | ___PPC_RA(a) | ___PPC_RB(b) | __PPC_EH(eh))
+#define PPC_RAW_LWARX(t, a, b, eh) (0x7c000028 | ___PPC_RT(t) | ___PPC_RA(a) | ___PPC_RB(b) | __PPC_EH(eh))
+#define PPC_RAW_PHWSYNC (0x7c8004ac)
+#define PPC_RAW_PLWSYNC (0x7ca004ac)
+#define PPC_RAW_STQCX(t, a, b) (0x7c00016d | ___PPC_RT(t) | ___PPC_RA(a) | ___PPC_RB(b))
+#define PPC_RAW_MADDHD(t, a, b, c) (0x10000030 | ___PPC_RT(t) | ___PPC_RA(a) | ___PPC_RB(b) | ___PPC_RC(c))
+#define PPC_RAW_MADDHDU(t, a, b, c) (0x10000031 | ___PPC_RT(t) | ___PPC_RA(a) | ___PPC_RB(b) | ___PPC_RC(c))
+#define PPC_RAW_MADDLD(t, a, b, c) (0x10000033 | ___PPC_RT(t) | ___PPC_RA(a) | ___PPC_RB(b) | ___PPC_RC(c))
+#define PPC_RAW_MSGSND(b) (0x7c00019c | ___PPC_RB(b))
+#define PPC_RAW_MSGSYNC (0x7c0006ec)
+#define PPC_RAW_MSGCLR(b) (0x7c0001dc | ___PPC_RB(b))
+#define PPC_RAW_MSGSNDP(b) (0x7c00011c | ___PPC_RB(b))
+#define PPC_RAW_MSGCLRP(b) (0x7c00015c | ___PPC_RB(b))
+#define PPC_RAW_PASTE(a, b) (0x7c20070d | ___PPC_RA(a) | ___PPC_RB(b))
+#define PPC_RAW_POPCNTB(a, s) (PPC_INST_POPCNTB | __PPC_RA(a) | __PPC_RS(s))
+#define PPC_RAW_POPCNTD(a, s) (0x7c0003f4 | __PPC_RA(a) | __PPC_RS(s))
+#define PPC_RAW_POPCNTW(a, s) (0x7c0002f4 | __PPC_RA(a) | __PPC_RS(s))
+#define PPC_RAW_RFCI (0x4c000066)
+#define PPC_RAW_RFDI (0x4c00004e)
+#define PPC_RAW_RFMCI (0x4c00004c)
+#define PPC_RAW_TLBILX(t, a, b) (0x7c000024 | __PPC_T_TLB(t) | __PPC_RA0(a) | __PPC_RB(b))
+#define PPC_RAW_WAIT(w) (0x7c00007c | __PPC_WC(w))
+#define PPC_RAW_TLBIE(lp, a) (0x7c000264 | ___PPC_RB(a) | ___PPC_RS(lp))
+#define PPC_RAW_TLBIE_5(rb, rs, ric, prs, r) \
+ (0x7c000264 | ___PPC_RB(rb) | ___PPC_RS(rs) | ___PPC_RIC(ric) | ___PPC_PRS(prs) | ___PPC_R(r))
+#define PPC_RAW_TLBIEL(rb, rs, ric, prs, r) \
+ (0x7c000224 | ___PPC_RB(rb) | ___PPC_RS(rs) | ___PPC_RIC(ric) | ___PPC_PRS(prs) | ___PPC_R(r))
+#define PPC_RAW_TLBSRX_DOT(a, b) (0x7c0006a5 | __PPC_RA0(a) | __PPC_RB(b))
+#define PPC_RAW_TLBIVAX(a, b) (0x7c000624 | __PPC_RA0(a) | __PPC_RB(b))
+#define PPC_RAW_ERATWE(s, a, w) (0x7c0001a6 | __PPC_RS(s) | __PPC_RA(a) | __PPC_WS(w))
+#define PPC_RAW_ERATRE(s, a, w) (0x7c000166 | __PPC_RS(s) | __PPC_RA(a) | __PPC_WS(w))
+#define PPC_RAW_ERATILX(t, a, b) (0x7c000066 | __PPC_T_TLB(t) | __PPC_RA0(a) | __PPC_RB(b))
+#define PPC_RAW_ERATIVAX(s, a, b) (0x7c000666 | __PPC_RS(s) | __PPC_RA0(a) | __PPC_RB(b))
+#define PPC_RAW_ERATSX(t, a, w) (0x7c000126 | __PPC_RS(t) | __PPC_RA0(a) | __PPC_RB(b))
+#define PPC_RAW_ERATSX_DOT(t, a, w) (0x7c000127 | __PPC_RS(t) | __PPC_RA0(a) | __PPC_RB(b))
+#define PPC_RAW_SLBFEE_DOT(t, b) (0x7c0007a7 | __PPC_RT(t) | __PPC_RB(b))
+#define __PPC_RAW_SLBFEE_DOT(t, b) (0x7c0007a7 | ___PPC_RT(t) | ___PPC_RB(b))
+#define PPC_RAW_ICBT(c, a, b) (0x7c00002c | __PPC_CT(c) | __PPC_RA0(a) | __PPC_RB(b))
+#define PPC_RAW_LBZCIX(t, a, b) (0x7c0006aa | __PPC_RT(t) | __PPC_RA(a) | __PPC_RB(b))
+#define PPC_RAW_STBCIX(s, a, b) (0x7c0007aa | __PPC_RS(s) | __PPC_RA(a) | __PPC_RB(b))
+#define PPC_RAW_DCBFPS(a, b) (0x7c0000ac | ___PPC_RA(a) | ___PPC_RB(b) | (4 << 21))
+#define PPC_RAW_DCBSTPS(a, b) (0x7c0000ac | ___PPC_RA(a) | ___PPC_RB(b) | (6 << 21))
/*
* Define what the VSX XX1 form instructions will look like, then add
* the 128 bit load store instructions based on that.
*/
-#define VSX_XX1(s, a, b) (__PPC_XS(s) | __PPC_RA(a) | __PPC_RB(b))
-#define VSX_XX3(t, a, b) (__PPC_XT(t) | __PPC_XA(a) | __PPC_XB(b))
-#define STXVD2X(s, a, b) stringify_in_c(.long PPC_INST_STXVD2X | \
- VSX_XX1((s), a, b))
-#define LXVD2X(s, a, b) stringify_in_c(.long PPC_INST_LXVD2X | \
- VSX_XX1((s), a, b))
-#define MFVRD(a, t) stringify_in_c(.long PPC_INST_MFVSRD | \
- VSX_XX1((t)+32, a, R0))
-#define MTVRD(t, a) stringify_in_c(.long PPC_INST_MTVSRD | \
- VSX_XX1((t)+32, a, R0))
-#define VPMSUMW(t, a, b) stringify_in_c(.long PPC_INST_VPMSUMW | \
- VSX_XX3((t), a, b))
-#define VPMSUMD(t, a, b) stringify_in_c(.long PPC_INST_VPMSUMD | \
- VSX_XX3((t), a, b))
-#define XXLOR(t, a, b) stringify_in_c(.long PPC_INST_XXLOR | \
- VSX_XX3((t), a, b))
-#define XXSWAPD(t, a) stringify_in_c(.long PPC_INST_XXSWAPD | \
- VSX_XX3((t), a, a))
-#define XVCPSGNDP(t, a, b) stringify_in_c(.long (PPC_INST_XVCPSGNDP | \
- VSX_XX3((t), (a), (b))))
+#define VSX_XX1(s, a, b) (__PPC_XS(s) | __PPC_RA(a) | __PPC_RB(b))
+#define VSX_XX3(t, a, b) (__PPC_XT(t) | __PPC_XA(a) | __PPC_XB(b))
+#define PPC_RAW_STXVD2X(s, a, b) (0x7c000798 | VSX_XX1((s), a, b))
+#define PPC_RAW_LXVD2X(s, a, b) (0x7c000698 | VSX_XX1((s), a, b))
+#define PPC_RAW_MFVRD(a, t) (0x7c000066 | VSX_XX1((t) + 32, a, R0))
+#define PPC_RAW_MTVRD(t, a) (0x7c000166 | VSX_XX1((t) + 32, a, R0))
+#define PPC_RAW_VPMSUMW(t, a, b) (0x10000488 | VSX_XX3((t), a, b))
+#define PPC_RAW_VPMSUMD(t, a, b) (0x100004c8 | VSX_XX3((t), a, b))
+#define PPC_RAW_XXLOR(t, a, b) (0xf0000490 | VSX_XX3((t), a, b))
+#define PPC_RAW_XXSWAPD(t, a) (0xf0000250 | VSX_XX3((t), a, a))
+#define PPC_RAW_XVCPSGNDP(t, a, b) ((0xf0000780 | VSX_XX3((t), (a), (b))))
+#define PPC_RAW_VPERMXOR(vrt, vra, vrb, vrc) \
+ ((0x1000002d | ___PPC_RT(vrt) | ___PPC_RA(vra) | ___PPC_RB(vrb) | (((vrc) & 0x1f) << 6)))
+#define PPC_RAW_NAP (0x4c000364)
+#define PPC_RAW_SLEEP (0x4c0003a4)
+#define PPC_RAW_WINKLE (0x4c0003e4)
+#define PPC_RAW_STOP (0x4c0002e4)
+#define PPC_RAW_CLRBHRB (0x7c00035c)
+#define PPC_RAW_MFBHRBE(r, n) (0x7c00025c | __PPC_RT(r) | (((n) & 0x3ff) << 11))
+#define PPC_RAW_TRECHKPT (PPC_INST_TRECHKPT)
+#define PPC_RAW_TRECLAIM(r) (PPC_INST_TRECLAIM | __PPC_RA(r))
+#define PPC_RAW_TABORT(r) (0x7c00071d | __PPC_RA(r))
+#define TMRN(x) ((((x) & 0x1f) << 16) | (((x) & 0x3e0) << 6))
+#define PPC_RAW_MTTMR(tmr, r) (0x7c0003dc | TMRN(tmr) | ___PPC_RS(r))
+#define PPC_RAW_MFTMR(tmr, r) (0x7c0002dc | TMRN(tmr) | ___PPC_RT(r))
+#define PPC_RAW_ICSWX(s, a, b) (0x7c00032d | ___PPC_RS(s) | ___PPC_RA(a) | ___PPC_RB(b))
+#define PPC_RAW_ICSWEPX(s, a, b) (0x7c00076d | ___PPC_RS(s) | ___PPC_RA(a) | ___PPC_RB(b))
+#define PPC_RAW_SLBIA(IH) (0x7c0003e4 | (((IH) & 0x7) << 21))
+#define PPC_RAW_VCMPEQUD_RC(vrt, vra, vrb) \
+ (0x100000c7 | ___PPC_RT(vrt) | ___PPC_RA(vra) | ___PPC_RB(vrb) | __PPC_RC21)
+#define PPC_RAW_VCMPEQUB_RC(vrt, vra, vrb) \
+ (0x10000006 | ___PPC_RT(vrt) | ___PPC_RA(vra) | ___PPC_RB(vrb) | __PPC_RC21)
+#define PPC_RAW_LD(r, base, i) (PPC_INST_LD | ___PPC_RT(r) | ___PPC_RA(base) | IMM_DS(i))
+#define PPC_RAW_LWZ(r, base, i) (0x80000000 | ___PPC_RT(r) | ___PPC_RA(base) | IMM_L(i))
+#define PPC_RAW_LWZX(t, a, b) (0x7c00002e | ___PPC_RT(t) | ___PPC_RA(a) | ___PPC_RB(b))
+#define PPC_RAW_STD(r, base, i) (PPC_INST_STD | ___PPC_RS(r) | ___PPC_RA(base) | IMM_DS(i))
+#define PPC_RAW_STDCX(s, a, b) (0x7c0001ad | ___PPC_RS(s) | ___PPC_RA(a) | ___PPC_RB(b))
+#define PPC_RAW_LFSX(t, a, b) (0x7c00042e | ___PPC_RT(t) | ___PPC_RA(a) | ___PPC_RB(b))
+#define PPC_RAW_STFSX(s, a, b) (0x7c00052e | ___PPC_RS(s) | ___PPC_RA(a) | ___PPC_RB(b))
+#define PPC_RAW_LFDX(t, a, b) (0x7c0004ae | ___PPC_RT(t) | ___PPC_RA(a) | ___PPC_RB(b))
+#define PPC_RAW_STFDX(s, a, b) (0x7c0005ae | ___PPC_RS(s) | ___PPC_RA(a) | ___PPC_RB(b))
+#define PPC_RAW_LVX(t, a, b) (0x7c0000ce | ___PPC_RT(t) | ___PPC_RA(a) | ___PPC_RB(b))
+#define PPC_RAW_STVX(s, a, b) (0x7c0001ce | ___PPC_RS(s) | ___PPC_RA(a) | ___PPC_RB(b))
+#define PPC_RAW_ADD(t, a, b) (PPC_INST_ADD | ___PPC_RT(t) | ___PPC_RA(a) | ___PPC_RB(b))
+#define PPC_RAW_ADD_DOT(t, a, b) (PPC_INST_ADD | ___PPC_RT(t) | ___PPC_RA(a) | ___PPC_RB(b) | 0x1)
+#define PPC_RAW_ADDC(t, a, b) (0x7c000014 | ___PPC_RT(t) | ___PPC_RA(a) | ___PPC_RB(b))
+#define PPC_RAW_ADDC_DOT(t, a, b) (0x7c000014 | ___PPC_RT(t) | ___PPC_RA(a) | ___PPC_RB(b) | 0x1)
+#define PPC_RAW_NOP() (PPC_INST_NOP)
+#define PPC_RAW_BLR() (PPC_INST_BLR)
+#define PPC_RAW_BLRL() (0x4e800021)
+#define PPC_RAW_MTLR(r) (0x7c0803a6 | ___PPC_RT(r))
+#define PPC_RAW_BCTR() (PPC_INST_BCTR)
+#define PPC_RAW_MTCTR(r) (PPC_INST_MTCTR | ___PPC_RT(r))
+#define PPC_RAW_ADDI(d, a, i) (PPC_INST_ADDI | ___PPC_RT(d) | ___PPC_RA(a) | IMM_L(i))
+#define PPC_RAW_LI(r, i) PPC_RAW_ADDI(r, 0, i)
+#define PPC_RAW_ADDIS(d, a, i) (PPC_INST_ADDIS | ___PPC_RT(d) | ___PPC_RA(a) | IMM_L(i))
+#define PPC_RAW_LIS(r, i) PPC_RAW_ADDIS(r, 0, i)
+#define PPC_RAW_STDX(r, base, b) (0x7c00012a | ___PPC_RS(r) | ___PPC_RA(base) | ___PPC_RB(b))
+#define PPC_RAW_STDU(r, base, i) (0xf8000001 | ___PPC_RS(r) | ___PPC_RA(base) | ((i) & 0xfffc))
+#define PPC_RAW_STW(r, base, i) (0x90000000 | ___PPC_RS(r) | ___PPC_RA(base) | IMM_L(i))
+#define PPC_RAW_STWU(r, base, i) (0x94000000 | ___PPC_RS(r) | ___PPC_RA(base) | IMM_L(i))
+#define PPC_RAW_STH(r, base, i) (0xb0000000 | ___PPC_RS(r) | ___PPC_RA(base) | IMM_L(i))
+#define PPC_RAW_STB(r, base, i) (0x98000000 | ___PPC_RS(r) | ___PPC_RA(base) | IMM_L(i))
+#define PPC_RAW_LBZ(r, base, i) (0x88000000 | ___PPC_RT(r) | ___PPC_RA(base) | IMM_L(i))
+#define PPC_RAW_LDX(r, base, b) (0x7c00002a | ___PPC_RT(r) | ___PPC_RA(base) | ___PPC_RB(b))
+#define PPC_RAW_LHZ(r, base, i) (0xa0000000 | ___PPC_RT(r) | ___PPC_RA(base) | IMM_L(i))
+#define PPC_RAW_LHBRX(r, base, b) (0x7c00062c | ___PPC_RT(r) | ___PPC_RA(base) | ___PPC_RB(b))
+#define PPC_RAW_LDBRX(r, base, b) (0x7c000428 | ___PPC_RT(r) | ___PPC_RA(base) | ___PPC_RB(b))
+#define PPC_RAW_STWCX(s, a, b) (0x7c00012d | ___PPC_RS(s) | ___PPC_RA(a) | ___PPC_RB(b))
+#define PPC_RAW_CMPWI(a, i) (0x2c000000 | ___PPC_RA(a) | IMM_L(i))
+#define PPC_RAW_CMPDI(a, i) (0x2c200000 | ___PPC_RA(a) | IMM_L(i))
+#define PPC_RAW_CMPW(a, b) (0x7c000000 | ___PPC_RA(a) | ___PPC_RB(b))
+#define PPC_RAW_CMPD(a, b) (0x7c200000 | ___PPC_RA(a) | ___PPC_RB(b))
+#define PPC_RAW_CMPLWI(a, i) (0x28000000 | ___PPC_RA(a) | IMM_L(i))
+#define PPC_RAW_CMPLDI(a, i) (0x28200000 | ___PPC_RA(a) | IMM_L(i))
+#define PPC_RAW_CMPLW(a, b) (0x7c000040 | ___PPC_RA(a) | ___PPC_RB(b))
+#define PPC_RAW_CMPLD(a, b) (0x7c200040 | ___PPC_RA(a) | ___PPC_RB(b))
+#define PPC_RAW_SUB(d, a, b) (0x7c000050 | ___PPC_RT(d) | ___PPC_RB(a) | ___PPC_RA(b))
+#define PPC_RAW_MULD(d, a, b) (0x7c0001d2 | ___PPC_RT(d) | ___PPC_RA(a) | ___PPC_RB(b))
+#define PPC_RAW_MULW(d, a, b) (0x7c0001d6 | ___PPC_RT(d) | ___PPC_RA(a) | ___PPC_RB(b))
+#define PPC_RAW_MULHWU(d, a, b) (0x7c000016 | ___PPC_RT(d) | ___PPC_RA(a) | ___PPC_RB(b))
+#define PPC_RAW_MULI(d, a, i) (0x1c000000 | ___PPC_RT(d) | ___PPC_RA(a) | IMM_L(i))
+#define PPC_RAW_DIVWU(d, a, b) (0x7c000396 | ___PPC_RT(d) | ___PPC_RA(a) | ___PPC_RB(b))
+#define PPC_RAW_DIVDU(d, a, b) (0x7c000392 | ___PPC_RT(d) | ___PPC_RA(a) | ___PPC_RB(b))
+#define PPC_RAW_DIVDE(t, a, b) (0x7c000352 | ___PPC_RT(t) | ___PPC_RA(a) | ___PPC_RB(b))
+#define PPC_RAW_DIVDE_DOT(t, a, b) (0x7c000352 | ___PPC_RT(t) | ___PPC_RA(a) | ___PPC_RB(b) | 0x1)
+#define PPC_RAW_DIVDEU(t, a, b) (0x7c000312 | ___PPC_RT(t) | ___PPC_RA(a) | ___PPC_RB(b))
+#define PPC_RAW_DIVDEU_DOT(t, a, b) (0x7c000312 | ___PPC_RT(t) | ___PPC_RA(a) | ___PPC_RB(b) | 0x1)
+#define PPC_RAW_AND(d, a, b) (0x7c000038 | ___PPC_RA(d) | ___PPC_RS(a) | ___PPC_RB(b))
+#define PPC_RAW_ANDI(d, a, i) (0x70000000 | ___PPC_RA(d) | ___PPC_RS(a) | IMM_L(i))
+#define PPC_RAW_AND_DOT(d, a, b) (0x7c000039 | ___PPC_RA(d) | ___PPC_RS(a) | ___PPC_RB(b))
+#define PPC_RAW_OR(d, a, b) (0x7c000378 | ___PPC_RA(d) | ___PPC_RS(a) | ___PPC_RB(b))
+#define PPC_RAW_MR(d, a) PPC_RAW_OR(d, a, a)
+#define PPC_RAW_ORI(d, a, i) (PPC_INST_ORI | ___PPC_RA(d) | ___PPC_RS(a) | IMM_L(i))
+#define PPC_RAW_ORIS(d, a, i) (PPC_INST_ORIS | ___PPC_RA(d) | ___PPC_RS(a) | IMM_L(i))
+#define PPC_RAW_XOR(d, a, b) (0x7c000278 | ___PPC_RA(d) | ___PPC_RS(a) | ___PPC_RB(b))
+#define PPC_RAW_XORI(d, a, i) (0x68000000 | ___PPC_RA(d) | ___PPC_RS(a) | IMM_L(i))
+#define PPC_RAW_XORIS(d, a, i) (0x6c000000 | ___PPC_RA(d) | ___PPC_RS(a) | IMM_L(i))
+#define PPC_RAW_EXTSW(d, a) (0x7c0007b4 | ___PPC_RA(d) | ___PPC_RS(a))
+#define PPC_RAW_SLW(d, a, s) (0x7c000030 | ___PPC_RA(d) | ___PPC_RS(a) | ___PPC_RB(s))
+#define PPC_RAW_SLD(d, a, s) (0x7c000036 | ___PPC_RA(d) | ___PPC_RS(a) | ___PPC_RB(s))
+#define PPC_RAW_SRW(d, a, s) (0x7c000430 | ___PPC_RA(d) | ___PPC_RS(a) | ___PPC_RB(s))
+#define PPC_RAW_SRAW(d, a, s) (0x7c000630 | ___PPC_RA(d) | ___PPC_RS(a) | ___PPC_RB(s))
+#define PPC_RAW_SRAWI(d, a, i) (0x7c000670 | ___PPC_RA(d) | ___PPC_RS(a) | __PPC_SH(i))
+#define PPC_RAW_SRD(d, a, s) (0x7c000436 | ___PPC_RA(d) | ___PPC_RS(a) | ___PPC_RB(s))
+#define PPC_RAW_SRAD(d, a, s) (0x7c000634 | ___PPC_RA(d) | ___PPC_RS(a) | ___PPC_RB(s))
+#define PPC_RAW_SRADI(d, a, i) (0x7c000674 | ___PPC_RA(d) | ___PPC_RS(a) | __PPC_SH64(i))
+#define PPC_RAW_RLWINM(d, a, i, mb, me) (0x54000000 | ___PPC_RA(d) | ___PPC_RS(a) | __PPC_SH(i) | __PPC_MB(mb) | __PPC_ME(me))
+#define PPC_RAW_RLWINM_DOT(d, a, i, mb, me) \
+ (0x54000001 | ___PPC_RA(d) | ___PPC_RS(a) | __PPC_SH(i) | __PPC_MB(mb) | __PPC_ME(me))
+#define PPC_RAW_RLWIMI(d, a, i, mb, me) (0x50000000 | ___PPC_RA(d) | ___PPC_RS(a) | __PPC_SH(i) | __PPC_MB(mb) | __PPC_ME(me))
+#define PPC_RAW_RLDICL(d, a, i, mb) (0x78000000 | ___PPC_RA(d) | ___PPC_RS(a) | __PPC_SH64(i) | __PPC_MB64(mb))
+#define PPC_RAW_RLDICR(d, a, i, me) (PPC_INST_RLDICR | ___PPC_RA(d) | ___PPC_RS(a) | __PPC_SH64(i) | __PPC_ME64(me))
+
+/* slwi = rlwinm Rx, Ry, n, 0, 31-n */
+#define PPC_RAW_SLWI(d, a, i) PPC_RAW_RLWINM(d, a, i, 0, 31-(i))
+/* srwi = rlwinm Rx, Ry, 32-n, n, 31 */
+#define PPC_RAW_SRWI(d, a, i) PPC_RAW_RLWINM(d, a, 32-(i), i, 31)
+/* sldi = rldicr Rx, Ry, n, 63-n */
+#define PPC_RAW_SLDI(d, a, i) PPC_RAW_RLDICR(d, a, i, 63-(i))
+/* sldi = rldicl Rx, Ry, 64-n, n */
+#define PPC_RAW_SRDI(d, a, i) PPC_RAW_RLDICL(d, a, 64-(i), i)
+
+#define PPC_RAW_NEG(d, a) (0x7c0000d0 | ___PPC_RT(d) | ___PPC_RA(a))
+
+/* Deal with instructions that older assemblers aren't aware of */
+#define PPC_BCCTR_FLUSH stringify_in_c(.long PPC_INST_BCCTR_FLUSH)
+#define PPC_CP_ABORT stringify_in_c(.long PPC_RAW_CP_ABORT)
+#define PPC_COPY(a, b) stringify_in_c(.long PPC_RAW_COPY(a, b))
+#define PPC_DARN(t, l) stringify_in_c(.long PPC_RAW_DARN(t, l))
+#define PPC_DCBAL(a, b) stringify_in_c(.long PPC_RAW_DCBAL(a, b))
+#define PPC_DCBZL(a, b) stringify_in_c(.long PPC_RAW_DCBZL(a, b))
+#define PPC_DIVDE(t, a, b) stringify_in_c(.long PPC_RAW_DIVDE(t, a, b))
+#define PPC_DIVDEU(t, a, b) stringify_in_c(.long PPC_RAW_DIVDEU(t, a, b))
+#define PPC_LQARX(t, a, b, eh) stringify_in_c(.long PPC_RAW_LQARX(t, a, b, eh))
+#define PPC_LDARX(t, a, b, eh) stringify_in_c(.long PPC_RAW_LDARX(t, a, b, eh))
+#define PPC_LWARX(t, a, b, eh) stringify_in_c(.long PPC_RAW_LWARX(t, a, b, eh))
+#define PPC_STQCX(t, a, b) stringify_in_c(.long PPC_RAW_STQCX(t, a, b))
+#define PPC_MADDHD(t, a, b, c) stringify_in_c(.long PPC_RAW_MADDHD(t, a, b, c))
+#define PPC_MADDHDU(t, a, b, c) stringify_in_c(.long PPC_RAW_MADDHDU(t, a, b, c))
+#define PPC_MADDLD(t, a, b, c) stringify_in_c(.long PPC_RAW_MADDLD(t, a, b, c))
+#define PPC_MSGSND(b) stringify_in_c(.long PPC_RAW_MSGSND(b))
+#define PPC_MSGSYNC stringify_in_c(.long PPC_RAW_MSGSYNC)
+#define PPC_MSGCLR(b) stringify_in_c(.long PPC_RAW_MSGCLR(b))
+#define PPC_MSGSNDP(b) stringify_in_c(.long PPC_RAW_MSGSNDP(b))
+#define PPC_MSGCLRP(b) stringify_in_c(.long PPC_RAW_MSGCLRP(b))
+#define PPC_PASTE(a, b) stringify_in_c(.long PPC_RAW_PASTE(a, b))
+#define PPC_POPCNTB(a, s) stringify_in_c(.long PPC_RAW_POPCNTB(a, s))
+#define PPC_POPCNTD(a, s) stringify_in_c(.long PPC_RAW_POPCNTD(a, s))
+#define PPC_POPCNTW(a, s) stringify_in_c(.long PPC_RAW_POPCNTW(a, s))
+#define PPC_RFCI stringify_in_c(.long PPC_RAW_RFCI)
+#define PPC_RFDI stringify_in_c(.long PPC_RAW_RFDI)
+#define PPC_RFMCI stringify_in_c(.long PPC_RAW_RFMCI)
+#define PPC_TLBILX(t, a, b) stringify_in_c(.long PPC_RAW_TLBILX(t, a, b))
+#define PPC_TLBILX_ALL(a, b) PPC_TLBILX(0, a, b)
+#define PPC_TLBILX_PID(a, b) PPC_TLBILX(1, a, b)
+#define PPC_TLBILX_VA(a, b) PPC_TLBILX(3, a, b)
+#define PPC_WAIT(w) stringify_in_c(.long PPC_RAW_WAIT(w))
+#define PPC_TLBIE(lp, a) stringify_in_c(.long PPC_RAW_TLBIE(lp, a))
+#define PPC_TLBIE_5(rb, rs, ric, prs, r) \
+ stringify_in_c(.long PPC_RAW_TLBIE_5(rb, rs, ric, prs, r))
+#define PPC_TLBIEL(rb,rs,ric,prs,r) \
+ stringify_in_c(.long PPC_RAW_TLBIEL(rb, rs, ric, prs, r))
+#define PPC_TLBSRX_DOT(a, b) stringify_in_c(.long PPC_RAW_TLBSRX_DOT(a, b))
+#define PPC_TLBIVAX(a, b) stringify_in_c(.long PPC_RAW_TLBIVAX(a, b))
+
+#define PPC_ERATWE(s, a, w) stringify_in_c(.long PPC_RAW_ERATWE(s, a, w))
+#define PPC_ERATRE(s, a, w) stringify_in_c(.long PPC_RAW_ERATRE(a, a, w))
+#define PPC_ERATILX(t, a, b) stringify_in_c(.long PPC_RAW_ERATILX(t, a, b))
+#define PPC_ERATIVAX(s, a, b) stringify_in_c(.long PPC_RAW_ERATIVAX(s, a, b))
+#define PPC_ERATSX(t, a, w) stringify_in_c(.long PPC_RAW_ERATSX(t, a, w))
+#define PPC_ERATSX_DOT(t, a, w) stringify_in_c(.long PPC_RAW_ERATSX_DOT(t, a, w))
+#define PPC_SLBFEE_DOT(t, b) stringify_in_c(.long PPC_RAW_SLBFEE_DOT(t, b))
+#define __PPC_SLBFEE_DOT(t, b) stringify_in_c(.long __PPC_RAW_SLBFEE_DOT(t, b))
+#define PPC_ICBT(c, a, b) stringify_in_c(.long PPC_RAW_ICBT(c, a, b))
+/* PASemi instructions */
+#define LBZCIX(t, a, b) stringify_in_c(.long PPC_RAW_LBZCIX(t, a, b))
+#define STBCIX(s, a, b) stringify_in_c(.long PPC_RAW_STBCIX(s, a, b))
+#define PPC_DCBFPS(a, b) stringify_in_c(.long PPC_RAW_DCBFPS(a, b))
+#define PPC_DCBSTPS(a, b) stringify_in_c(.long PPC_RAW_DCBSTPS(a, b))
+#define PPC_PHWSYNC stringify_in_c(.long PPC_RAW_PHWSYNC)
+#define PPC_PLWSYNC stringify_in_c(.long PPC_RAW_PLWSYNC)
+#define STXVD2X(s, a, b) stringify_in_c(.long PPC_RAW_STXVD2X(s, a, b))
+#define LXVD2X(s, a, b) stringify_in_c(.long PPC_RAW_LXVD2X(s, a, b))
+#define MFVRD(a, t) stringify_in_c(.long PPC_RAW_MFVRD(a, t))
+#define MTVRD(t, a) stringify_in_c(.long PPC_RAW_MTVRD(t, a))
+#define VPMSUMW(t, a, b) stringify_in_c(.long PPC_RAW_VPMSUMW(t, a, b))
+#define VPMSUMD(t, a, b) stringify_in_c(.long PPC_RAW_VPMSUMD(t, a, b))
+#define XXLOR(t, a, b) stringify_in_c(.long PPC_RAW_XXLOR(t, a, b))
+#define XXSWAPD(t, a) stringify_in_c(.long PPC_RAW_XXSWAPD(t, a))
+#define XVCPSGNDP(t, a, b) stringify_in_c(.long (PPC_RAW_XVCPSGNDP(t, a, b)))
#define VPERMXOR(vrt, vra, vrb, vrc) \
- stringify_in_c(.long (PPC_INST_VPERMXOR | \
- ___PPC_RT(vrt) | ___PPC_RA(vra) | \
- ___PPC_RB(vrb) | (((vrc) & 0x1f) << 6)))
+ stringify_in_c(.long (PPC_RAW_VPERMXOR(vrt, vra, vrb, vrc)))
-#define PPC_NAP stringify_in_c(.long PPC_INST_NAP)
-#define PPC_SLEEP stringify_in_c(.long PPC_INST_SLEEP)
-#define PPC_WINKLE stringify_in_c(.long PPC_INST_WINKLE)
+#define PPC_NAP stringify_in_c(.long PPC_RAW_NAP)
+#define PPC_SLEEP stringify_in_c(.long PPC_RAW_SLEEP)
+#define PPC_WINKLE stringify_in_c(.long PPC_RAW_WINKLE)
-#define PPC_STOP stringify_in_c(.long PPC_INST_STOP)
+#define PPC_STOP stringify_in_c(.long PPC_RAW_STOP)
/* BHRB instructions */
-#define PPC_CLRBHRB stringify_in_c(.long PPC_INST_CLRBHRB)
-#define PPC_MFBHRBE(r, n) stringify_in_c(.long PPC_INST_BHRBE | \
- __PPC_RT(r) | \
- (((n) & 0x3ff) << 11))
+#define PPC_CLRBHRB stringify_in_c(.long PPC_RAW_CLRBHRB)
+#define PPC_MFBHRBE(r, n) stringify_in_c(.long PPC_RAW_MFBHRBE(r, n))
/* Transactional memory instructions */
-#define TRECHKPT stringify_in_c(.long PPC_INST_TRECHKPT)
-#define TRECLAIM(r) stringify_in_c(.long PPC_INST_TRECLAIM \
- | __PPC_RA(r))
-#define TABORT(r) stringify_in_c(.long PPC_INST_TABORT \
- | __PPC_RA(r))
+#define TRECHKPT stringify_in_c(.long PPC_RAW_TRECHKPT)
+#define TRECLAIM(r) stringify_in_c(.long PPC_RAW_TRECLAIM(r))
+#define TABORT(r) stringify_in_c(.long PPC_RAW_TABORT(r))
/* book3e thread control instructions */
-#define TMRN(x) ((((x) & 0x1f) << 16) | (((x) & 0x3e0) << 6))
-#define MTTMR(tmr, r) stringify_in_c(.long PPC_INST_MTTMR | \
- TMRN(tmr) | ___PPC_RS(r))
-#define MFTMR(tmr, r) stringify_in_c(.long PPC_INST_MFTMR | \
- TMRN(tmr) | ___PPC_RT(r))
+#define MTTMR(tmr, r) stringify_in_c(.long PPC_RAW_MTTMR(tmr, r))
+#define MFTMR(tmr, r) stringify_in_c(.long PPC_RAW_MFTMR(tmr, r))
/* Coprocessor instructions */
-#define PPC_ICSWX(s, a, b) stringify_in_c(.long PPC_INST_ICSWX | \
- ___PPC_RS(s) | \
- ___PPC_RA(a) | \
- ___PPC_RB(b))
-#define PPC_ICSWEPX(s, a, b) stringify_in_c(.long PPC_INST_ICSWEPX | \
- ___PPC_RS(s) | \
- ___PPC_RA(a) | \
- ___PPC_RB(b))
+#define PPC_ICSWX(s, a, b) stringify_in_c(.long PPC_RAW_ICSWX(s, a, b))
+#define PPC_ICSWEPX(s, a, b) stringify_in_c(.long PPC_RAW_ICSWEPX(s, a, b))
-#define PPC_SLBIA(IH) stringify_in_c(.long PPC_INST_SLBIA | \
- ((IH & 0x7) << 21))
+#define PPC_SLBIA(IH) stringify_in_c(.long PPC_RAW_SLBIA(IH))
/*
* These may only be used on ISA v3.0 or later (aka. CPU_FTR_ARCH_300, radix
@@ -611,12 +612,8 @@
#define PPC_RADIX_INVALIDATE_ERAT_USER PPC_SLBIA(3)
#define PPC_RADIX_INVALIDATE_ERAT_GUEST PPC_SLBIA(6)
-#define VCMPEQUD_RC(vrt, vra, vrb) stringify_in_c(.long PPC_INST_VCMPEQUD | \
- ___PPC_RT(vrt) | ___PPC_RA(vra) | \
- ___PPC_RB(vrb) | __PPC_RC21)
+#define VCMPEQUD_RC(vrt, vra, vrb) stringify_in_c(.long PPC_RAW_VCMPEQUD_RC(vrt, vra, vrb))
-#define VCMPEQUB_RC(vrt, vra, vrb) stringify_in_c(.long PPC_INST_VCMPEQUB | \
- ___PPC_RT(vrt) | ___PPC_RA(vra) | \
- ___PPC_RB(vrb) | __PPC_RC21)
+#define VCMPEQUB_RC(vrt, vra, vrb) stringify_in_c(.long PPC_RAW_VCMPEQUB_RC(vrt, vra, vrb))
#endif /* _ASM_POWERPC_PPC_OPCODE_H */
diff --git a/arch/powerpc/include/asm/ppc_asm.h b/arch/powerpc/include/asm/ppc_asm.h
index 6b03dff..b4cc660 100644
--- a/arch/powerpc/include/asm/ppc_asm.h
+++ b/arch/powerpc/include/asm/ppc_asm.h
@@ -755,6 +755,8 @@
#define N_SLINE 68
#define N_SO 100
+#define RFSCV .long 0x4c0000a4
+
/*
* Create an endian fixup trampoline
*
@@ -774,7 +776,7 @@
#define FIXUP_ENDIAN
#else
/*
- * This version may be used in in HV or non-HV context.
+ * This version may be used in HV or non-HV context.
* MSR[EE] must be disabled.
*/
#define FIXUP_ENDIAN \
diff --git a/arch/powerpc/include/asm/processor.h b/arch/powerpc/include/asm/processor.h
index 52a6783..ed0d633 100644
--- a/arch/powerpc/include/asm/processor.h
+++ b/arch/powerpc/include/asm/processor.h
@@ -237,7 +237,6 @@
#ifdef CONFIG_PPC_MEM_KEYS
unsigned long amr;
unsigned long iamr;
- unsigned long uamor;
#endif
#ifdef CONFIG_KVM_BOOK3S_32_HANDLER
void* kvm_shadow_vcpu; /* KVM internal data */
@@ -272,6 +271,10 @@
unsigned mmcr0;
unsigned used_ebb;
+ unsigned long mmcr3;
+ unsigned long sier2;
+ unsigned long sier3;
+
#endif
};
diff --git a/arch/powerpc/include/asm/ptrace.h b/arch/powerpc/include/asm/ptrace.h
index ac3970f..155a197 100644
--- a/arch/powerpc/include/asm/ptrace.h
+++ b/arch/powerpc/include/asm/ptrace.h
@@ -222,9 +222,14 @@
regs->trap = (regs->trap & TRAP_FLAGS_MASK) | (val & ~TRAP_FLAGS_MASK);
}
+static inline bool trap_is_scv(struct pt_regs *regs)
+{
+ return (IS_ENABLED(CONFIG_PPC_BOOK3S_64) && TRAP(regs) == 0x3000);
+}
+
static inline bool trap_is_syscall(struct pt_regs *regs)
{
- return TRAP(regs) == 0xc00;
+ return (trap_is_scv(regs) || TRAP(regs) == 0xc00);
}
static inline bool trap_norestart(struct pt_regs *regs)
@@ -238,7 +243,7 @@
}
#define arch_has_single_step() (1)
-#ifndef CONFIG_BOOK3S_601
+#ifndef CONFIG_PPC_BOOK3S_601
#define arch_has_block_step() (true)
#else
#define arch_has_block_step() (false)
diff --git a/arch/powerpc/include/asm/qspinlock.h b/arch/powerpc/include/asm/qspinlock.h
new file mode 100644
index 0000000..b752d34
--- /dev/null
+++ b/arch/powerpc/include/asm/qspinlock.h
@@ -0,0 +1,91 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#ifndef _ASM_POWERPC_QSPINLOCK_H
+#define _ASM_POWERPC_QSPINLOCK_H
+
+#include <asm-generic/qspinlock_types.h>
+#include <asm/paravirt.h>
+
+#define _Q_PENDING_LOOPS (1 << 9) /* not tuned */
+
+#ifdef CONFIG_PARAVIRT_SPINLOCKS
+extern void native_queued_spin_lock_slowpath(struct qspinlock *lock, u32 val);
+extern void __pv_queued_spin_lock_slowpath(struct qspinlock *lock, u32 val);
+extern void __pv_queued_spin_unlock(struct qspinlock *lock);
+
+static __always_inline void queued_spin_lock_slowpath(struct qspinlock *lock, u32 val)
+{
+ if (!is_shared_processor())
+ native_queued_spin_lock_slowpath(lock, val);
+ else
+ __pv_queued_spin_lock_slowpath(lock, val);
+}
+
+#define queued_spin_unlock queued_spin_unlock
+static inline void queued_spin_unlock(struct qspinlock *lock)
+{
+ if (!is_shared_processor())
+ smp_store_release(&lock->locked, 0);
+ else
+ __pv_queued_spin_unlock(lock);
+}
+
+#else
+extern void queued_spin_lock_slowpath(struct qspinlock *lock, u32 val);
+#endif
+
+static __always_inline void queued_spin_lock(struct qspinlock *lock)
+{
+ u32 val = 0;
+
+ if (likely(atomic_try_cmpxchg_lock(&lock->val, &val, _Q_LOCKED_VAL)))
+ return;
+
+ queued_spin_lock_slowpath(lock, val);
+}
+#define queued_spin_lock queued_spin_lock
+
+#define smp_mb__after_spinlock() smp_mb()
+
+static __always_inline int queued_spin_is_locked(struct qspinlock *lock)
+{
+ /*
+ * This barrier was added to simple spinlocks by commit 51d7d5205d338,
+ * but it should now be possible to remove it, asm arm64 has done with
+ * commit c6f5d02b6a0f.
+ */
+ smp_mb();
+ return atomic_read(&lock->val);
+}
+#define queued_spin_is_locked queued_spin_is_locked
+
+#ifdef CONFIG_PARAVIRT_SPINLOCKS
+#define SPIN_THRESHOLD (1<<15) /* not tuned */
+
+static __always_inline void pv_wait(u8 *ptr, u8 val)
+{
+ if (*ptr != val)
+ return;
+ yield_to_any();
+ /*
+ * We could pass in a CPU here if waiting in the queue and yield to
+ * the previous CPU in the queue.
+ */
+}
+
+static __always_inline void pv_kick(int cpu)
+{
+ prod_cpu(cpu);
+}
+
+extern void __pv_init_lock_hash(void);
+
+static inline void pv_spinlocks_init(void)
+{
+ __pv_init_lock_hash();
+}
+
+#endif
+
+#include <asm-generic/qspinlock.h>
+
+#endif /* _ASM_POWERPC_QSPINLOCK_H */
diff --git a/arch/powerpc/include/asm/qspinlock_paravirt.h b/arch/powerpc/include/asm/qspinlock_paravirt.h
new file mode 100644
index 0000000..6b60e77
--- /dev/null
+++ b/arch/powerpc/include/asm/qspinlock_paravirt.h
@@ -0,0 +1,7 @@
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+#ifndef _ASM_POWERPC_QSPINLOCK_PARAVIRT_H
+#define _ASM_POWERPC_QSPINLOCK_PARAVIRT_H
+
+EXPORT_SYMBOL(__pv_queued_spin_unlock);
+
+#endif /* _ASM_POWERPC_QSPINLOCK_PARAVIRT_H */
diff --git a/arch/powerpc/include/asm/reg.h b/arch/powerpc/include/asm/reg.h
index 88e6c78..41419f1 100644
--- a/arch/powerpc/include/asm/reg.h
+++ b/arch/powerpc/include/asm/reg.h
@@ -12,6 +12,7 @@
#ifdef __KERNEL__
#include <linux/stringify.h>
+#include <linux/const.h>
#include <asm/cputable.h>
#include <asm/asm-const.h>
#include <asm/feature-fixups.h>
@@ -876,7 +877,9 @@
#define MMCR0_FCHV 0x00000001UL /* freeze conditions in hypervisor mode */
#define SPRN_MMCR1 798
#define SPRN_MMCR2 785
+#define SPRN_MMCR3 754
#define SPRN_UMMCR2 769
+#define SPRN_UMMCR3 738
#define SPRN_MMCRA 0x312
#define MMCRA_SDSYNC 0x80000000UL /* SDAR synced with SIAR */
#define MMCRA_SDAR_DCACHE_MISS 0x40000000UL
@@ -886,6 +889,7 @@
#define MMCRA_SLOT 0x07000000UL /* SLOT bits (37-39) */
#define MMCRA_SLOT_SHIFT 24
#define MMCRA_SAMPLE_ENABLE 0x00000001UL /* enable sampling */
+#define MMCRA_BHRB_DISABLE _UL(0x2000000000) // BHRB disable bit for ISA v3.1
#define POWER6_MMCRA_SDSYNC 0x0000080000000000ULL /* SDAR/SIAR synced */
#define POWER6_MMCRA_SIHV 0x0000040000000000ULL
#define POWER6_MMCRA_SIPR 0x0000020000000000ULL
@@ -918,6 +922,10 @@
#define SIER_SIHV 0x1000000 /* Sampled MSR_HV */
#define SIER_SIAR_VALID 0x0400000 /* SIAR contents valid */
#define SIER_SDAR_VALID 0x0200000 /* SDAR contents valid */
+#define SPRN_SIER2 752
+#define SPRN_SIER3 753
+#define SPRN_USIER2 736
+#define SPRN_USIER3 737
#define SPRN_SIAR 796
#define SPRN_SDAR 797
#define SPRN_TACR 888
@@ -1472,7 +1480,7 @@
{
/*
* The HID0 update on Power8 should at the very least be
- * preceded by a a SYNC instruction followed by an ISYNC
+ * preceded by a SYNC instruction followed by an ISYNC
* instruction
*/
asm volatile("sync; mtspr %0,%1; isync":: "i"(SPRN_HID0), "r"(hid0));
diff --git a/arch/powerpc/include/asm/rtas.h b/arch/powerpc/include/asm/rtas.h
index 014968f..55f9a15 100644
--- a/arch/powerpc/include/asm/rtas.h
+++ b/arch/powerpc/include/asm/rtas.h
@@ -215,7 +215,6 @@
#define PSERIES_HP_ELOG_ACTION_ADD 1
#define PSERIES_HP_ELOG_ACTION_REMOVE 2
-#define PSERIES_HP_ELOG_ACTION_READD 3
#define PSERIES_HP_ELOG_ID_DRC_NAME 1
#define PSERIES_HP_ELOG_ID_DRC_INDEX 2
@@ -253,8 +252,6 @@
extern void rtas_progress(char *s, unsigned short hex);
extern int rtas_suspend_cpu(struct rtas_suspend_me_data *data);
extern int rtas_suspend_last_cpu(struct rtas_suspend_me_data *data);
-extern int rtas_online_cpus_mask(cpumask_var_t cpus);
-extern int rtas_offline_cpus_mask(cpumask_var_t cpus);
extern int rtas_ibm_suspend_me(u64 handle);
struct rtc_time;
diff --git a/arch/powerpc/include/asm/security_features.h b/arch/powerpc/include/asm/security_features.h
index 7c05e95..fbb8fa3 100644
--- a/arch/powerpc/include/asm/security_features.h
+++ b/arch/powerpc/include/asm/security_features.h
@@ -63,6 +63,8 @@
// bcctr 2,0,0 triggers a hardware assisted count cache flush
#define SEC_FTR_BCCTR_FLUSH_ASSIST 0x0000000000000800ull
+// bcctr 2,0,0 triggers a hardware assisted link stack flush
+#define SEC_FTR_BCCTR_LINK_FLUSH_ASSIST 0x0000000000002000ull
// Features indicating need for Spectre/Meltdown mitigations
diff --git a/arch/powerpc/include/asm/setup.h b/arch/powerpc/include/asm/setup.h
index 65676e2..9efbdde 100644
--- a/arch/powerpc/include/asm/setup.h
+++ b/arch/powerpc/include/asm/setup.h
@@ -30,12 +30,12 @@
#define ARCH_PANIC_TIMEOUT 180
#ifdef CONFIG_PPC_PSERIES
-extern void pseries_enable_reloc_on_exc(void);
+extern bool pseries_enable_reloc_on_exc(void);
extern void pseries_disable_reloc_on_exc(void);
extern void pseries_big_endian_exceptions(void);
extern void pseries_little_endian_exceptions(void);
#else
-static inline void pseries_enable_reloc_on_exc(void) {}
+static inline bool pseries_enable_reloc_on_exc(void) { return false; }
static inline void pseries_disable_reloc_on_exc(void) {}
static inline void pseries_big_endian_exceptions(void) {}
static inline void pseries_little_endian_exceptions(void) {}
diff --git a/arch/powerpc/include/asm/simple_spinlock.h b/arch/powerpc/include/asm/simple_spinlock.h
new file mode 100644
index 0000000..9c3c305
--- /dev/null
+++ b/arch/powerpc/include/asm/simple_spinlock.h
@@ -0,0 +1,288 @@
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+#ifndef _ASM_POWERPC_SIMPLE_SPINLOCK_H
+#define _ASM_POWERPC_SIMPLE_SPINLOCK_H
+
+/*
+ * Simple spin lock operations.
+ *
+ * Copyright (C) 2001-2004 Paul Mackerras <paulus@au.ibm.com>, IBM
+ * Copyright (C) 2001 Anton Blanchard <anton@au.ibm.com>, IBM
+ * Copyright (C) 2002 Dave Engebretsen <engebret@us.ibm.com>, IBM
+ * Rework to support virtual processors
+ *
+ * Type of int is used as a full 64b word is not necessary.
+ *
+ * (the type definitions are in asm/simple_spinlock_types.h)
+ */
+#include <linux/irqflags.h>
+#include <asm/paravirt.h>
+#include <asm/paca.h>
+#include <asm/synch.h>
+#include <asm/ppc-opcode.h>
+
+#ifdef CONFIG_PPC64
+/* use 0x800000yy when locked, where yy == CPU number */
+#ifdef __BIG_ENDIAN__
+#define LOCK_TOKEN (*(u32 *)(&get_paca()->lock_token))
+#else
+#define LOCK_TOKEN (*(u32 *)(&get_paca()->paca_index))
+#endif
+#else
+#define LOCK_TOKEN 1
+#endif
+
+static __always_inline int arch_spin_value_unlocked(arch_spinlock_t lock)
+{
+ return lock.slock == 0;
+}
+
+static inline int arch_spin_is_locked(arch_spinlock_t *lock)
+{
+ smp_mb();
+ return !arch_spin_value_unlocked(*lock);
+}
+
+/*
+ * This returns the old value in the lock, so we succeeded
+ * in getting the lock if the return value is 0.
+ */
+static inline unsigned long __arch_spin_trylock(arch_spinlock_t *lock)
+{
+ unsigned long tmp, token;
+
+ token = LOCK_TOKEN;
+ __asm__ __volatile__(
+"1: " PPC_LWARX(%0,0,%2,1) "\n\
+ cmpwi 0,%0,0\n\
+ bne- 2f\n\
+ stwcx. %1,0,%2\n\
+ bne- 1b\n"
+ PPC_ACQUIRE_BARRIER
+"2:"
+ : "=&r" (tmp)
+ : "r" (token), "r" (&lock->slock)
+ : "cr0", "memory");
+
+ return tmp;
+}
+
+static inline int arch_spin_trylock(arch_spinlock_t *lock)
+{
+ return __arch_spin_trylock(lock) == 0;
+}
+
+/*
+ * On a system with shared processors (that is, where a physical
+ * processor is multiplexed between several virtual processors),
+ * there is no point spinning on a lock if the holder of the lock
+ * isn't currently scheduled on a physical processor. Instead
+ * we detect this situation and ask the hypervisor to give the
+ * rest of our timeslice to the lock holder.
+ *
+ * So that we can tell which virtual processor is holding a lock,
+ * we put 0x80000000 | smp_processor_id() in the lock when it is
+ * held. Conveniently, we have a word in the paca that holds this
+ * value.
+ */
+
+#if defined(CONFIG_PPC_SPLPAR)
+/* We only yield to the hypervisor if we are in shared processor mode */
+void splpar_spin_yield(arch_spinlock_t *lock);
+void splpar_rw_yield(arch_rwlock_t *lock);
+#else /* SPLPAR */
+static inline void splpar_spin_yield(arch_spinlock_t *lock) {};
+static inline void splpar_rw_yield(arch_rwlock_t *lock) {};
+#endif
+
+static inline void spin_yield(arch_spinlock_t *lock)
+{
+ if (is_shared_processor())
+ splpar_spin_yield(lock);
+ else
+ barrier();
+}
+
+static inline void rw_yield(arch_rwlock_t *lock)
+{
+ if (is_shared_processor())
+ splpar_rw_yield(lock);
+ else
+ barrier();
+}
+
+static inline void arch_spin_lock(arch_spinlock_t *lock)
+{
+ while (1) {
+ if (likely(__arch_spin_trylock(lock) == 0))
+ break;
+ do {
+ HMT_low();
+ if (is_shared_processor())
+ splpar_spin_yield(lock);
+ } while (unlikely(lock->slock != 0));
+ HMT_medium();
+ }
+}
+
+static inline
+void arch_spin_lock_flags(arch_spinlock_t *lock, unsigned long flags)
+{
+ unsigned long flags_dis;
+
+ while (1) {
+ if (likely(__arch_spin_trylock(lock) == 0))
+ break;
+ local_save_flags(flags_dis);
+ local_irq_restore(flags);
+ do {
+ HMT_low();
+ if (is_shared_processor())
+ splpar_spin_yield(lock);
+ } while (unlikely(lock->slock != 0));
+ HMT_medium();
+ local_irq_restore(flags_dis);
+ }
+}
+#define arch_spin_lock_flags arch_spin_lock_flags
+
+static inline void arch_spin_unlock(arch_spinlock_t *lock)
+{
+ __asm__ __volatile__("# arch_spin_unlock\n\t"
+ PPC_RELEASE_BARRIER: : :"memory");
+ lock->slock = 0;
+}
+
+/*
+ * Read-write spinlocks, allowing multiple readers
+ * but only one writer.
+ *
+ * NOTE! it is quite common to have readers in interrupts
+ * but no interrupt writers. For those circumstances we
+ * can "mix" irq-safe locks - any writer needs to get a
+ * irq-safe write-lock, but readers can get non-irqsafe
+ * read-locks.
+ */
+
+#ifdef CONFIG_PPC64
+#define __DO_SIGN_EXTEND "extsw %0,%0\n"
+#define WRLOCK_TOKEN LOCK_TOKEN /* it's negative */
+#else
+#define __DO_SIGN_EXTEND
+#define WRLOCK_TOKEN (-1)
+#endif
+
+/*
+ * This returns the old value in the lock + 1,
+ * so we got a read lock if the return value is > 0.
+ */
+static inline long __arch_read_trylock(arch_rwlock_t *rw)
+{
+ long tmp;
+
+ __asm__ __volatile__(
+"1: " PPC_LWARX(%0,0,%1,1) "\n"
+ __DO_SIGN_EXTEND
+" addic. %0,%0,1\n\
+ ble- 2f\n"
+" stwcx. %0,0,%1\n\
+ bne- 1b\n"
+ PPC_ACQUIRE_BARRIER
+"2:" : "=&r" (tmp)
+ : "r" (&rw->lock)
+ : "cr0", "xer", "memory");
+
+ return tmp;
+}
+
+/*
+ * This returns the old value in the lock,
+ * so we got the write lock if the return value is 0.
+ */
+static inline long __arch_write_trylock(arch_rwlock_t *rw)
+{
+ long tmp, token;
+
+ token = WRLOCK_TOKEN;
+ __asm__ __volatile__(
+"1: " PPC_LWARX(%0,0,%2,1) "\n\
+ cmpwi 0,%0,0\n\
+ bne- 2f\n"
+" stwcx. %1,0,%2\n\
+ bne- 1b\n"
+ PPC_ACQUIRE_BARRIER
+"2:" : "=&r" (tmp)
+ : "r" (token), "r" (&rw->lock)
+ : "cr0", "memory");
+
+ return tmp;
+}
+
+static inline void arch_read_lock(arch_rwlock_t *rw)
+{
+ while (1) {
+ if (likely(__arch_read_trylock(rw) > 0))
+ break;
+ do {
+ HMT_low();
+ if (is_shared_processor())
+ splpar_rw_yield(rw);
+ } while (unlikely(rw->lock < 0));
+ HMT_medium();
+ }
+}
+
+static inline void arch_write_lock(arch_rwlock_t *rw)
+{
+ while (1) {
+ if (likely(__arch_write_trylock(rw) == 0))
+ break;
+ do {
+ HMT_low();
+ if (is_shared_processor())
+ splpar_rw_yield(rw);
+ } while (unlikely(rw->lock != 0));
+ HMT_medium();
+ }
+}
+
+static inline int arch_read_trylock(arch_rwlock_t *rw)
+{
+ return __arch_read_trylock(rw) > 0;
+}
+
+static inline int arch_write_trylock(arch_rwlock_t *rw)
+{
+ return __arch_write_trylock(rw) == 0;
+}
+
+static inline void arch_read_unlock(arch_rwlock_t *rw)
+{
+ long tmp;
+
+ __asm__ __volatile__(
+ "# read_unlock\n\t"
+ PPC_RELEASE_BARRIER
+"1: lwarx %0,0,%1\n\
+ addic %0,%0,-1\n"
+" stwcx. %0,0,%1\n\
+ bne- 1b"
+ : "=&r"(tmp)
+ : "r"(&rw->lock)
+ : "cr0", "xer", "memory");
+}
+
+static inline void arch_write_unlock(arch_rwlock_t *rw)
+{
+ __asm__ __volatile__("# write_unlock\n\t"
+ PPC_RELEASE_BARRIER: : :"memory");
+ rw->lock = 0;
+}
+
+#define arch_spin_relax(lock) spin_yield(lock)
+#define arch_read_relax(lock) rw_yield(lock)
+#define arch_write_relax(lock) rw_yield(lock)
+
+/* See include/linux/spinlock.h */
+#define smp_mb__after_spinlock() smp_mb()
+
+#endif /* _ASM_POWERPC_SIMPLE_SPINLOCK_H */
diff --git a/arch/powerpc/include/asm/simple_spinlock_types.h b/arch/powerpc/include/asm/simple_spinlock_types.h
new file mode 100644
index 0000000..0f3cdd8
--- /dev/null
+++ b/arch/powerpc/include/asm/simple_spinlock_types.h
@@ -0,0 +1,21 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#ifndef _ASM_POWERPC_SIMPLE_SPINLOCK_TYPES_H
+#define _ASM_POWERPC_SIMPLE_SPINLOCK_TYPES_H
+
+#ifndef __LINUX_SPINLOCK_TYPES_H
+# error "please don't include this file directly"
+#endif
+
+typedef struct {
+ volatile unsigned int slock;
+} arch_spinlock_t;
+
+#define __ARCH_SPIN_LOCK_UNLOCKED { 0 }
+
+typedef struct {
+ volatile signed int lock;
+} arch_rwlock_t;
+
+#define __ARCH_RW_LOCK_UNLOCKED { 0 }
+
+#endif /* _ASM_POWERPC_SIMPLE_SPINLOCK_TYPES_H */
diff --git a/arch/powerpc/include/asm/smu.h b/arch/powerpc/include/asm/smu.h
index 8dff086..4b30a02 100644
--- a/arch/powerpc/include/asm/smu.h
+++ b/arch/powerpc/include/asm/smu.h
@@ -108,7 +108,7 @@
/*
* i2c commands
*
- * To issue an i2c command, first is to send a parameter block to the
+ * To issue an i2c command, first is to send a parameter block to
* the SMU. This is a command of type 0x9a with 9 bytes of header
* eventually followed by data for a write:
*
diff --git a/arch/powerpc/include/asm/sparsemem.h b/arch/powerpc/include/asm/sparsemem.h
index c89b324..1e6fa37 100644
--- a/arch/powerpc/include/asm/sparsemem.h
+++ b/arch/powerpc/include/asm/sparsemem.h
@@ -17,12 +17,6 @@
int nid, pgprot_t prot);
extern int remove_section_mapping(unsigned long start, unsigned long end);
-#ifdef CONFIG_PPC_BOOK3S_64
-extern int resize_hpt_for_hotplug(unsigned long new_mem_size);
-#else
-static inline int resize_hpt_for_hotplug(unsigned long new_mem_size) { return 0; }
-#endif
-
#ifdef CONFIG_NUMA
extern int hot_add_scn_to_nid(unsigned long scn_addr);
#else
diff --git a/arch/powerpc/include/asm/spinlock.h b/arch/powerpc/include/asm/spinlock.h
index 2d62089..6ec7228 100644
--- a/arch/powerpc/include/asm/spinlock.h
+++ b/arch/powerpc/include/asm/spinlock.h
@@ -3,312 +3,16 @@
#define __ASM_SPINLOCK_H
#ifdef __KERNEL__
-/*
- * Simple spin lock operations.
- *
- * Copyright (C) 2001-2004 Paul Mackerras <paulus@au.ibm.com>, IBM
- * Copyright (C) 2001 Anton Blanchard <anton@au.ibm.com>, IBM
- * Copyright (C) 2002 Dave Engebretsen <engebret@us.ibm.com>, IBM
- * Rework to support virtual processors
- *
- * Type of int is used as a full 64b word is not necessary.
- *
- * (the type definitions are in asm/spinlock_types.h)
- */
-#include <linux/jump_label.h>
-#include <linux/irqflags.h>
-#ifdef CONFIG_PPC64
-#include <asm/paca.h>
-#include <asm/hvcall.h>
-#endif
-#include <asm/synch.h>
-#include <asm/ppc-opcode.h>
-
-#ifdef CONFIG_PPC64
-/* use 0x800000yy when locked, where yy == CPU number */
-#ifdef __BIG_ENDIAN__
-#define LOCK_TOKEN (*(u32 *)(&get_paca()->lock_token))
+#ifdef CONFIG_PPC_QUEUED_SPINLOCKS
+#include <asm/qspinlock.h>
+#include <asm/qrwlock.h>
#else
-#define LOCK_TOKEN (*(u32 *)(&get_paca()->paca_index))
-#endif
-#else
-#define LOCK_TOKEN 1
+#include <asm/simple_spinlock.h>
#endif
-#ifdef CONFIG_PPC_PSERIES
-DECLARE_STATIC_KEY_FALSE(shared_processor);
-
-#define vcpu_is_preempted vcpu_is_preempted
-static inline bool vcpu_is_preempted(int cpu)
-{
- if (!static_branch_unlikely(&shared_processor))
- return false;
- return !!(be32_to_cpu(lppaca_of(cpu).yield_count) & 1);
-}
+#ifndef CONFIG_PARAVIRT_SPINLOCKS
+static inline void pv_spinlocks_init(void) { }
#endif
-static __always_inline int arch_spin_value_unlocked(arch_spinlock_t lock)
-{
- return lock.slock == 0;
-}
-
-static inline int arch_spin_is_locked(arch_spinlock_t *lock)
-{
- smp_mb();
- return !arch_spin_value_unlocked(*lock);
-}
-
-/*
- * This returns the old value in the lock, so we succeeded
- * in getting the lock if the return value is 0.
- */
-static inline unsigned long __arch_spin_trylock(arch_spinlock_t *lock)
-{
- unsigned long tmp, token;
-
- token = LOCK_TOKEN;
- __asm__ __volatile__(
-"1: " PPC_LWARX(%0,0,%2,1) "\n\
- cmpwi 0,%0,0\n\
- bne- 2f\n\
- stwcx. %1,0,%2\n\
- bne- 1b\n"
- PPC_ACQUIRE_BARRIER
-"2:"
- : "=&r" (tmp)
- : "r" (token), "r" (&lock->slock)
- : "cr0", "memory");
-
- return tmp;
-}
-
-static inline int arch_spin_trylock(arch_spinlock_t *lock)
-{
- return __arch_spin_trylock(lock) == 0;
-}
-
-/*
- * On a system with shared processors (that is, where a physical
- * processor is multiplexed between several virtual processors),
- * there is no point spinning on a lock if the holder of the lock
- * isn't currently scheduled on a physical processor. Instead
- * we detect this situation and ask the hypervisor to give the
- * rest of our timeslice to the lock holder.
- *
- * So that we can tell which virtual processor is holding a lock,
- * we put 0x80000000 | smp_processor_id() in the lock when it is
- * held. Conveniently, we have a word in the paca that holds this
- * value.
- */
-
-#if defined(CONFIG_PPC_SPLPAR)
-/* We only yield to the hypervisor if we are in shared processor mode */
-void splpar_spin_yield(arch_spinlock_t *lock);
-void splpar_rw_yield(arch_rwlock_t *lock);
-#else /* SPLPAR */
-static inline void splpar_spin_yield(arch_spinlock_t *lock) {};
-static inline void splpar_rw_yield(arch_rwlock_t *lock) {};
-#endif
-
-static inline bool is_shared_processor(void)
-{
-#ifdef CONFIG_PPC_SPLPAR
- return static_branch_unlikely(&shared_processor);
-#else
- return false;
-#endif
-}
-
-static inline void spin_yield(arch_spinlock_t *lock)
-{
- if (is_shared_processor())
- splpar_spin_yield(lock);
- else
- barrier();
-}
-
-static inline void rw_yield(arch_rwlock_t *lock)
-{
- if (is_shared_processor())
- splpar_rw_yield(lock);
- else
- barrier();
-}
-
-static inline void arch_spin_lock(arch_spinlock_t *lock)
-{
- while (1) {
- if (likely(__arch_spin_trylock(lock) == 0))
- break;
- do {
- HMT_low();
- if (is_shared_processor())
- splpar_spin_yield(lock);
- } while (unlikely(lock->slock != 0));
- HMT_medium();
- }
-}
-
-static inline
-void arch_spin_lock_flags(arch_spinlock_t *lock, unsigned long flags)
-{
- unsigned long flags_dis;
-
- while (1) {
- if (likely(__arch_spin_trylock(lock) == 0))
- break;
- local_save_flags(flags_dis);
- local_irq_restore(flags);
- do {
- HMT_low();
- if (is_shared_processor())
- splpar_spin_yield(lock);
- } while (unlikely(lock->slock != 0));
- HMT_medium();
- local_irq_restore(flags_dis);
- }
-}
-#define arch_spin_lock_flags arch_spin_lock_flags
-
-static inline void arch_spin_unlock(arch_spinlock_t *lock)
-{
- __asm__ __volatile__("# arch_spin_unlock\n\t"
- PPC_RELEASE_BARRIER: : :"memory");
- lock->slock = 0;
-}
-
-/*
- * Read-write spinlocks, allowing multiple readers
- * but only one writer.
- *
- * NOTE! it is quite common to have readers in interrupts
- * but no interrupt writers. For those circumstances we
- * can "mix" irq-safe locks - any writer needs to get a
- * irq-safe write-lock, but readers can get non-irqsafe
- * read-locks.
- */
-
-#ifdef CONFIG_PPC64
-#define __DO_SIGN_EXTEND "extsw %0,%0\n"
-#define WRLOCK_TOKEN LOCK_TOKEN /* it's negative */
-#else
-#define __DO_SIGN_EXTEND
-#define WRLOCK_TOKEN (-1)
-#endif
-
-/*
- * This returns the old value in the lock + 1,
- * so we got a read lock if the return value is > 0.
- */
-static inline long __arch_read_trylock(arch_rwlock_t *rw)
-{
- long tmp;
-
- __asm__ __volatile__(
-"1: " PPC_LWARX(%0,0,%1,1) "\n"
- __DO_SIGN_EXTEND
-" addic. %0,%0,1\n\
- ble- 2f\n"
-" stwcx. %0,0,%1\n\
- bne- 1b\n"
- PPC_ACQUIRE_BARRIER
-"2:" : "=&r" (tmp)
- : "r" (&rw->lock)
- : "cr0", "xer", "memory");
-
- return tmp;
-}
-
-/*
- * This returns the old value in the lock,
- * so we got the write lock if the return value is 0.
- */
-static inline long __arch_write_trylock(arch_rwlock_t *rw)
-{
- long tmp, token;
-
- token = WRLOCK_TOKEN;
- __asm__ __volatile__(
-"1: " PPC_LWARX(%0,0,%2,1) "\n\
- cmpwi 0,%0,0\n\
- bne- 2f\n"
-" stwcx. %1,0,%2\n\
- bne- 1b\n"
- PPC_ACQUIRE_BARRIER
-"2:" : "=&r" (tmp)
- : "r" (token), "r" (&rw->lock)
- : "cr0", "memory");
-
- return tmp;
-}
-
-static inline void arch_read_lock(arch_rwlock_t *rw)
-{
- while (1) {
- if (likely(__arch_read_trylock(rw) > 0))
- break;
- do {
- HMT_low();
- if (is_shared_processor())
- splpar_rw_yield(rw);
- } while (unlikely(rw->lock < 0));
- HMT_medium();
- }
-}
-
-static inline void arch_write_lock(arch_rwlock_t *rw)
-{
- while (1) {
- if (likely(__arch_write_trylock(rw) == 0))
- break;
- do {
- HMT_low();
- if (is_shared_processor())
- splpar_rw_yield(rw);
- } while (unlikely(rw->lock != 0));
- HMT_medium();
- }
-}
-
-static inline int arch_read_trylock(arch_rwlock_t *rw)
-{
- return __arch_read_trylock(rw) > 0;
-}
-
-static inline int arch_write_trylock(arch_rwlock_t *rw)
-{
- return __arch_write_trylock(rw) == 0;
-}
-
-static inline void arch_read_unlock(arch_rwlock_t *rw)
-{
- long tmp;
-
- __asm__ __volatile__(
- "# read_unlock\n\t"
- PPC_RELEASE_BARRIER
-"1: lwarx %0,0,%1\n\
- addic %0,%0,-1\n"
-" stwcx. %0,0,%1\n\
- bne- 1b"
- : "=&r"(tmp)
- : "r"(&rw->lock)
- : "cr0", "xer", "memory");
-}
-
-static inline void arch_write_unlock(arch_rwlock_t *rw)
-{
- __asm__ __volatile__("# write_unlock\n\t"
- PPC_RELEASE_BARRIER: : :"memory");
- rw->lock = 0;
-}
-
-#define arch_spin_relax(lock) spin_yield(lock)
-#define arch_read_relax(lock) rw_yield(lock)
-#define arch_write_relax(lock) rw_yield(lock)
-
-/* See include/linux/spinlock.h */
-#define smp_mb__after_spinlock() smp_mb()
-
#endif /* __KERNEL__ */
#endif /* __ASM_SPINLOCK_H */
diff --git a/arch/powerpc/include/asm/spinlock_types.h b/arch/powerpc/include/asm/spinlock_types.h
index 87adaf1..c5d742f 100644
--- a/arch/powerpc/include/asm/spinlock_types.h
+++ b/arch/powerpc/include/asm/spinlock_types.h
@@ -6,16 +6,11 @@
# error "please don't include this file directly"
#endif
-typedef struct {
- volatile unsigned int slock;
-} arch_spinlock_t;
-
-#define __ARCH_SPIN_LOCK_UNLOCKED { 0 }
-
-typedef struct {
- volatile signed int lock;
-} arch_rwlock_t;
-
-#define __ARCH_RW_LOCK_UNLOCKED { 0 }
+#ifdef CONFIG_PPC_QUEUED_SPINLOCKS
+#include <asm-generic/qspinlock_types.h>
+#include <asm-generic/qrwlock_types.h>
+#else
+#include <asm/simple_spinlock_types.h>
+#endif
#endif
diff --git a/arch/powerpc/include/asm/sstep.h b/arch/powerpc/include/asm/sstep.h
index 3b01c69..972ed0d 100644
--- a/arch/powerpc/include/asm/sstep.h
+++ b/arch/powerpc/include/asm/sstep.h
@@ -40,6 +40,7 @@
CACHEOP,
BARRIER,
SYSCALL,
+ SYSCALL_VECTORED_0,
MFMSR,
MTMSR,
RFI,
@@ -104,6 +105,12 @@
#define MKOP(t, f, s) ((t) | (f) | SIZE(s))
+/* Prefix instruction operands */
+#define GET_PREFIX_RA(i) (((i) >> 16) & 0x1f)
+#define GET_PREFIX_R(i) ((i) & (1ul << 20))
+
+extern s32 patch__exec_instr;
+
struct instruction_op {
int type;
int reg;
diff --git a/arch/powerpc/include/asm/string.h b/arch/powerpc/include/asm/string.h
index b726927..283552c 100644
--- a/arch/powerpc/include/asm/string.h
+++ b/arch/powerpc/include/asm/string.h
@@ -28,7 +28,7 @@
extern void * memmove(void *,const void *,__kernel_size_t);
extern int memcmp(const void *,const void *,__kernel_size_t);
extern void * memchr(const void *,int,__kernel_size_t);
-extern void * memcpy_flushcache(void *,const void *,__kernel_size_t);
+void memcpy_flushcache(void *dest, const void *src, size_t size);
void *__memset(void *s, int c, __kernel_size_t count);
void *__memcpy(void *to, const void *from, __kernel_size_t n);
diff --git a/arch/powerpc/include/asm/time.h b/arch/powerpc/include/asm/time.h
index b287cfc..cb32672 100644
--- a/arch/powerpc/include/asm/time.h
+++ b/arch/powerpc/include/asm/time.h
@@ -39,7 +39,6 @@
};
/* Accessor functions for the timebase (RTC on 601) registers. */
-/* If one day CONFIG_POWER is added just define __USE_RTC as 1 */
#define __USE_RTC() (IS_ENABLED(CONFIG_PPC_BOOK3S_601))
#ifdef CONFIG_PPC64
diff --git a/arch/powerpc/include/asm/timex.h b/arch/powerpc/include/asm/timex.h
index d2d2c4b..6047402 100644
--- a/arch/powerpc/include/asm/timex.h
+++ b/arch/powerpc/include/asm/timex.h
@@ -17,7 +17,7 @@
static inline cycles_t get_cycles(void)
{
- if (IS_ENABLED(CONFIG_BOOK3S_601))
+ if (IS_ENABLED(CONFIG_PPC_BOOK3S_601))
return 0;
return mftb();
diff --git a/arch/powerpc/include/asm/topology.h b/arch/powerpc/include/asm/topology.h
index 2db7ba7..f0b6300 100644
--- a/arch/powerpc/include/asm/topology.h
+++ b/arch/powerpc/include/asm/topology.h
@@ -43,7 +43,6 @@
extern int sysfs_add_device_to_node(struct device *dev, int nid);
extern void sysfs_remove_device_from_node(struct device *dev, int nid);
-extern int numa_update_cpu_topology(bool cpus_locked);
static inline void update_numa_cpu_lookup_table(unsigned int cpu, int node)
{
@@ -78,11 +77,6 @@
{
}
-static inline int numa_update_cpu_topology(bool cpus_locked)
-{
- return 0;
-}
-
static inline void update_numa_cpu_lookup_table(unsigned int cpu, int node) {}
static inline int cpu_distance(__be32 *cpu1_assoc, __be32 *cpu2_assoc)
@@ -93,32 +87,12 @@
#endif /* CONFIG_NUMA */
#if defined(CONFIG_NUMA) && defined(CONFIG_PPC_SPLPAR)
-extern int start_topology_update(void);
-extern int stop_topology_update(void);
-extern int prrn_is_enabled(void);
extern int find_and_online_cpu_nid(int cpu);
-extern int timed_topology_update(int nsecs);
#else
-static inline int start_topology_update(void)
-{
- return 0;
-}
-static inline int stop_topology_update(void)
-{
- return 0;
-}
-static inline int prrn_is_enabled(void)
-{
- return 0;
-}
static inline int find_and_online_cpu_nid(int cpu)
{
return 0;
}
-static inline int timed_topology_update(int nsecs)
-{
- return 0;
-}
#endif /* CONFIG_NUMA && CONFIG_PPC_SPLPAR */
@@ -141,7 +115,6 @@
#define topology_core_cpumask(cpu) (per_cpu(cpu_core_map, cpu))
#define topology_core_id(cpu) (cpu_to_core_id(cpu))
-int dlpar_cpu_readd(int cpu);
#endif
#endif
diff --git a/arch/powerpc/include/asm/xive.h b/arch/powerpc/include/asm/xive.h
index d08ea11..309b4d6 100644
--- a/arch/powerpc/include/asm/xive.h
+++ b/arch/powerpc/include/asm/xive.h
@@ -155,7 +155,6 @@
static inline int xive_smp_prepare_cpu(unsigned int cpu) { return -EINVAL; }
static inline void xive_smp_setup_cpu(void) { }
static inline void xive_smp_disable_cpu(void) { }
-static inline void xive_kexec_teardown_cpu(int secondary) { }
static inline void xive_shutdown(void) { }
static inline void xive_flush_interrupt(void) { }
diff --git a/arch/powerpc/include/uapi/asm/kvm.h b/arch/powerpc/include/uapi/asm/kvm.h
index 264e266..c3af3f3 100644
--- a/arch/powerpc/include/uapi/asm/kvm.h
+++ b/arch/powerpc/include/uapi/asm/kvm.h
@@ -640,6 +640,11 @@
#define KVM_REG_PPC_ONLINE (KVM_REG_PPC | KVM_REG_SIZE_U32 | 0xbf)
#define KVM_REG_PPC_PTCR (KVM_REG_PPC | KVM_REG_SIZE_U64 | 0xc0)
+/* POWER10 registers */
+#define KVM_REG_PPC_MMCR3 (KVM_REG_PPC | KVM_REG_SIZE_U64 | 0xc1)
+#define KVM_REG_PPC_SIER2 (KVM_REG_PPC | KVM_REG_SIZE_U64 | 0xc2)
+#define KVM_REG_PPC_SIER3 (KVM_REG_PPC | KVM_REG_SIZE_U64 | 0xc3)
+
/* Transactional Memory checkpointed state:
* This is all GPRs, all VSX regs and a subset of SPRs
*/
diff --git a/arch/powerpc/include/uapi/asm/mman.h b/arch/powerpc/include/uapi/asm/mman.h
index c0c7372..3a70035 100644
--- a/arch/powerpc/include/uapi/asm/mman.h
+++ b/arch/powerpc/include/uapi/asm/mman.h
@@ -11,7 +11,7 @@
#include <asm-generic/mman-common.h>
-#define PROT_SAO 0x10 /* Strong Access Ordering */
+#define PROT_SAO 0x10 /* Unsupported since v5.9 */
#define MAP_RENAME MAP_ANONYMOUS /* In SunOS terminology */
#define MAP_NORESERVE 0x40 /* don't reserve swap pages */
diff --git a/arch/powerpc/include/uapi/asm/papr_pdsm.h b/arch/powerpc/include/uapi/asm/papr_pdsm.h
index 9ccecc1d..50ef95e 100644
--- a/arch/powerpc/include/uapi/asm/papr_pdsm.h
+++ b/arch/powerpc/include/uapi/asm/papr_pdsm.h
@@ -72,6 +72,11 @@
#define PAPR_PDSM_DIMM_CRITICAL 2
#define PAPR_PDSM_DIMM_FATAL 3
+/* struct nd_papr_pdsm_health.extension_flags field flags */
+
+/* Indicate that the 'dimm_fuel_gauge' field is valid */
+#define PDSM_DIMM_HEALTH_RUN_GAUGE_VALID 1
+
/*
* Struct exchanged between kernel & ndctl in for PAPR_PDSM_HEALTH
* Various flags indicate the health status of the dimm.
@@ -84,6 +89,7 @@
* dimm_locked : Contents of the dimm cant be modified until CEC reboot
* dimm_encrypted : Contents of dimm are encrypted.
* dimm_health : Dimm health indicator. One of PAPR_PDSM_DIMM_XXXX
+ * dimm_fuel_gauge : Life remaining of DIMM as a percentage from 0-100
*/
struct nd_papr_pdsm_health {
union {
@@ -96,6 +102,9 @@
__u8 dimm_locked;
__u8 dimm_encrypted;
__u16 dimm_health;
+
+ /* Extension flag PDSM_DIMM_HEALTH_RUN_GAUGE_VALID */
+ __u16 dimm_fuel_gauge;
};
__u8 buf[ND_PDSM_PAYLOAD_MAX_SIZE];
};
diff --git a/arch/powerpc/kernel/Makefile b/arch/powerpc/kernel/Makefile
index 244542a..d4d5946 100644
--- a/arch/powerpc/kernel/Makefile
+++ b/arch/powerpc/kernel/Makefile
@@ -45,11 +45,10 @@
signal.o sysfs.o cacheinfo.o time.o \
prom.o traps.o setup-common.o \
udbg.o misc.o io.o misc_$(BITS).o \
- of_platform.o prom_parse.o
+ of_platform.o prom_parse.o firmware.o
obj-y += ptrace/
obj-$(CONFIG_PPC64) += setup_64.o \
- paca.o nvram_64.o firmware.o note.o \
- syscall_64.o
+ paca.o nvram_64.o note.o syscall_64.o
obj-$(CONFIG_COMPAT) += sys_ppc32.o signal_32.o
obj-$(CONFIG_VDSO32) += vdso32/
obj-$(CONFIG_PPC_WATCHDOG) += watchdog.o
@@ -71,7 +70,7 @@
obj-$(CONFIG_RTAS_FLASH) += rtas_flash.o
obj-$(CONFIG_RTAS_PROC) += rtas-proc.o
obj-$(CONFIG_PPC_DT_CPU_FTRS) += dt_cpu_ftrs.o
-obj-$(CONFIG_EEH) += eeh.o eeh_pe.o eeh_dev.o eeh_cache.o \
+obj-$(CONFIG_EEH) += eeh.o eeh_pe.o eeh_cache.o \
eeh_driver.o eeh_event.o eeh_sysfs.o
obj-$(CONFIG_GENERIC_TBSYNC) += smp-tbsync.o
obj-$(CONFIG_CRASH_DUMP) += crash_dump.o
diff --git a/arch/powerpc/kernel/align.c b/arch/powerpc/kernel/align.c
index 1f1ce8b..c7797eb9 100644
--- a/arch/powerpc/kernel/align.c
+++ b/arch/powerpc/kernel/align.c
@@ -178,11 +178,11 @@
ret |= __get_user_inatomic(temp.v[1], p++);
ret |= __get_user_inatomic(temp.v[2], p++);
ret |= __get_user_inatomic(temp.v[3], p++);
- /* fall through */
+ fallthrough;
case 4:
ret |= __get_user_inatomic(temp.v[4], p++);
ret |= __get_user_inatomic(temp.v[5], p++);
- /* fall through */
+ fallthrough;
case 2:
ret |= __get_user_inatomic(temp.v[6], p++);
ret |= __get_user_inatomic(temp.v[7], p++);
@@ -263,11 +263,11 @@
ret |= __put_user_inatomic(data.v[1], p++);
ret |= __put_user_inatomic(data.v[2], p++);
ret |= __put_user_inatomic(data.v[3], p++);
- /* fall through */
+ fallthrough;
case 4:
ret |= __put_user_inatomic(data.v[4], p++);
ret |= __put_user_inatomic(data.v[5], p++);
- /* fall through */
+ fallthrough;
case 2:
ret |= __put_user_inatomic(data.v[6], p++);
ret |= __put_user_inatomic(data.v[7], p++);
diff --git a/arch/powerpc/kernel/asm-offsets.c b/arch/powerpc/kernel/asm-offsets.c
index 6657dc6..8711c21 100644
--- a/arch/powerpc/kernel/asm-offsets.c
+++ b/arch/powerpc/kernel/asm-offsets.c
@@ -559,6 +559,8 @@
OFFSET(VCPU_IRQ_PENDING, kvm_vcpu, arch.irq_pending);
OFFSET(VCPU_DBELL_REQ, kvm_vcpu, arch.doorbell_request);
OFFSET(VCPU_MMCR, kvm_vcpu, arch.mmcr);
+ OFFSET(VCPU_MMCRA, kvm_vcpu, arch.mmcra);
+ OFFSET(VCPU_MMCRS, kvm_vcpu, arch.mmcrs);
OFFSET(VCPU_PMC, kvm_vcpu, arch.pmc);
OFFSET(VCPU_SPMC, kvm_vcpu, arch.spmc);
OFFSET(VCPU_SIAR, kvm_vcpu, arch.siar);
@@ -696,6 +698,9 @@
HSTATE_FIELD(HSTATE_SDAR, host_mmcr[4]);
HSTATE_FIELD(HSTATE_MMCR2, host_mmcr[5]);
HSTATE_FIELD(HSTATE_SIER, host_mmcr[6]);
+ HSTATE_FIELD(HSTATE_MMCR3, host_mmcr[7]);
+ HSTATE_FIELD(HSTATE_SIER2, host_mmcr[8]);
+ HSTATE_FIELD(HSTATE_SIER3, host_mmcr[9]);
HSTATE_FIELD(HSTATE_PMC1, host_pmc[0]);
HSTATE_FIELD(HSTATE_PMC2, host_pmc[1]);
HSTATE_FIELD(HSTATE_PMC3, host_pmc[2]);
diff --git a/arch/powerpc/kernel/cacheinfo.c b/arch/powerpc/kernel/cacheinfo.c
index 4703362..65ab9fc 100644
--- a/arch/powerpc/kernel/cacheinfo.c
+++ b/arch/powerpc/kernel/cacheinfo.c
@@ -7,6 +7,8 @@
* Author: Nathan Lynch
*/
+#define pr_fmt(fmt) "cacheinfo: " fmt
+
#include <linux/cpu.h>
#include <linux/cpumask.h>
#include <linux/kernel.h>
@@ -166,7 +168,7 @@
list_for_each_entry(iter, &cache_list, list)
WARN_ONCE(iter->next_local == cache,
- "cache for %pOF(%s) refers to cache for %pOF(%s)\n",
+ "cache for %pOFP(%s) refers to cache for %pOFP(%s)\n",
iter->ofnode,
cache_type_string(iter),
cache->ofnode,
@@ -178,7 +180,7 @@
if (!cache)
return;
- pr_debug("freeing L%d %s cache for %pOF\n", cache->level,
+ pr_debug("freeing L%d %s cache for %pOFP\n", cache->level,
cache_type_string(cache), cache->ofnode);
release_cache_debugcheck(cache);
@@ -193,7 +195,7 @@
while (next) {
WARN_ONCE(cpumask_test_cpu(cpu, &next->shared_cpu_map),
- "CPU %i already accounted in %pOF(%s)\n",
+ "CPU %i already accounted in %pOFP(%s)\n",
cpu, next->ofnode,
cache_type_string(next));
cpumask_set_cpu(cpu, &next->shared_cpu_map);
@@ -352,7 +354,7 @@
static struct cache *cache_do_one_devnode_unified(struct device_node *node, int level)
{
- pr_debug("creating L%d ucache for %pOF\n", level, node);
+ pr_debug("creating L%d ucache for %pOFP\n", level, node);
return new_cache(cache_is_unified_d(node), level, node);
}
@@ -362,7 +364,7 @@
{
struct cache *dcache, *icache;
- pr_debug("creating L%d dcache and icache for %pOF\n", level,
+ pr_debug("creating L%d dcache and icache for %pOFP\n", level,
node);
dcache = new_cache(CACHE_TYPE_DATA, level, node);
@@ -418,12 +420,27 @@
}
smaller->next_local = bigger;
+
+ /*
+ * The cache->next_local list sorts by level ascending:
+ * L1d -> L1i -> L2 -> L3 ...
+ */
+ WARN_ONCE((smaller->level == 1 && bigger->level > 2) ||
+ (smaller->level > 1 && bigger->level != smaller->level + 1),
+ "linking L%i cache %pOFP to L%i cache %pOFP; skipped a level?\n",
+ smaller->level, smaller->ofnode, bigger->level, bigger->ofnode);
}
static void do_subsidiary_caches_debugcheck(struct cache *cache)
{
- WARN_ON_ONCE(cache->level != 1);
- WARN_ON_ONCE(!of_node_is_type(cache->ofnode, "cpu"));
+ WARN_ONCE(cache->level != 1,
+ "instantiating cache chain from L%d %s cache for "
+ "%pOFP instead of an L1\n", cache->level,
+ cache_type_string(cache), cache->ofnode);
+ WARN_ONCE(!of_node_is_type(cache->ofnode, "cpu"),
+ "instantiating cache chain from node %pOFP of type '%s' "
+ "instead of a cpu node\n", cache->ofnode,
+ of_node_get_device_type(cache->ofnode));
}
static void do_subsidiary_caches(struct cache *cache)
@@ -647,12 +664,13 @@
return &cache->shared_cpu_map;
}
-static ssize_t shared_cpu_map_show(struct kobject *k, struct kobj_attribute *attr, char *buf)
+static ssize_t
+show_shared_cpumap(struct kobject *k, struct kobj_attribute *attr, char *buf, bool list)
{
struct cache_index_dir *index;
struct cache *cache;
const struct cpumask *mask;
- int ret, cpu;
+ int cpu;
index = kobj_to_cache_index_dir(k);
cache = index->cache;
@@ -664,16 +682,25 @@
mask = &cache->shared_cpu_map;
}
- ret = scnprintf(buf, PAGE_SIZE - 1, "%*pb\n",
- cpumask_pr_args(mask));
- buf[ret++] = '\n';
- buf[ret] = '\0';
- return ret;
+ return cpumap_print_to_pagebuf(list, buf, mask);
+}
+
+static ssize_t shared_cpu_map_show(struct kobject *k, struct kobj_attribute *attr, char *buf)
+{
+ return show_shared_cpumap(k, attr, buf, false);
+}
+
+static ssize_t shared_cpu_list_show(struct kobject *k, struct kobj_attribute *attr, char *buf)
+{
+ return show_shared_cpumap(k, attr, buf, true);
}
static struct kobj_attribute cache_shared_cpu_map_attr =
__ATTR(shared_cpu_map, 0444, shared_cpu_map_show, NULL);
+static struct kobj_attribute cache_shared_cpu_list_attr =
+ __ATTR(shared_cpu_list, 0444, shared_cpu_list_show, NULL);
+
/* Attributes which should always be created -- the kobject/sysfs core
* does this automatically via kobj_type->default_attrs. This is the
* minimum data required to uniquely identify a cache.
@@ -682,6 +709,7 @@
&cache_type_attr.attr,
&cache_level_attr.attr,
&cache_shared_cpu_map_attr.attr,
+ &cache_shared_cpu_list_attr.attr,
NULL,
};
@@ -733,13 +761,13 @@
rc = attr->show(&dir->kobj, attr, buf);
if (rc <= 0) {
pr_debug("not creating %s attribute for "
- "%pOF(%s) (rc = %zd)\n",
+ "%pOFP(%s) (rc = %zd)\n",
attr->attr.name, cache->ofnode,
cache_type, rc);
continue;
}
if (sysfs_create_file(&dir->kobj, &attr->attr))
- pr_debug("could not create %s attribute for %pOF(%s)\n",
+ pr_debug("could not create %s attribute for %pOFP(%s)\n",
attr->attr.name, cache->ofnode, cache_type);
}
@@ -855,7 +883,7 @@
struct cache *next = cache->next_local;
WARN_ONCE(!cpumask_test_cpu(cpu, &cache->shared_cpu_map),
- "CPU %i not accounted in %pOF(%s)\n",
+ "CPU %i not accounted in %pOFP(%s)\n",
cpu, cache->ofnode,
cache_type_string(cache));
diff --git a/arch/powerpc/kernel/cpu_setup_power.S b/arch/powerpc/kernel/cpu_setup_power.S
index efdcfa7..704e8b9 100644
--- a/arch/powerpc/kernel/cpu_setup_power.S
+++ b/arch/powerpc/kernel/cpu_setup_power.S
@@ -94,13 +94,15 @@
_GLOBAL(__setup_cpu_power10)
mflr r11
bl __init_FSCR_power10
+ bl __init_PMU
+ bl __init_PMU_ISA31
b 1f
_GLOBAL(__setup_cpu_power9)
mflr r11
- bl __init_FSCR
-1: bl __init_PMU
- bl __init_hvmode_206
+ bl __init_FSCR_power9
+ bl __init_PMU
+1: bl __init_hvmode_206
mtlr r11
beqlr
li r0,0
@@ -124,13 +126,15 @@
_GLOBAL(__restore_cpu_power10)
mflr r11
bl __init_FSCR_power10
+ bl __init_PMU
+ bl __init_PMU_ISA31
b 1f
_GLOBAL(__restore_cpu_power9)
mflr r11
- bl __init_FSCR
-1: bl __init_PMU
- mfmsr r3
+ bl __init_FSCR_power9
+ bl __init_PMU
+1: mfmsr r3
rldicl. r0,r3,4,63
mtlr r11
beqlr
@@ -198,6 +202,12 @@
mtspr SPRN_FSCR, r3
// fall through
+__init_FSCR_power9:
+ mfspr r3, SPRN_FSCR
+ ori r3, r3, FSCR_SCV
+ mtspr SPRN_FSCR, r3
+ // fall through
+
__init_FSCR:
mfspr r3,SPRN_FSCR
ori r3,r3,FSCR_TAR|FSCR_EBB
@@ -233,3 +243,10 @@
li r5,0
mtspr SPRN_MMCRS,r5
blr
+
+__init_PMU_ISA31:
+ li r5,0
+ mtspr SPRN_MMCR3,r5
+ LOAD_REG_IMMEDIATE(r5, MMCRA_BHRB_DISABLE)
+ mtspr SPRN_MMCRA,r5
+ blr
diff --git a/arch/powerpc/kernel/cputable.c b/arch/powerpc/kernel/cputable.c
index b406635..3d406a9 100644
--- a/arch/powerpc/kernel/cputable.c
+++ b/arch/powerpc/kernel/cputable.c
@@ -120,7 +120,8 @@
#define COMMON_USER2_POWER9 (COMMON_USER2_POWER8 | \
PPC_FEATURE2_ARCH_3_00 | \
PPC_FEATURE2_HAS_IEEE128 | \
- PPC_FEATURE2_DARN )
+ PPC_FEATURE2_DARN | \
+ PPC_FEATURE2_SCV)
#define COMMON_USER_POWER10 COMMON_USER_POWER9
#define COMMON_USER2_POWER10 (COMMON_USER2_POWER9 | \
PPC_FEATURE2_ARCH_3_1 | \
diff --git a/arch/powerpc/kernel/dawr.c b/arch/powerpc/kernel/dawr.c
index 500f52f..cdc2dcc 100644
--- a/arch/powerpc/kernel/dawr.c
+++ b/arch/powerpc/kernel/dawr.c
@@ -37,7 +37,7 @@
dawrx |= (mrd & 0x3f) << (63 - 53);
if (ppc_md.set_dawr)
- return ppc_md.set_dawr(dawr, dawrx);
+ return ppc_md.set_dawr(nr, dawr, dawrx);
if (nr == 0) {
mtspr(SPRN_DAWR0, dawr);
diff --git a/arch/powerpc/kernel/dbell.c b/arch/powerpc/kernel/dbell.c
index f17ff12..52680cf 100644
--- a/arch/powerpc/kernel/dbell.c
+++ b/arch/powerpc/kernel/dbell.c
@@ -18,61 +18,6 @@
#ifdef CONFIG_SMP
-/*
- * Doorbells must only be used if CPU_FTR_DBELL is available.
- * msgsnd is used in HV, and msgsndp is used in !HV.
- *
- * These should be used by platform code that is aware of restrictions.
- * Other arch code should use ->cause_ipi.
- *
- * doorbell_global_ipi() sends a dbell to any target CPU.
- * Must be used only by architectures that address msgsnd target
- * by PIR/get_hard_smp_processor_id.
- */
-void doorbell_global_ipi(int cpu)
-{
- u32 tag = get_hard_smp_processor_id(cpu);
-
- kvmppc_set_host_ipi(cpu);
- /* Order previous accesses vs. msgsnd, which is treated as a store */
- ppc_msgsnd_sync();
- ppc_msgsnd(PPC_DBELL_MSGTYPE, 0, tag);
-}
-
-/*
- * doorbell_core_ipi() sends a dbell to a target CPU in the same core.
- * Must be used only by architectures that address msgsnd target
- * by TIR/cpu_thread_in_core.
- */
-void doorbell_core_ipi(int cpu)
-{
- u32 tag = cpu_thread_in_core(cpu);
-
- kvmppc_set_host_ipi(cpu);
- /* Order previous accesses vs. msgsnd, which is treated as a store */
- ppc_msgsnd_sync();
- ppc_msgsnd(PPC_DBELL_MSGTYPE, 0, tag);
-}
-
-/*
- * Attempt to cause a core doorbell if destination is on the same core.
- * Returns 1 on success, 0 on failure.
- */
-int doorbell_try_core_ipi(int cpu)
-{
- int this_cpu = get_cpu();
- int ret = 0;
-
- if (cpumask_test_cpu(cpu, cpu_sibling_mask(this_cpu))) {
- doorbell_core_ipi(cpu);
- ret = 1;
- }
-
- put_cpu();
-
- return ret;
-}
-
void doorbell_exception(struct pt_regs *regs)
{
struct pt_regs *old_regs = set_irq_regs(regs);
diff --git a/arch/powerpc/kernel/dt_cpu_ftrs.c b/arch/powerpc/kernel/dt_cpu_ftrs.c
index 3a40951..6f8c0c6 100644
--- a/arch/powerpc/kernel/dt_cpu_ftrs.c
+++ b/arch/powerpc/kernel/dt_cpu_ftrs.c
@@ -24,7 +24,6 @@
/* Device-tree visible constants follow */
-#define ISA_V2_07B 2070
#define ISA_V3_0B 3000
#define ISA_V3_1 3100
@@ -67,6 +66,7 @@
extern long __machine_check_early_realmode_p8(struct pt_regs *regs);
extern long __machine_check_early_realmode_p9(struct pt_regs *regs);
+extern long __machine_check_early_realmode_p10(struct pt_regs *regs);
static int hv_mode;
@@ -337,6 +337,7 @@
#ifdef CONFIG_PPC_RADIX_MMU
cur_cpu_spec->mmu_features |= MMU_FTR_TYPE_RADIX;
cur_cpu_spec->mmu_features |= MMU_FTRS_HASH_BASE;
+ cur_cpu_spec->mmu_features |= MMU_FTR_GTSE;
cur_cpu_spec->cpu_user_features |= PPC_FEATURE_HAS_MMU;
return 1;
@@ -450,6 +451,39 @@
return 1;
}
+static void init_pmu_power10(void)
+{
+ init_pmu_power9();
+
+ mtspr(SPRN_MMCR3, 0);
+ mtspr(SPRN_MMCRA, MMCRA_BHRB_DISABLE);
+}
+
+static int __init feat_enable_pmu_power10(struct dt_cpu_feature *f)
+{
+ hfscr_pmu_enable();
+
+ init_pmu_power10();
+ init_pmu_registers = init_pmu_power10;
+
+ cur_cpu_spec->cpu_features |= CPU_FTR_MMCRA;
+ cur_cpu_spec->cpu_user_features |= PPC_FEATURE_PSERIES_PERFMON_COMPAT;
+
+ cur_cpu_spec->num_pmcs = 6;
+ cur_cpu_spec->pmc_type = PPC_PMC_IBM;
+ cur_cpu_spec->oprofile_cpu_type = "ppc64/power10";
+
+ return 1;
+}
+
+static int __init feat_enable_mce_power10(struct dt_cpu_feature *f)
+{
+ cur_cpu_spec->platform = "power10";
+ cur_cpu_spec->machine_check_early = __machine_check_early_realmode_p10;
+
+ return 1;
+}
+
static int __init feat_enable_tm(struct dt_cpu_feature *f)
{
#ifdef CONFIG_PPC_TRANSACTIONAL_MEM
@@ -587,6 +621,7 @@
{"little-endian", feat_enable_le, CPU_FTR_REAL_LE},
{"smt", feat_enable_smt, 0},
{"interrupt-facilities", feat_enable, 0},
+ {"system-call-vectored", feat_enable, 0},
{"timer-facilities", feat_enable, 0},
{"timer-facilities-v3", feat_enable, 0},
{"debug-facilities", feat_enable, 0},
@@ -622,7 +657,7 @@
{"processor-control-facility-v3", feat_enable_dbell, CPU_FTR_DBELL},
{"processor-utilization-of-resources-register", feat_enable_purr, 0},
{"no-execute", feat_enable, 0},
- {"strong-access-ordering", feat_enable, CPU_FTR_SAO},
+ /* strong-access-ordering is unused */
{"cache-inhibited-large-page", feat_enable_large_ci, 0},
{"coprocessor-icswx", feat_enable, 0},
{"hypervisor-virtualization-interrupt", feat_enable_hvi, 0},
@@ -638,7 +673,9 @@
{"group-start-register", feat_enable, 0},
{"pc-relative-addressing", feat_enable, 0},
{"machine-check-power9", feat_enable_mce_power9, 0},
+ {"machine-check-power10", feat_enable_mce_power10, 0},
{"performance-monitor-power9", feat_enable_pmu_power9, 0},
+ {"performance-monitor-power10", feat_enable_pmu_power10, 0},
{"event-based-branch-v3", feat_enable, 0},
{"random-number-generator", feat_enable, 0},
{"system-call-vectored", feat_disable, 0},
@@ -649,6 +686,7 @@
{"wait-v3", feat_enable, 0},
{"prefix-instructions", feat_enable, 0},
{"matrix-multiply-assist", feat_enable_mma, 0},
+ {"debug-facilities-v31", feat_enable, CPU_FTR_DAWR1},
};
static bool __initdata using_dt_cpu_ftrs;
@@ -674,12 +712,12 @@
{
pr_info("setup for ISA %d\n", isa);
- if (isa >= 3000) {
+ if (isa >= ISA_V3_0B) {
cur_cpu_spec->cpu_features |= CPU_FTR_ARCH_300;
cur_cpu_spec->cpu_user_features2 |= PPC_FEATURE2_ARCH_3_00;
}
- if (isa >= 3100) {
+ if (isa >= ISA_V3_1) {
cur_cpu_spec->cpu_features |= CPU_FTR_ARCH_31;
cur_cpu_spec->cpu_user_features2 |= PPC_FEATURE2_ARCH_3_1;
}
@@ -776,12 +814,6 @@
}
update_tlbie_feature_flag(version);
- /*
- * PKEY was not in the initial base or feature node
- * specification, but it should become optional in the next
- * cpu feature version sequence.
- */
- cur_cpu_spec->cpu_features |= CPU_FTR_PKEY;
}
static void __init cpufeatures_setup_finished(void)
diff --git a/arch/powerpc/kernel/eeh.c b/arch/powerpc/kernel/eeh.c
index d407981..9468238 100644
--- a/arch/powerpc/kernel/eeh.c
+++ b/arch/powerpc/kernel/eeh.c
@@ -167,39 +167,33 @@
*/
static size_t eeh_dump_dev_log(struct eeh_dev *edev, char *buf, size_t len)
{
- struct pci_dn *pdn = eeh_dev_to_pdn(edev);
u32 cfg;
int cap, i;
int n = 0, l = 0;
char buffer[128];
- if (!pdn) {
- pr_warn("EEH: Note: No error log for absent device.\n");
- return 0;
- }
-
n += scnprintf(buf+n, len-n, "%04x:%02x:%02x.%01x\n",
- pdn->phb->global_number, pdn->busno,
- PCI_SLOT(pdn->devfn), PCI_FUNC(pdn->devfn));
+ edev->pe->phb->global_number, edev->bdfn >> 8,
+ PCI_SLOT(edev->bdfn), PCI_FUNC(edev->bdfn));
pr_warn("EEH: of node=%04x:%02x:%02x.%01x\n",
- pdn->phb->global_number, pdn->busno,
- PCI_SLOT(pdn->devfn), PCI_FUNC(pdn->devfn));
+ edev->pe->phb->global_number, edev->bdfn >> 8,
+ PCI_SLOT(edev->bdfn), PCI_FUNC(edev->bdfn));
- eeh_ops->read_config(pdn, PCI_VENDOR_ID, 4, &cfg);
+ eeh_ops->read_config(edev, PCI_VENDOR_ID, 4, &cfg);
n += scnprintf(buf+n, len-n, "dev/vend:%08x\n", cfg);
pr_warn("EEH: PCI device/vendor: %08x\n", cfg);
- eeh_ops->read_config(pdn, PCI_COMMAND, 4, &cfg);
+ eeh_ops->read_config(edev, PCI_COMMAND, 4, &cfg);
n += scnprintf(buf+n, len-n, "cmd/stat:%x\n", cfg);
pr_warn("EEH: PCI cmd/status register: %08x\n", cfg);
/* Gather bridge-specific registers */
if (edev->mode & EEH_DEV_BRIDGE) {
- eeh_ops->read_config(pdn, PCI_SEC_STATUS, 2, &cfg);
+ eeh_ops->read_config(edev, PCI_SEC_STATUS, 2, &cfg);
n += scnprintf(buf+n, len-n, "sec stat:%x\n", cfg);
pr_warn("EEH: Bridge secondary status: %04x\n", cfg);
- eeh_ops->read_config(pdn, PCI_BRIDGE_CONTROL, 2, &cfg);
+ eeh_ops->read_config(edev, PCI_BRIDGE_CONTROL, 2, &cfg);
n += scnprintf(buf+n, len-n, "brdg ctl:%x\n", cfg);
pr_warn("EEH: Bridge control: %04x\n", cfg);
}
@@ -207,11 +201,11 @@
/* Dump out the PCI-X command and status regs */
cap = edev->pcix_cap;
if (cap) {
- eeh_ops->read_config(pdn, cap, 4, &cfg);
+ eeh_ops->read_config(edev, cap, 4, &cfg);
n += scnprintf(buf+n, len-n, "pcix-cmd:%x\n", cfg);
pr_warn("EEH: PCI-X cmd: %08x\n", cfg);
- eeh_ops->read_config(pdn, cap+4, 4, &cfg);
+ eeh_ops->read_config(edev, cap+4, 4, &cfg);
n += scnprintf(buf+n, len-n, "pcix-stat:%x\n", cfg);
pr_warn("EEH: PCI-X status: %08x\n", cfg);
}
@@ -223,7 +217,7 @@
pr_warn("EEH: PCI-E capabilities and status follow:\n");
for (i=0; i<=8; i++) {
- eeh_ops->read_config(pdn, cap+4*i, 4, &cfg);
+ eeh_ops->read_config(edev, cap+4*i, 4, &cfg);
n += scnprintf(buf+n, len-n, "%02x:%x\n", 4*i, cfg);
if ((i % 4) == 0) {
@@ -250,7 +244,7 @@
pr_warn("EEH: PCI-E AER capability register set follows:\n");
for (i=0; i<=13; i++) {
- eeh_ops->read_config(pdn, cap+4*i, 4, &cfg);
+ eeh_ops->read_config(edev, cap+4*i, 4, &cfg);
n += scnprintf(buf+n, len-n, "%02x:%x\n", 4*i, cfg);
if ((i % 4) == 0) {
@@ -726,7 +720,6 @@
static void eeh_restore_dev_state(struct eeh_dev *edev, void *userdata)
{
- struct pci_dn *pdn = eeh_dev_to_pdn(edev);
struct pci_dev *pdev = eeh_dev_to_pci_dev(edev);
struct pci_dev *dev = userdata;
@@ -734,73 +727,14 @@
return;
/* Apply customization from firmware */
- if (pdn && eeh_ops->restore_config)
- eeh_ops->restore_config(pdn);
+ if (eeh_ops->restore_config)
+ eeh_ops->restore_config(edev);
/* The caller should restore state for the specified device */
if (pdev != dev)
pci_restore_state(pdev);
}
-int eeh_restore_vf_config(struct pci_dn *pdn)
-{
- struct eeh_dev *edev = pdn_to_eeh_dev(pdn);
- u32 devctl, cmd, cap2, aer_capctl;
- int old_mps;
-
- if (edev->pcie_cap) {
- /* Restore MPS */
- old_mps = (ffs(pdn->mps) - 8) << 5;
- eeh_ops->read_config(pdn, edev->pcie_cap + PCI_EXP_DEVCTL,
- 2, &devctl);
- devctl &= ~PCI_EXP_DEVCTL_PAYLOAD;
- devctl |= old_mps;
- eeh_ops->write_config(pdn, edev->pcie_cap + PCI_EXP_DEVCTL,
- 2, devctl);
-
- /* Disable Completion Timeout if possible */
- eeh_ops->read_config(pdn, edev->pcie_cap + PCI_EXP_DEVCAP2,
- 4, &cap2);
- if (cap2 & PCI_EXP_DEVCAP2_COMP_TMOUT_DIS) {
- eeh_ops->read_config(pdn,
- edev->pcie_cap + PCI_EXP_DEVCTL2,
- 4, &cap2);
- cap2 |= PCI_EXP_DEVCTL2_COMP_TMOUT_DIS;
- eeh_ops->write_config(pdn,
- edev->pcie_cap + PCI_EXP_DEVCTL2,
- 4, cap2);
- }
- }
-
- /* Enable SERR and parity checking */
- eeh_ops->read_config(pdn, PCI_COMMAND, 2, &cmd);
- cmd |= (PCI_COMMAND_PARITY | PCI_COMMAND_SERR);
- eeh_ops->write_config(pdn, PCI_COMMAND, 2, cmd);
-
- /* Enable report various errors */
- if (edev->pcie_cap) {
- eeh_ops->read_config(pdn, edev->pcie_cap + PCI_EXP_DEVCTL,
- 2, &devctl);
- devctl &= ~PCI_EXP_DEVCTL_CERE;
- devctl |= (PCI_EXP_DEVCTL_NFERE |
- PCI_EXP_DEVCTL_FERE |
- PCI_EXP_DEVCTL_URRE);
- eeh_ops->write_config(pdn, edev->pcie_cap + PCI_EXP_DEVCTL,
- 2, devctl);
- }
-
- /* Enable ECRC generation and check */
- if (edev->pcie_cap && edev->aer_cap) {
- eeh_ops->read_config(pdn, edev->aer_cap + PCI_ERR_CAP,
- 4, &aer_capctl);
- aer_capctl |= (PCI_ERR_CAP_ECRC_GENE | PCI_ERR_CAP_ECRC_CHKE);
- eeh_ops->write_config(pdn, edev->aer_cap + PCI_ERR_CAP,
- 4, aer_capctl);
- }
-
- return 0;
-}
-
/**
* pcibios_set_pcie_reset_state - Set PCI-E reset state
* @dev: pci device struct
@@ -977,15 +911,13 @@
*/
void eeh_save_bars(struct eeh_dev *edev)
{
- struct pci_dn *pdn;
int i;
- pdn = eeh_dev_to_pdn(edev);
- if (!pdn)
+ if (!edev)
return;
for (i = 0; i < 16; i++)
- eeh_ops->read_config(pdn, i * 4, 4, &edev->config_space[i]);
+ eeh_ops->read_config(edev, i * 4, 4, &edev->config_space[i]);
/*
* For PCI bridges including root port, we need enable bus
@@ -1096,7 +1028,7 @@
/* Initialize PHB PEs */
list_for_each_entry_safe(hose, tmp, &hose_list, list_node)
- eeh_dev_phb_init_dynamic(hose);
+ eeh_phb_pe_create(hose);
eeh_addr_cache_init();
@@ -1175,7 +1107,7 @@
* FIXME: HEY MA, LOOK AT ME, NO LOCKING!
*/
if (edev->pdev && edev->pdev != dev) {
- eeh_rmv_from_parent_pe(edev);
+ eeh_pe_tree_remove(edev);
eeh_addr_cache_rmv_dev(edev->pdev);
eeh_sysfs_remove_device(edev->pdev);
@@ -1254,7 +1186,7 @@
edev->in_error = false;
dev->dev.archdata.edev = NULL;
if (!(edev->pe->state & EEH_PE_KEEP))
- eeh_rmv_from_parent_pe(edev);
+ eeh_pe_tree_remove(edev);
else
edev->mode |= EEH_DEV_DISCONNECTED;
}
diff --git a/arch/powerpc/kernel/eeh_dev.c b/arch/powerpc/kernel/eeh_dev.c
deleted file mode 100644
index 7370185..0000000
--- a/arch/powerpc/kernel/eeh_dev.c
+++ /dev/null
@@ -1,67 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-or-later
-/*
- * The file intends to implement dynamic creation of EEH device, which will
- * be bound with OF node and PCI device simutaneously. The EEH devices would
- * be foundamental information for EEH core components to work proerly. Besides,
- * We have to support multiple situations where dynamic creation of EEH device
- * is required:
- *
- * 1) Before PCI emunation starts, we need create EEH devices according to the
- * PCI sensitive OF nodes.
- * 2) When PCI emunation is done, we need do the binding between PCI device and
- * the associated EEH device.
- * 3) DR (Dynamic Reconfiguration) would create PCI sensitive OF node. EEH device
- * will be created while PCI sensitive OF node is detected from DR.
- * 4) PCI hotplug needs redoing the binding between PCI device and EEH device. If
- * PHB is newly inserted, we also need create EEH devices accordingly.
- *
- * Copyright Benjamin Herrenschmidt & Gavin Shan, IBM Corporation 2012.
- */
-
-#include <linux/export.h>
-#include <linux/gfp.h>
-#include <linux/init.h>
-#include <linux/kernel.h>
-#include <linux/pci.h>
-#include <linux/string.h>
-
-#include <asm/pci-bridge.h>
-#include <asm/ppc-pci.h>
-
-/**
- * eeh_dev_init - Create EEH device according to OF node
- * @pdn: PCI device node
- *
- * It will create EEH device according to the given OF node. The function
- * might be called by PCI emunation, DR, PHB hotplug.
- */
-struct eeh_dev *eeh_dev_init(struct pci_dn *pdn)
-{
- struct eeh_dev *edev;
-
- /* Allocate EEH device */
- edev = kzalloc(sizeof(*edev), GFP_KERNEL);
- if (!edev)
- return NULL;
-
- /* Associate EEH device with OF node */
- pdn->edev = edev;
- edev->pdn = pdn;
- edev->bdfn = (pdn->busno << 8) | pdn->devfn;
- edev->controller = pdn->phb;
-
- return edev;
-}
-
-/**
- * eeh_dev_phb_init_dynamic - Create EEH devices for devices included in PHB
- * @phb: PHB
- *
- * Scan the PHB OF node and its child association, then create the
- * EEH devices accordingly
- */
-void eeh_dev_phb_init_dynamic(struct pci_controller *phb)
-{
- /* EEH PE for PHB */
- eeh_phb_pe_create(phb);
-}
diff --git a/arch/powerpc/kernel/eeh_driver.c b/arch/powerpc/kernel/eeh_driver.c
index 7b048ce..4197e45 100644
--- a/arch/powerpc/kernel/eeh_driver.c
+++ b/arch/powerpc/kernel/eeh_driver.c
@@ -425,8 +425,8 @@
pci_uevent_ers(edev->pdev, PCI_ERS_RESULT_RECOVERED);
#ifdef CONFIG_PCI_IOV
- if (eeh_ops->notify_resume && eeh_dev_to_pdn(edev))
- eeh_ops->notify_resume(eeh_dev_to_pdn(edev));
+ if (eeh_ops->notify_resume)
+ eeh_ops->notify_resume(edev);
#endif
return PCI_ERS_RESULT_NONE;
}
@@ -477,7 +477,7 @@
}
#ifdef CONFIG_PCI_IOV
- pci_iov_add_virtfn(edev->physfn, eeh_dev_to_pdn(edev)->vf_index);
+ pci_iov_add_virtfn(edev->physfn, edev->vf_index);
#endif
return NULL;
}
@@ -521,9 +521,7 @@
if (edev->physfn) {
#ifdef CONFIG_PCI_IOV
- struct pci_dn *pdn = eeh_dev_to_pdn(edev);
-
- pci_iov_remove_virtfn(edev->physfn, pdn->vf_index);
+ pci_iov_remove_virtfn(edev->physfn, edev->vf_index);
edev->pdev = NULL;
#endif
if (rmv_data)
@@ -544,7 +542,7 @@
continue;
edev->mode &= ~(EEH_DEV_DISCONNECTED | EEH_DEV_IRQ_DISABLED);
- eeh_rmv_from_parent_pe(edev);
+ eeh_pe_tree_remove(edev);
}
return NULL;
diff --git a/arch/powerpc/kernel/eeh_pe.c b/arch/powerpc/kernel/eeh_pe.c
index 177852e..d2aaaa7 100644
--- a/arch/powerpc/kernel/eeh_pe.c
+++ b/arch/powerpc/kernel/eeh_pe.c
@@ -319,56 +319,22 @@
}
/**
- * eeh_pe_get_parent - Retrieve the parent PE
+ * eeh_pe_tree_insert - Add EEH device to parent PE
* @edev: EEH device
+ * @new_pe_parent: PE to create additional PEs under
*
- * The whole PEs existing in the system are organized as hierarchy
- * tree. The function is used to retrieve the parent PE according
- * to the parent EEH device.
- */
-static struct eeh_pe *eeh_pe_get_parent(struct eeh_dev *edev)
-{
- struct eeh_dev *parent;
- struct pci_dn *pdn = eeh_dev_to_pdn(edev);
-
- /*
- * It might have the case for the indirect parent
- * EEH device already having associated PE, but
- * the direct parent EEH device doesn't have yet.
- */
- if (edev->physfn)
- pdn = pci_get_pdn(edev->physfn);
- else
- pdn = pdn ? pdn->parent : NULL;
- while (pdn) {
- /* We're poking out of PCI territory */
- parent = pdn_to_eeh_dev(pdn);
- if (!parent)
- return NULL;
-
- if (parent->pe)
- return parent->pe;
-
- pdn = pdn->parent;
- }
-
- return NULL;
-}
-
-/**
- * eeh_add_to_parent_pe - Add EEH device to parent PE
- * @edev: EEH device
+ * Add EEH device to the PE in edev->pe_config_addr. If a PE already
+ * exists with that address then @edev is added to that PE. Otherwise
+ * a new PE is created and inserted into the PE tree as a child of
+ * @new_pe_parent.
*
- * Add EEH device to the parent PE. If the parent PE already
- * exists, the PE type will be changed to EEH_PE_BUS. Otherwise,
- * we have to create new PE to hold the EEH device and the new
- * PE will be linked to its parent PE as well.
+ * If @new_pe_parent is NULL then the new PE will be inserted under
+ * directly under the the PHB.
*/
-int eeh_add_to_parent_pe(struct eeh_dev *edev)
+int eeh_pe_tree_insert(struct eeh_dev *edev, struct eeh_pe *new_pe_parent)
{
+ struct pci_controller *hose = edev->controller;
struct eeh_pe *pe, *parent;
- struct pci_dn *pdn = eeh_dev_to_pdn(edev);
- int config_addr = (pdn->busno << 8) | (pdn->devfn);
/* Check if the PE number is valid */
if (!eeh_has_flag(EEH_VALID_PE_ZERO) && !edev->pe_config_addr) {
@@ -382,7 +348,7 @@
* PE should be composed of PCI bus and its subordinate
* components.
*/
- pe = eeh_pe_get(pdn->phb, edev->pe_config_addr, config_addr);
+ pe = eeh_pe_get(hose, edev->pe_config_addr, edev->bdfn);
if (pe) {
if (pe->type & EEH_PE_INVALID) {
list_add_tail(&edev->entry, &pe->edevs);
@@ -399,8 +365,7 @@
parent = parent->parent;
}
- eeh_edev_dbg(edev,
- "Added to device PE (parent: PE#%x)\n",
+ eeh_edev_dbg(edev, "Added to existing PE (parent: PE#%x)\n",
pe->parent->addr);
} else {
/* Mark the PE as type of PCI bus */
@@ -416,15 +381,15 @@
/* Create a new EEH PE */
if (edev->physfn)
- pe = eeh_pe_alloc(pdn->phb, EEH_PE_VF);
+ pe = eeh_pe_alloc(hose, EEH_PE_VF);
else
- pe = eeh_pe_alloc(pdn->phb, EEH_PE_DEVICE);
+ pe = eeh_pe_alloc(hose, EEH_PE_DEVICE);
if (!pe) {
pr_err("%s: out of memory!\n", __func__);
return -ENOMEM;
}
pe->addr = edev->pe_config_addr;
- pe->config_addr = config_addr;
+ pe->config_addr = edev->bdfn;
/*
* Put the new EEH PE into hierarchy tree. If the parent
@@ -432,34 +397,35 @@
* to PHB directly. Otherwise, we have to associate the
* PE with its parent.
*/
- parent = eeh_pe_get_parent(edev);
- if (!parent) {
- parent = eeh_phb_pe_get(pdn->phb);
- if (!parent) {
+ if (!new_pe_parent) {
+ new_pe_parent = eeh_phb_pe_get(hose);
+ if (!new_pe_parent) {
pr_err("%s: No PHB PE is found (PHB Domain=%d)\n",
- __func__, pdn->phb->global_number);
+ __func__, hose->global_number);
edev->pe = NULL;
kfree(pe);
return -EEXIST;
}
}
- pe->parent = parent;
+
+ /* link new PE into the tree */
+ pe->parent = new_pe_parent;
+ list_add_tail(&pe->child, &new_pe_parent->child_list);
/*
* Put the newly created PE into the child list and
* link the EEH device accordingly.
*/
- list_add_tail(&pe->child, &parent->child_list);
list_add_tail(&edev->entry, &pe->edevs);
edev->pe = pe;
- eeh_edev_dbg(edev, "Added to device PE (parent: PE#%x)\n",
- pe->parent->addr);
+ eeh_edev_dbg(edev, "Added to new (parent: PE#%x)\n",
+ new_pe_parent->addr);
return 0;
}
/**
- * eeh_rmv_from_parent_pe - Remove one EEH device from the associated PE
+ * eeh_pe_tree_remove - Remove one EEH device from the associated PE
* @edev: EEH device
*
* The PE hierarchy tree might be changed when doing PCI hotplug.
@@ -467,7 +433,7 @@
* during EEH recovery. So we have to call the function remove the
* corresponding PE accordingly if necessary.
*/
-int eeh_rmv_from_parent_pe(struct eeh_dev *edev)
+int eeh_pe_tree_remove(struct eeh_dev *edev)
{
struct eeh_pe *pe, *parent, *child;
bool keep, recover;
@@ -698,7 +664,6 @@
*/
static void eeh_bridge_check_link(struct eeh_dev *edev)
{
- struct pci_dn *pdn = eeh_dev_to_pdn(edev);
int cap;
uint32_t val;
int timeout = 0;
@@ -714,32 +679,32 @@
/* Check slot status */
cap = edev->pcie_cap;
- eeh_ops->read_config(pdn, cap + PCI_EXP_SLTSTA, 2, &val);
+ eeh_ops->read_config(edev, cap + PCI_EXP_SLTSTA, 2, &val);
if (!(val & PCI_EXP_SLTSTA_PDS)) {
eeh_edev_dbg(edev, "No card in the slot (0x%04x) !\n", val);
return;
}
/* Check power status if we have the capability */
- eeh_ops->read_config(pdn, cap + PCI_EXP_SLTCAP, 2, &val);
+ eeh_ops->read_config(edev, cap + PCI_EXP_SLTCAP, 2, &val);
if (val & PCI_EXP_SLTCAP_PCP) {
- eeh_ops->read_config(pdn, cap + PCI_EXP_SLTCTL, 2, &val);
+ eeh_ops->read_config(edev, cap + PCI_EXP_SLTCTL, 2, &val);
if (val & PCI_EXP_SLTCTL_PCC) {
eeh_edev_dbg(edev, "In power-off state, power it on ...\n");
val &= ~(PCI_EXP_SLTCTL_PCC | PCI_EXP_SLTCTL_PIC);
val |= (0x0100 & PCI_EXP_SLTCTL_PIC);
- eeh_ops->write_config(pdn, cap + PCI_EXP_SLTCTL, 2, val);
+ eeh_ops->write_config(edev, cap + PCI_EXP_SLTCTL, 2, val);
msleep(2 * 1000);
}
}
/* Enable link */
- eeh_ops->read_config(pdn, cap + PCI_EXP_LNKCTL, 2, &val);
+ eeh_ops->read_config(edev, cap + PCI_EXP_LNKCTL, 2, &val);
val &= ~PCI_EXP_LNKCTL_LD;
- eeh_ops->write_config(pdn, cap + PCI_EXP_LNKCTL, 2, val);
+ eeh_ops->write_config(edev, cap + PCI_EXP_LNKCTL, 2, val);
/* Check link */
- eeh_ops->read_config(pdn, cap + PCI_EXP_LNKCAP, 4, &val);
+ eeh_ops->read_config(edev, cap + PCI_EXP_LNKCAP, 4, &val);
if (!(val & PCI_EXP_LNKCAP_DLLLARC)) {
eeh_edev_dbg(edev, "No link reporting capability (0x%08x) \n", val);
msleep(1000);
@@ -752,7 +717,7 @@
msleep(20);
timeout += 20;
- eeh_ops->read_config(pdn, cap + PCI_EXP_LNKSTA, 2, &val);
+ eeh_ops->read_config(edev, cap + PCI_EXP_LNKSTA, 2, &val);
if (val & PCI_EXP_LNKSTA_DLLLA)
break;
}
@@ -769,7 +734,6 @@
static void eeh_restore_bridge_bars(struct eeh_dev *edev)
{
- struct pci_dn *pdn = eeh_dev_to_pdn(edev);
int i;
/*
@@ -777,20 +741,20 @@
* Bus numbers and windows: 0x18 - 0x30
*/
for (i = 4; i < 13; i++)
- eeh_ops->write_config(pdn, i*4, 4, edev->config_space[i]);
+ eeh_ops->write_config(edev, i*4, 4, edev->config_space[i]);
/* Rom: 0x38 */
- eeh_ops->write_config(pdn, 14*4, 4, edev->config_space[14]);
+ eeh_ops->write_config(edev, 14*4, 4, edev->config_space[14]);
/* Cache line & Latency timer: 0xC 0xD */
- eeh_ops->write_config(pdn, PCI_CACHE_LINE_SIZE, 1,
+ eeh_ops->write_config(edev, PCI_CACHE_LINE_SIZE, 1,
SAVED_BYTE(PCI_CACHE_LINE_SIZE));
- eeh_ops->write_config(pdn, PCI_LATENCY_TIMER, 1,
- SAVED_BYTE(PCI_LATENCY_TIMER));
+ eeh_ops->write_config(edev, PCI_LATENCY_TIMER, 1,
+ SAVED_BYTE(PCI_LATENCY_TIMER));
/* Max latency, min grant, interrupt ping and line: 0x3C */
- eeh_ops->write_config(pdn, 15*4, 4, edev->config_space[15]);
+ eeh_ops->write_config(edev, 15*4, 4, edev->config_space[15]);
/* PCI Command: 0x4 */
- eeh_ops->write_config(pdn, PCI_COMMAND, 4, edev->config_space[1] |
+ eeh_ops->write_config(edev, PCI_COMMAND, 4, edev->config_space[1] |
PCI_COMMAND_MEMORY | PCI_COMMAND_MASTER);
/* Check the PCIe link is ready */
@@ -799,28 +763,27 @@
static void eeh_restore_device_bars(struct eeh_dev *edev)
{
- struct pci_dn *pdn = eeh_dev_to_pdn(edev);
int i;
u32 cmd;
for (i = 4; i < 10; i++)
- eeh_ops->write_config(pdn, i*4, 4, edev->config_space[i]);
+ eeh_ops->write_config(edev, i*4, 4, edev->config_space[i]);
/* 12 == Expansion ROM Address */
- eeh_ops->write_config(pdn, 12*4, 4, edev->config_space[12]);
+ eeh_ops->write_config(edev, 12*4, 4, edev->config_space[12]);
- eeh_ops->write_config(pdn, PCI_CACHE_LINE_SIZE, 1,
+ eeh_ops->write_config(edev, PCI_CACHE_LINE_SIZE, 1,
SAVED_BYTE(PCI_CACHE_LINE_SIZE));
- eeh_ops->write_config(pdn, PCI_LATENCY_TIMER, 1,
+ eeh_ops->write_config(edev, PCI_LATENCY_TIMER, 1,
SAVED_BYTE(PCI_LATENCY_TIMER));
/* max latency, min grant, interrupt pin and line */
- eeh_ops->write_config(pdn, 15*4, 4, edev->config_space[15]);
+ eeh_ops->write_config(edev, 15*4, 4, edev->config_space[15]);
/*
* Restore PERR & SERR bits, some devices require it,
* don't touch the other command bits
*/
- eeh_ops->read_config(pdn, PCI_COMMAND, 4, &cmd);
+ eeh_ops->read_config(edev, PCI_COMMAND, 4, &cmd);
if (edev->config_space[1] & PCI_COMMAND_PARITY)
cmd |= PCI_COMMAND_PARITY;
else
@@ -829,7 +792,7 @@
cmd |= PCI_COMMAND_SERR;
else
cmd &= ~PCI_COMMAND_SERR;
- eeh_ops->write_config(pdn, PCI_COMMAND, 4, cmd);
+ eeh_ops->write_config(edev, PCI_COMMAND, 4, cmd);
}
/**
@@ -843,16 +806,14 @@
*/
static void eeh_restore_one_device_bars(struct eeh_dev *edev, void *flag)
{
- struct pci_dn *pdn = eeh_dev_to_pdn(edev);
-
/* Do special restore for bridges */
if (edev->mode & EEH_DEV_BRIDGE)
eeh_restore_bridge_bars(edev);
else
eeh_restore_device_bars(edev);
- if (eeh_ops->restore_config && pdn)
- eeh_ops->restore_config(pdn);
+ if (eeh_ops->restore_config)
+ eeh_ops->restore_config(edev);
}
/**
diff --git a/arch/powerpc/kernel/eeh_sysfs.c b/arch/powerpc/kernel/eeh_sysfs.c
index 4fb0f1e..429620d 100644
--- a/arch/powerpc/kernel/eeh_sysfs.c
+++ b/arch/powerpc/kernel/eeh_sysfs.c
@@ -99,7 +99,7 @@
if (!edev || !edev->pe || !eeh_ops->notify_resume)
return -ENODEV;
- if (eeh_ops->notify_resume(pci_get_pdn(pdev)))
+ if (eeh_ops->notify_resume(edev))
return -EIO;
return count;
diff --git a/arch/powerpc/kernel/entry_32.S b/arch/powerpc/kernel/entry_32.S
index 217ebdf..f4d0af8 100644
--- a/arch/powerpc/kernel/entry_32.S
+++ b/arch/powerpc/kernel/entry_32.S
@@ -36,6 +36,12 @@
#include "head_32.h"
/*
+ * powerpc relies on return from interrupt/syscall being context synchronising
+ * (which rfi is) to support ARCH_HAS_MEMBARRIER_SYNC_CORE without additional
+ * synchronisation instructions.
+ */
+
+/*
* Align to 4k in order to ensure that all functions modyfing srr0/srr1
* fit into one page in order to not encounter a TLB miss between the
* modification of srr0/srr1 and the associated rfi.
diff --git a/arch/powerpc/kernel/entry_64.S b/arch/powerpc/kernel/entry_64.S
index 9d49338..33a42e4 100644
--- a/arch/powerpc/kernel/entry_64.S
+++ b/arch/powerpc/kernel/entry_64.S
@@ -64,15 +64,173 @@
.section ".text"
.align 7
- .globl system_call_common
-system_call_common:
+#ifdef CONFIG_PPC_BOOK3S
+.macro system_call_vectored name trapnr
+ .globl system_call_vectored_\name
+system_call_vectored_\name:
+_ASM_NOKPROBE_SYMBOL(system_call_vectored_\name)
#ifdef CONFIG_PPC_TRANSACTIONAL_MEM
BEGIN_FTR_SECTION
extrdi. r10, r12, 1, (63-MSR_TS_T_LG) /* transaction active? */
bne .Ltabort_syscall
END_FTR_SECTION_IFSET(CPU_FTR_TM)
#endif
+ INTERRUPT_TO_KERNEL
+ mr r10,r1
+ ld r1,PACAKSAVE(r13)
+ std r10,0(r1)
+ std r11,_NIP(r1)
+ std r12,_MSR(r1)
+ std r0,GPR0(r1)
+ std r10,GPR1(r1)
+ std r2,GPR2(r1)
+ ld r2,PACATOC(r13)
+ mfcr r12
+ li r11,0
+ /* Can we avoid saving r3-r8 in common case? */
+ std r3,GPR3(r1)
+ std r4,GPR4(r1)
+ std r5,GPR5(r1)
+ std r6,GPR6(r1)
+ std r7,GPR7(r1)
+ std r8,GPR8(r1)
+ /* Zero r9-r12, this should only be required when restoring all GPRs */
+ std r11,GPR9(r1)
+ std r11,GPR10(r1)
+ std r11,GPR11(r1)
+ std r11,GPR12(r1)
+ std r9,GPR13(r1)
+ SAVE_NVGPRS(r1)
+ std r11,_XER(r1)
+ std r11,_LINK(r1)
+ std r11,_CTR(r1)
+
+ li r11,\trapnr
+ std r11,_TRAP(r1)
+ std r12,_CCR(r1)
+ std r3,ORIG_GPR3(r1)
+ addi r10,r1,STACK_FRAME_OVERHEAD
+ ld r11,exception_marker@toc(r2)
+ std r11,-16(r10) /* "regshere" marker */
+
+ /*
+ * RECONCILE_IRQ_STATE without calling trace_hardirqs_off(), which
+ * would clobber syscall parameters. Also we always enter with IRQs
+ * enabled and nothing pending. system_call_exception() will call
+ * trace_hardirqs_off().
+ *
+ * scv enters with MSR[EE]=1, so don't set PACA_IRQ_HARD_DIS. The
+ * entry vector already sets PACAIRQSOFTMASK to IRQS_ALL_DISABLED.
+ */
+
+ /* Calling convention has r9 = orig r0, r10 = regs */
+ mr r9,r0
+ bl system_call_exception
+
+.Lsyscall_vectored_\name\()_exit:
+ addi r4,r1,STACK_FRAME_OVERHEAD
+ li r5,1 /* scv */
+ bl syscall_exit_prepare
+
+ ld r2,_CCR(r1)
+ ld r4,_NIP(r1)
+ ld r5,_MSR(r1)
+
+BEGIN_FTR_SECTION
+ stdcx. r0,0,r1 /* to clear the reservation */
+END_FTR_SECTION_IFCLR(CPU_FTR_STCX_CHECKS_ADDRESS)
+
+BEGIN_FTR_SECTION
+ HMT_MEDIUM_LOW
+END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR)
+
+ cmpdi r3,0
+ bne .Lsyscall_vectored_\name\()_restore_regs
+
+ /* rfscv returns with LR->NIA and CTR->MSR */
+ mtlr r4
+ mtctr r5
+
+ /* Could zero these as per ABI, but we may consider a stricter ABI
+ * which preserves these if libc implementations can benefit, so
+ * restore them for now until further measurement is done. */
+ ld r0,GPR0(r1)
+ ld r4,GPR4(r1)
+ ld r5,GPR5(r1)
+ ld r6,GPR6(r1)
+ ld r7,GPR7(r1)
+ ld r8,GPR8(r1)
+ /* Zero volatile regs that may contain sensitive kernel data */
+ li r9,0
+ li r10,0
+ li r11,0
+ li r12,0
+ mtspr SPRN_XER,r0
+
+ /*
+ * We don't need to restore AMR on the way back to userspace for KUAP.
+ * The value of AMR only matters while we're in the kernel.
+ */
+ mtcr r2
+ ld r2,GPR2(r1)
+ ld r3,GPR3(r1)
+ ld r13,GPR13(r1)
+ ld r1,GPR1(r1)
+ RFSCV_TO_USER
+ b . /* prevent speculative execution */
+
+.Lsyscall_vectored_\name\()_restore_regs:
+ li r3,0
+ mtmsrd r3,1
+ mtspr SPRN_SRR0,r4
+ mtspr SPRN_SRR1,r5
+
+ ld r3,_CTR(r1)
+ ld r4,_LINK(r1)
+ ld r5,_XER(r1)
+
+ REST_NVGPRS(r1)
+ ld r0,GPR0(r1)
+ mtcr r2
+ mtctr r3
+ mtlr r4
+ mtspr SPRN_XER,r5
+ REST_10GPRS(2, r1)
+ REST_2GPRS(12, r1)
+ ld r1,GPR1(r1)
+ RFI_TO_USER
+.endm
+
+system_call_vectored common 0x3000
+/*
+ * We instantiate another entry copy for the SIGILL variant, with TRAP=0x7ff0
+ * which is tested by system_call_exception when r0 is -1 (as set by vector
+ * entry code).
+ */
+system_call_vectored sigill 0x7ff0
+
+
+/*
+ * Entered via kernel return set up by kernel/sstep.c, must match entry regs
+ */
+ .globl system_call_vectored_emulate
+system_call_vectored_emulate:
+_ASM_NOKPROBE_SYMBOL(system_call_vectored_emulate)
+ li r10,IRQS_ALL_DISABLED
+ stb r10,PACAIRQSOFTMASK(r13)
+ b system_call_vectored_common
+#endif
+
+ .balign IFETCH_ALIGN_BYTES
+ .globl system_call_common
+system_call_common:
_ASM_NOKPROBE_SYMBOL(system_call_common)
+#ifdef CONFIG_PPC_TRANSACTIONAL_MEM
+BEGIN_FTR_SECTION
+ extrdi. r10, r12, 1, (63-MSR_TS_T_LG) /* transaction active? */
+ bne .Ltabort_syscall
+END_FTR_SECTION_IFSET(CPU_FTR_TM)
+#endif
mr r10,r1
ld r1,PACAKSAVE(r13)
std r10,0(r1)
@@ -138,6 +296,7 @@
.Lsyscall_exit:
addi r4,r1,STACK_FRAME_OVERHEAD
+ li r5,0 /* !scv */
bl syscall_exit_prepare
ld r2,_CCR(r1)
@@ -224,21 +383,29 @@
b . /* prevent speculative execution */
#endif
+#ifdef CONFIG_PPC_BOOK3S
+_GLOBAL(ret_from_fork_scv)
+ bl schedule_tail
+ REST_NVGPRS(r1)
+ li r3,0 /* fork() return value */
+ b .Lsyscall_vectored_common_exit
+#endif
+
_GLOBAL(ret_from_fork)
bl schedule_tail
REST_NVGPRS(r1)
- li r3,0
+ li r3,0 /* fork() return value */
b .Lsyscall_exit
_GLOBAL(ret_from_kernel_thread)
bl schedule_tail
REST_NVGPRS(r1)
- mtlr r14
+ mtctr r14
mr r3,r15
#ifdef PPC64_ELF_ABI_v2
mr r12,r14
#endif
- blrl
+ bctrl
li r3,0
b .Lsyscall_exit
@@ -259,10 +426,7 @@
#define FLUSH_COUNT_CACHE \
1: nop; \
- patch_site 1b, patch__call_flush_count_cache
-
-
-#define BCCTR_FLUSH .long 0x4c400420
+ patch_site 1b, patch__call_flush_branch_caches
.macro nops number
.rept \number
@@ -271,8 +435,8 @@
.endm
.balign 32
-.global flush_count_cache
-flush_count_cache:
+.global flush_branch_caches
+flush_branch_caches:
/* Save LR into r9 */
mflr r9
@@ -294,7 +458,7 @@
li r9,0x7fff
mtctr r9
- BCCTR_FLUSH
+ PPC_BCCTR_FLUSH
2: nop
patch_site 2b patch__flush_count_cache_return
@@ -303,7 +467,7 @@
.rept 278
.balign 32
- BCCTR_FLUSH
+ PPC_BCCTR_FLUSH
nops 7
.endr
@@ -357,7 +521,7 @@
* kernel/sched/core.c).
*
* Uncacheable stores in the case of involuntary preemption must
- * be taken care of. The smp_mb__before_spin_lock() in __schedule()
+ * be taken care of. The smp_mb__after_spinlock() in __schedule()
* is implemented as hwsync on powerpc, which orders MMIO too. So
* long as there is an hwsync in the context switch path, it will
* be executed on the source CPU after the task has performed
diff --git a/arch/powerpc/kernel/exceptions-64s.S b/arch/powerpc/kernel/exceptions-64s.S
index 446e54c..f7d748b 100644
--- a/arch/powerpc/kernel/exceptions-64s.S
+++ b/arch/powerpc/kernel/exceptions-64s.S
@@ -508,8 +508,24 @@
.macro __GEN_COMMON_BODY name
.if IMASK
+ .if ! ISTACK
+ .error "No support for masked interrupt to use custom stack"
+ .endif
+
+ /* If coming from user, skip soft-mask tests. */
+ andi. r10,r12,MSR_PR
+ bne 2f
+
+ /* Kernel code running below __end_interrupts is implicitly
+ * soft-masked */
+ LOAD_HANDLER(r10, __end_interrupts)
+ cmpld r11,r10
+ li r10,IMASK
+ blt- 1f
+
+ /* Test the soft mask state against our interrupt's bit */
lbz r10,PACAIRQSOFTMASK(r13)
- andi. r10,r10,IMASK
+1: andi. r10,r10,IMASK
/* Associate vector numbers with bits in paca->irq_happened */
.if IVEC == 0x500 || IVEC == 0xea0
li r10,PACA_IRQ_EE
@@ -540,7 +556,7 @@
.if ISTACK
andi. r10,r12,MSR_PR /* See if coming from user */
- mr r10,r1 /* Save r1 */
+2: mr r10,r1 /* Save r1 */
subi r1,r1,INT_FRAME_SIZE /* alloc frame on kernel stack */
beq- 100f
ld r1,PACAKSAVE(r13) /* kernel stack to use */
@@ -740,6 +756,9 @@
* guarantee they will be delivered virtually. Some conditions (see the ISA)
* cause exceptions to be delivered in real mode.
*
+ * The scv instructions are a special case. They get a 0x3000 offset applied.
+ * scv exceptions have unique reentrancy properties, see below.
+ *
* It's impossible to receive interrupts below 0x300 via AIL.
*
* KVM: None of the virtual exceptions are from the guest. Anything that
@@ -749,8 +768,8 @@
* We layout physical memory as follows:
* 0x0000 - 0x00ff : Secondary processor spin code
* 0x0100 - 0x18ff : Real mode pSeries interrupt vectors
- * 0x1900 - 0x3fff : Real mode trampolines
- * 0x4000 - 0x58ff : Relon (IR=1,DR=1) mode pSeries interrupt vectors
+ * 0x1900 - 0x2fff : Real mode trampolines
+ * 0x3000 - 0x58ff : Relon (IR=1,DR=1) mode pSeries interrupt vectors
* 0x5900 - 0x6fff : Relon mode trampolines
* 0x7000 - 0x7fff : FWNMI data area
* 0x8000 - .... : Common interrupt handlers, remaining early
@@ -761,8 +780,8 @@
* vectors there.
*/
OPEN_FIXED_SECTION(real_vectors, 0x0100, 0x1900)
-OPEN_FIXED_SECTION(real_trampolines, 0x1900, 0x4000)
-OPEN_FIXED_SECTION(virt_vectors, 0x4000, 0x5900)
+OPEN_FIXED_SECTION(real_trampolines, 0x1900, 0x3000)
+OPEN_FIXED_SECTION(virt_vectors, 0x3000, 0x5900)
OPEN_FIXED_SECTION(virt_trampolines, 0x5900, 0x7000)
#ifdef CONFIG_PPC_POWERNV
@@ -798,6 +817,77 @@
.globl __start_interrupts
__start_interrupts:
+/**
+ * Interrupt 0x3000 - System Call Vectored Interrupt (syscall).
+ * This is a synchronous interrupt invoked with the "scv" instruction. The
+ * system call does not alter the HV bit, so it is directed to the OS.
+ *
+ * Handling:
+ * scv instructions enter the kernel without changing EE, RI, ME, or HV.
+ * In particular, this means we can take a maskable interrupt at any point
+ * in the scv handler, which is unlike any other interrupt. This is solved
+ * by treating the instruction addresses below __end_interrupts as being
+ * soft-masked.
+ *
+ * AIL-0 mode scv exceptions go to 0x17000-0x17fff, but we set AIL-3 and
+ * ensure scv is never executed with relocation off, which means AIL-0
+ * should never happen.
+ *
+ * Before leaving the below __end_interrupts text, at least of the following
+ * must be true:
+ * - MSR[PR]=1 (i.e., return to userspace)
+ * - MSR_EE|MSR_RI is set (no reentrant exceptions)
+ * - Standard kernel environment is set up (stack, paca, etc)
+ *
+ * Call convention:
+ *
+ * syscall register convention is in Documentation/powerpc/syscall64-abi.rst
+ */
+EXC_VIRT_BEGIN(system_call_vectored, 0x3000, 0x1000)
+ /* SCV 0 */
+ mr r9,r13
+ GET_PACA(r13)
+ mflr r11
+ mfctr r12
+ li r10,IRQS_ALL_DISABLED
+ stb r10,PACAIRQSOFTMASK(r13)
+#ifdef CONFIG_RELOCATABLE
+ b system_call_vectored_tramp
+#else
+ b system_call_vectored_common
+#endif
+ nop
+
+ /* SCV 1 - 127 */
+ .rept 127
+ mr r9,r13
+ GET_PACA(r13)
+ mflr r11
+ mfctr r12
+ li r10,IRQS_ALL_DISABLED
+ stb r10,PACAIRQSOFTMASK(r13)
+ li r0,-1 /* cause failure */
+#ifdef CONFIG_RELOCATABLE
+ b system_call_vectored_sigill_tramp
+#else
+ b system_call_vectored_sigill
+#endif
+ .endr
+EXC_VIRT_END(system_call_vectored, 0x3000, 0x1000)
+
+#ifdef CONFIG_RELOCATABLE
+TRAMP_VIRT_BEGIN(system_call_vectored_tramp)
+ __LOAD_HANDLER(r10, system_call_vectored_common)
+ mtctr r10
+ bctr
+
+TRAMP_VIRT_BEGIN(system_call_vectored_sigill_tramp)
+ __LOAD_HANDLER(r10, system_call_vectored_sigill)
+ mtctr r10
+ bctr
+#endif
+
+
/* No virt vectors corresponding with 0x0..0x100 */
EXC_VIRT_NONE(0x4000, 0x100)
@@ -2838,7 +2928,8 @@
ld r10,PACA_EXGEN+EX_R10(r13)
ld r11,PACA_EXGEN+EX_R11(r13)
ld r12,PACA_EXGEN+EX_R12(r13)
- /* returns to kernel where r13 must be set up, so don't restore it */
+ ld r13,PACA_EXGEN+EX_R13(r13)
+ /* May return to masked low address where r13 is not set up */
.if \hsrr
HRFI_TO_KERNEL
.else
@@ -2946,6 +3037,47 @@
GET_SCRATCH0(r13);
hrfid
+TRAMP_REAL_BEGIN(rfscv_flush_fallback)
+ /* system call volatile */
+ mr r7,r13
+ GET_PACA(r13);
+ mr r8,r1
+ ld r1,PACAKSAVE(r13)
+ mfctr r9
+ ld r10,PACA_RFI_FLUSH_FALLBACK_AREA(r13)
+ ld r11,PACA_L1D_FLUSH_SIZE(r13)
+ srdi r11,r11,(7 + 3) /* 128 byte lines, unrolled 8x */
+ mtctr r11
+ DCBT_BOOK3S_STOP_ALL_STREAM_IDS(r11) /* Stop prefetch streams */
+
+ /* order ld/st prior to dcbt stop all streams with flushing */
+ sync
+
+ /*
+ * The load adresses are at staggered offsets within cachelines,
+ * which suits some pipelines better (on others it should not
+ * hurt).
+ */
+1:
+ ld r11,(0x80 + 8)*0(r10)
+ ld r11,(0x80 + 8)*1(r10)
+ ld r11,(0x80 + 8)*2(r10)
+ ld r11,(0x80 + 8)*3(r10)
+ ld r11,(0x80 + 8)*4(r10)
+ ld r11,(0x80 + 8)*5(r10)
+ ld r11,(0x80 + 8)*6(r10)
+ ld r11,(0x80 + 8)*7(r10)
+ addi r10,r10,0x80*8
+ bdnz 1b
+
+ mtctr r9
+ li r9,0
+ li r10,0
+ li r11,0
+ mr r1,r8
+ mr r13,r7
+ RFSCV
+
USE_TEXT_SECTION()
MASKED_INTERRUPT
MASKED_INTERRUPT hsrr=1
@@ -2997,6 +3129,10 @@
USE_FIXED_SECTION(virt_trampolines)
/*
+ * All code below __end_interrupts is treated as soft-masked. If
+ * any code runs here with MSR[EE]=1, it must then cope with pending
+ * soft interrupt being raised (i.e., by ensuring it is replayed).
+ *
* The __end_interrupts marker must be past the out-of-line (OOL)
* handlers, so that they are copied to real address 0x100 when running
* a relocatable kernel. This ensures they can be reached from the short
diff --git a/arch/powerpc/kernel/fadump.c b/arch/powerpc/kernel/fadump.c
index 78ab9a6..10ebb4b 100644
--- a/arch/powerpc/kernel/fadump.c
+++ b/arch/powerpc/kernel/fadump.c
@@ -32,6 +32,14 @@
#include <asm/fadump-internal.h>
#include <asm/setup.h>
+/*
+ * The CPU who acquired the lock to trigger the fadump crash should
+ * wait for other CPUs to enter.
+ *
+ * The timeout is in milliseconds.
+ */
+#define CRASH_TIMEOUT 500
+
static struct fw_dump fw_dump;
static void __init fadump_reserve_crash_area(u64 base);
@@ -39,7 +47,10 @@
struct kobject *fadump_kobj;
#ifndef CONFIG_PRESERVE_FA_DUMP
+
+static atomic_t cpus_in_fadump;
static DEFINE_MUTEX(fadump_mutex);
+
struct fadump_mrange_info crash_mrange_info = { "crash", NULL, 0, 0, 0, false };
#define RESERVED_RNGS_SZ 16384 /* 16K - 128 entries */
@@ -668,8 +679,11 @@
void crash_fadump(struct pt_regs *regs, const char *str)
{
+ unsigned int msecs;
struct fadump_crash_info_header *fdh = NULL;
int old_cpu, this_cpu;
+ /* Do not include first CPU */
+ unsigned int ncpus = num_online_cpus() - 1;
if (!should_fadump_crash())
return;
@@ -685,6 +699,8 @@
old_cpu = cmpxchg(&crashing_cpu, -1, this_cpu);
if (old_cpu != -1) {
+ atomic_inc(&cpus_in_fadump);
+
/*
* We can't loop here indefinitely. Wait as long as fadump
* is in force. If we race with fadump un-registration this
@@ -708,6 +724,16 @@
fdh->online_mask = *cpu_online_mask;
+ /*
+ * If we came in via system reset, wait a while for the secondary
+ * CPUs to enter.
+ */
+ if (TRAP(&(fdh->regs)) == 0x100) {
+ msecs = CRASH_TIMEOUT;
+ while ((atomic_read(&cpus_in_fadump) < ncpus) && (--msecs > 0))
+ mdelay(1);
+ }
+
fw_dump.ops->fadump_trigger(fdh, str);
}
diff --git a/arch/powerpc/kernel/firmware.c b/arch/powerpc/kernel/firmware.c
index cc4a5e3f..fe48d31 100644
--- a/arch/powerpc/kernel/firmware.c
+++ b/arch/powerpc/kernel/firmware.c
@@ -11,8 +11,27 @@
#include <linux/export.h>
#include <linux/cache.h>
+#include <linux/of.h>
#include <asm/firmware.h>
+#ifdef CONFIG_PPC64
unsigned long powerpc_firmware_features __read_mostly;
EXPORT_SYMBOL_GPL(powerpc_firmware_features);
+#endif
+
+#if defined(CONFIG_PPC_PSERIES) || defined(CONFIG_KVM_GUEST)
+bool is_kvm_guest(void)
+{
+ struct device_node *hyper_node;
+
+ hyper_node = of_find_node_by_path("/hypervisor");
+ if (!hyper_node)
+ return 0;
+
+ if (!of_device_is_compatible(hyper_node, "linux,kvm"))
+ return 0;
+
+ return 1;
+}
+#endif
diff --git a/arch/powerpc/kernel/fpu.S b/arch/powerpc/kernel/fpu.S
index cac22cb..4ae39db 100644
--- a/arch/powerpc/kernel/fpu.S
+++ b/arch/powerpc/kernel/fpu.S
@@ -107,9 +107,7 @@
or r12,r12,r4
std r12,_MSR(r1)
#endif
- /* Don't care if r4 overflows, this is desired behaviour */
- lbz r4,THREAD_LOAD_FP(r5)
- addi r4,r4,1
+ li r4,1
stb r4,THREAD_LOAD_FP(r5)
addi r10,r5,THREAD_FPSTATE
lfd fr0,FPSTATE_FPSCR(r10)
diff --git a/arch/powerpc/kernel/head_32.S b/arch/powerpc/kernel/head_32.S
index 705c042..f3ab94d 100644
--- a/arch/powerpc/kernel/head_32.S
+++ b/arch/powerpc/kernel/head_32.S
@@ -474,7 +474,7 @@
/* Get PTE (linux-style) and check access */
mfspr r3,SPRN_IMISS
#if defined(CONFIG_MODULES) || defined(CONFIG_DEBUG_PAGEALLOC)
- lis r1,PAGE_OFFSET@h /* check if kernel address */
+ lis r1, TASK_SIZE@h /* check if kernel address */
cmplw 0,r1,r3
#endif
mfspr r2, SPRN_SPRG_PGDIR
@@ -484,7 +484,7 @@
li r1,_PAGE_PRESENT | _PAGE_EXEC
#endif
#if defined(CONFIG_MODULES) || defined(CONFIG_DEBUG_PAGEALLOC)
- bge- 112f
+ bgt- 112f
lis r2, (swapper_pg_dir - PAGE_OFFSET)@ha /* if kernel address, use */
addi r2, r2, (swapper_pg_dir - PAGE_OFFSET)@l /* kernel page table */
#endif
@@ -541,7 +541,7 @@
*/
/* Get PTE (linux-style) and check access */
mfspr r3,SPRN_DMISS
- lis r1,PAGE_OFFSET@h /* check if kernel address */
+ lis r1, TASK_SIZE@h /* check if kernel address */
cmplw 0,r1,r3
mfspr r2, SPRN_SPRG_PGDIR
#ifdef CONFIG_SWAP
@@ -549,7 +549,7 @@
#else
li r1, _PAGE_PRESENT
#endif
- bge- 112f
+ bgt- 112f
lis r2, (swapper_pg_dir - PAGE_OFFSET)@ha /* if kernel address, use */
addi r2, r2, (swapper_pg_dir - PAGE_OFFSET)@l /* kernel page table */
112: rlwimi r2,r3,12,20,29 /* insert top 10 bits of address */
@@ -621,7 +621,7 @@
*/
/* Get PTE (linux-style) and check access */
mfspr r3,SPRN_DMISS
- lis r1,PAGE_OFFSET@h /* check if kernel address */
+ lis r1, TASK_SIZE@h /* check if kernel address */
cmplw 0,r1,r3
mfspr r2, SPRN_SPRG_PGDIR
#ifdef CONFIG_SWAP
@@ -629,7 +629,7 @@
#else
li r1, _PAGE_RW | _PAGE_DIRTY | _PAGE_PRESENT
#endif
- bge- 112f
+ bgt- 112f
lis r2, (swapper_pg_dir - PAGE_OFFSET)@ha /* if kernel address, use */
addi r2, r2, (swapper_pg_dir - PAGE_OFFSET)@l /* kernel page table */
112: rlwimi r2,r3,12,20,29 /* insert top 10 bits of address */
@@ -673,6 +673,10 @@
#define altivec_assist_exception unknown_exception
#endif
+#ifndef CONFIG_TAU_INT
+#define TAUException unknown_exception
+#endif
+
EXCEPTION(0x1300, Trap_13, instruction_breakpoint_exception, EXC_XFER_STD)
EXCEPTION(0x1400, SMI, SMIException, EXC_XFER_STD)
EXCEPTION(0x1500, Trap_15, unknown_exception, EXC_XFER_STD)
diff --git a/arch/powerpc/kernel/head_40x.S b/arch/powerpc/kernel/head_40x.S
index 926bfa7..5b282d9 100644
--- a/arch/powerpc/kernel/head_40x.S
+++ b/arch/powerpc/kernel/head_40x.S
@@ -620,7 +620,7 @@
ori r6, r6, swapper_pg_dir@l
lis r5, abatron_pteptrs@h
ori r5, r5, abatron_pteptrs@l
- stw r5, 0xf0(r0) /* Must match your Abatron config file */
+ stw r5, 0xf0(0) /* Must match your Abatron config file */
tophys(r5,r5)
stw r6, 0(r5)
diff --git a/arch/powerpc/kernel/hw_breakpoint.c b/arch/powerpc/kernel/hw_breakpoint.c
index 0000daf..1f4a1ef 100644
--- a/arch/powerpc/kernel/hw_breakpoint.c
+++ b/arch/powerpc/kernel/hw_breakpoint.c
@@ -418,8 +418,9 @@
if (dawr_enabled()) {
max_len = DAWR_MAX_LEN;
- /* DAWR region can't cross 512 bytes boundary */
- if (ALIGN(start_addr, SZ_512M) != ALIGN(end_addr - 1, SZ_512M))
+ /* DAWR region can't cross 512 bytes boundary on p10 predecessors */
+ if (!cpu_has_feature(CPU_FTR_ARCH_31) &&
+ (ALIGN_DOWN(start_addr, SZ_512) != ALIGN_DOWN(end_addr - 1, SZ_512)))
return -EINVAL;
} else if (IS_ENABLED(CONFIG_PPC_8xx)) {
/* 8xx can setup a range without limitation */
@@ -498,11 +499,11 @@
return ((info->address <= dar) && (dar - info->address < info->len));
}
-static bool dar_user_range_overlaps(unsigned long dar, int size,
- struct arch_hw_breakpoint *info)
+static bool ea_user_range_overlaps(unsigned long ea, int size,
+ struct arch_hw_breakpoint *info)
{
- return ((dar < info->address + info->len) &&
- (dar + size > info->address));
+ return ((ea < info->address + info->len) &&
+ (ea + size > info->address));
}
static bool dar_in_hw_range(unsigned long dar, struct arch_hw_breakpoint *info)
@@ -515,20 +516,22 @@
return ((hw_start_addr <= dar) && (hw_end_addr > dar));
}
-static bool dar_hw_range_overlaps(unsigned long dar, int size,
- struct arch_hw_breakpoint *info)
+static bool ea_hw_range_overlaps(unsigned long ea, int size,
+ struct arch_hw_breakpoint *info)
{
unsigned long hw_start_addr, hw_end_addr;
hw_start_addr = ALIGN_DOWN(info->address, HW_BREAKPOINT_SIZE);
hw_end_addr = ALIGN(info->address + info->len, HW_BREAKPOINT_SIZE);
- return ((dar < hw_end_addr) && (dar + size > hw_start_addr));
+ return ((ea < hw_end_addr) && (ea + size > hw_start_addr));
}
/*
* If hw has multiple DAWR registers, we also need to check all
* dawrx constraint bits to confirm this is _really_ a valid event.
+ * If type is UNKNOWN, but privilege level matches, consider it as
+ * a positive match.
*/
static bool check_dawrx_constraints(struct pt_regs *regs, int type,
struct arch_hw_breakpoint *info)
@@ -536,7 +539,12 @@
if (OP_IS_LOAD(type) && !(info->type & HW_BRK_TYPE_READ))
return false;
- if (OP_IS_STORE(type) && !(info->type & HW_BRK_TYPE_WRITE))
+ /*
+ * The Cache Management instructions other than dcbz never
+ * cause a match. i.e. if type is CACHEOP, the instruction
+ * is dcbz, and dcbz is treated as Store.
+ */
+ if ((OP_IS_STORE(type) || type == CACHEOP) && !(info->type & HW_BRK_TYPE_WRITE))
return false;
if (is_kernel_addr(regs->nip) && !(info->type & HW_BRK_TYPE_KERNEL))
@@ -553,7 +561,8 @@
* including extraneous exception. Otherwise return false.
*/
static bool check_constraints(struct pt_regs *regs, struct ppc_inst instr,
- int type, int size, struct arch_hw_breakpoint *info)
+ unsigned long ea, int type, int size,
+ struct arch_hw_breakpoint *info)
{
bool in_user_range = dar_in_user_range(regs->dar, info);
bool dawrx_constraints;
@@ -569,22 +578,27 @@
}
if (unlikely(ppc_inst_equal(instr, ppc_inst(0)))) {
- if (in_user_range)
- return true;
+ if (cpu_has_feature(CPU_FTR_ARCH_31) &&
+ !dar_in_hw_range(regs->dar, info))
+ return false;
- if (dar_in_hw_range(regs->dar, info)) {
- info->type |= HW_BRK_TYPE_EXTRANEOUS_IRQ;
- return true;
- }
- return false;
+ return true;
}
dawrx_constraints = check_dawrx_constraints(regs, type, info);
- if (dar_user_range_overlaps(regs->dar, size, info))
+ if (type == UNKNOWN) {
+ if (cpu_has_feature(CPU_FTR_ARCH_31) &&
+ !dar_in_hw_range(regs->dar, info))
+ return false;
+
+ return dawrx_constraints;
+ }
+
+ if (ea_user_range_overlaps(ea, size, info))
return dawrx_constraints;
- if (dar_hw_range_overlaps(regs->dar, size, info)) {
+ if (ea_hw_range_overlaps(ea, size, info)) {
if (dawrx_constraints) {
info->type |= HW_BRK_TYPE_EXTRANEOUS_IRQ;
return true;
@@ -593,8 +607,17 @@
return false;
}
+static int cache_op_size(void)
+{
+#ifdef __powerpc64__
+ return ppc64_caches.l1d.block_size;
+#else
+ return L1_CACHE_BYTES;
+#endif
+}
+
static void get_instr_detail(struct pt_regs *regs, struct ppc_inst *instr,
- int *type, int *size, bool *larx_stcx)
+ int *type, int *size, unsigned long *ea)
{
struct instruction_op op;
@@ -602,16 +625,23 @@
return;
analyse_instr(&op, regs, *instr);
-
- /*
- * Set size = 8 if analyse_instr() fails. If it's a userspace
- * watchpoint(valid or extraneous), we can notify user about it.
- * If it's a kernel watchpoint, instruction emulation will fail
- * in stepping_handler() and watchpoint will be disabled.
- */
*type = GETTYPE(op.type);
- *size = !(*type == UNKNOWN) ? GETSIZE(op.type) : 8;
- *larx_stcx = (*type == LARX || *type == STCX);
+ *ea = op.ea;
+#ifdef __powerpc64__
+ if (!(regs->msr & MSR_64BIT))
+ *ea &= 0xffffffffUL;
+#endif
+
+ *size = GETSIZE(op.type);
+ if (*type == CACHEOP) {
+ *size = cache_op_size();
+ *ea &= ~(*size - 1);
+ }
+}
+
+static bool is_larx_stcx_instr(int type)
+{
+ return type == LARX || type == STCX;
}
/*
@@ -678,7 +708,7 @@
struct ppc_inst instr = ppc_inst(0);
int type = 0;
int size = 0;
- bool larx_stcx = false;
+ unsigned long ea;
/* Disable breakpoints during exception handling */
hw_breakpoint_disable();
@@ -692,7 +722,7 @@
rcu_read_lock();
if (!IS_ENABLED(CONFIG_PPC_8xx))
- get_instr_detail(regs, &instr, &type, &size, &larx_stcx);
+ get_instr_detail(regs, &instr, &type, &size, &ea);
for (i = 0; i < nr_wp_slots(); i++) {
bp[i] = __this_cpu_read(bp_per_reg[i]);
@@ -702,7 +732,7 @@
info[i] = counter_arch_bp(bp[i]);
info[i]->type &= ~HW_BRK_TYPE_EXTRANEOUS_IRQ;
- if (check_constraints(regs, instr, type, size, info[i])) {
+ if (check_constraints(regs, instr, ea, type, size, info[i])) {
if (!IS_ENABLED(CONFIG_PPC_8xx) &&
ppc_inst_equal(instr, ppc_inst(0))) {
handler_error(bp[i], info[i]);
@@ -744,7 +774,7 @@
}
if (!IS_ENABLED(CONFIG_PPC_8xx)) {
- if (larx_stcx) {
+ if (is_larx_stcx_instr(type)) {
for (i = 0; i < nr_wp_slots(); i++) {
if (!hit[i])
continue;
diff --git a/arch/powerpc/kernel/irq.c b/arch/powerpc/kernel/irq.c
index 05b1cc0..bf21ebd 100644
--- a/arch/powerpc/kernel/irq.c
+++ b/arch/powerpc/kernel/irq.c
@@ -621,13 +621,14 @@
seq_printf(p, "%10u ", per_cpu(irq_stat, j).mce_exceptions);
seq_printf(p, " Machine check exceptions\n");
+#ifdef CONFIG_PPC_BOOK3S_64
if (cpu_has_feature(CPU_FTR_HVMODE)) {
seq_printf(p, "%*s: ", prec, "HMI");
for_each_online_cpu(j)
- seq_printf(p, "%10u ",
- per_cpu(irq_stat, j).hmi_exceptions);
+ seq_printf(p, "%10u ", paca_ptrs[j]->hmi_irqs);
seq_printf(p, " Hypervisor Maintenance Interrupts\n");
}
+#endif
seq_printf(p, "%*s: ", prec, "NMI");
for_each_online_cpu(j)
@@ -665,7 +666,9 @@
sum += per_cpu(irq_stat, cpu).mce_exceptions;
sum += per_cpu(irq_stat, cpu).spurious_irqs;
sum += per_cpu(irq_stat, cpu).timer_irqs_others;
- sum += per_cpu(irq_stat, cpu).hmi_exceptions;
+#ifdef CONFIG_PPC_BOOK3S_64
+ sum += paca_ptrs[cpu]->hmi_irqs;
+#endif
sum += per_cpu(irq_stat, cpu).sreset_irqs;
#ifdef CONFIG_PPC_WATCHDOG
sum += per_cpu(irq_stat, cpu).soft_nmi_irqs;
diff --git a/arch/powerpc/kernel/kprobes.c b/arch/powerpc/kernel/kprobes.c
index 9cc792a..6ab9b4d 100644
--- a/arch/powerpc/kernel/kprobes.c
+++ b/arch/powerpc/kernel/kprobes.c
@@ -244,7 +244,7 @@
* So, we should never get here... but, its still
* good to catch them, just in case...
*/
- printk("Can't step on instruction %x\n", ppc_inst_val(insn));
+ printk("Can't step on instruction %s\n", ppc_inst_as_str(insn));
BUG();
} else {
/*
diff --git a/arch/powerpc/kernel/mce.c b/arch/powerpc/kernel/mce.c
index fd90c0e..ada59f6 100644
--- a/arch/powerpc/kernel/mce.c
+++ b/arch/powerpc/kernel/mce.c
@@ -49,6 +49,20 @@
DECLARE_WORK(mce_ue_event_work, machine_process_ue_event);
+static BLOCKING_NOTIFIER_HEAD(mce_notifier_list);
+
+int mce_register_notifier(struct notifier_block *nb)
+{
+ return blocking_notifier_chain_register(&mce_notifier_list, nb);
+}
+EXPORT_SYMBOL_GPL(mce_register_notifier);
+
+int mce_unregister_notifier(struct notifier_block *nb)
+{
+ return blocking_notifier_chain_unregister(&mce_notifier_list, nb);
+}
+EXPORT_SYMBOL_GPL(mce_unregister_notifier);
+
static void mce_set_error_info(struct machine_check_event *mce,
struct mce_error_info *mce_err)
{
@@ -278,6 +292,7 @@
while (__this_cpu_read(mce_ue_count) > 0) {
index = __this_cpu_read(mce_ue_count) - 1;
evt = this_cpu_ptr(&mce_ue_event_queue[index]);
+ blocking_notifier_call_chain(&mce_notifier_list, 0, evt);
#ifdef CONFIG_MEMORY_FAILURE
/*
* This should probably queued elsewhere, but
@@ -370,6 +385,7 @@
static const char *mc_user_types[] = {
"Indeterminate",
"tlbie(l) invalid",
+ "scv invalid",
};
static const char *mc_ra_types[] = {
"Indeterminate",
@@ -711,7 +727,7 @@
{
int ret;
- __this_cpu_inc(irq_stat.hmi_exceptions);
+ local_paca->hmi_irqs++;
ret = hmi_handle_debugtrig(regs);
if (ret >= 0)
diff --git a/arch/powerpc/kernel/mce_power.c b/arch/powerpc/kernel/mce_power.c
index c3b522b..b7e1737 100644
--- a/arch/powerpc/kernel/mce_power.c
+++ b/arch/powerpc/kernel/mce_power.c
@@ -243,6 +243,45 @@
MCE_INITIATOR_CPU, MCE_SEV_SEVERE, true },
{ 0, 0, 0, 0, 0, 0, 0 } };
+static const struct mce_ierror_table mce_p10_ierror_table[] = {
+{ 0x00000000081c0000, 0x0000000000040000, true,
+ MCE_ERROR_TYPE_UE, MCE_UE_ERROR_IFETCH, MCE_ECLASS_HARDWARE,
+ MCE_INITIATOR_CPU, MCE_SEV_SEVERE, true },
+{ 0x00000000081c0000, 0x0000000000080000, true,
+ MCE_ERROR_TYPE_SLB, MCE_SLB_ERROR_PARITY, MCE_ECLASS_HARD_INDETERMINATE,
+ MCE_INITIATOR_CPU, MCE_SEV_SEVERE, true },
+{ 0x00000000081c0000, 0x00000000000c0000, true,
+ MCE_ERROR_TYPE_SLB, MCE_SLB_ERROR_MULTIHIT, MCE_ECLASS_SOFT_INDETERMINATE,
+ MCE_INITIATOR_CPU, MCE_SEV_WARNING, true },
+{ 0x00000000081c0000, 0x0000000000100000, true,
+ MCE_ERROR_TYPE_ERAT, MCE_ERAT_ERROR_MULTIHIT, MCE_ECLASS_SOFT_INDETERMINATE,
+ MCE_INITIATOR_CPU, MCE_SEV_WARNING, true },
+{ 0x00000000081c0000, 0x0000000000140000, true,
+ MCE_ERROR_TYPE_TLB, MCE_TLB_ERROR_MULTIHIT, MCE_ECLASS_SOFT_INDETERMINATE,
+ MCE_INITIATOR_CPU, MCE_SEV_WARNING, true },
+{ 0x00000000081c0000, 0x0000000000180000, true,
+ MCE_ERROR_TYPE_UE, MCE_UE_ERROR_PAGE_TABLE_WALK_IFETCH, MCE_ECLASS_HARDWARE,
+ MCE_INITIATOR_CPU, MCE_SEV_SEVERE, true },
+{ 0x00000000081c0000, 0x00000000001c0000, true,
+ MCE_ERROR_TYPE_RA, MCE_RA_ERROR_IFETCH_FOREIGN, MCE_ECLASS_SOFTWARE,
+ MCE_INITIATOR_CPU, MCE_SEV_SEVERE, true },
+{ 0x00000000081c0000, 0x0000000008080000, true,
+ MCE_ERROR_TYPE_USER,MCE_USER_ERROR_SCV, MCE_ECLASS_SOFTWARE,
+ MCE_INITIATOR_CPU, MCE_SEV_WARNING, true },
+{ 0x00000000081c0000, 0x00000000080c0000, true,
+ MCE_ERROR_TYPE_RA, MCE_RA_ERROR_IFETCH, MCE_ECLASS_SOFTWARE,
+ MCE_INITIATOR_CPU, MCE_SEV_SEVERE, true },
+{ 0x00000000081c0000, 0x0000000008100000, true,
+ MCE_ERROR_TYPE_RA, MCE_RA_ERROR_PAGE_TABLE_WALK_IFETCH, MCE_ECLASS_SOFTWARE,
+ MCE_INITIATOR_CPU, MCE_SEV_SEVERE, true },
+{ 0x00000000081c0000, 0x0000000008140000, false,
+ MCE_ERROR_TYPE_RA, MCE_RA_ERROR_STORE, MCE_ECLASS_HARDWARE,
+ MCE_INITIATOR_CPU, MCE_SEV_FATAL, false }, /* ASYNC is fatal */
+{ 0x00000000081c0000, 0x00000000081c0000, true, MCE_ECLASS_HARDWARE,
+ MCE_ERROR_TYPE_RA, MCE_RA_ERROR_PAGE_TABLE_WALK_IFETCH_FOREIGN,
+ MCE_INITIATOR_CPU, MCE_SEV_SEVERE, true },
+{ 0, 0, 0, 0, 0, 0, 0 } };
+
struct mce_derror_table {
unsigned long dsisr_value;
bool dar_valid; /* dar is a valid indicator of faulting address */
@@ -361,6 +400,46 @@
MCE_INITIATOR_CPU, MCE_SEV_SEVERE, true },
{ 0, false, 0, 0, 0, 0, 0 } };
+static const struct mce_derror_table mce_p10_derror_table[] = {
+{ 0x00008000, false,
+ MCE_ERROR_TYPE_UE, MCE_UE_ERROR_LOAD_STORE, MCE_ECLASS_HARDWARE,
+ MCE_INITIATOR_CPU, MCE_SEV_SEVERE, true },
+{ 0x00004000, true,
+ MCE_ERROR_TYPE_UE, MCE_UE_ERROR_PAGE_TABLE_WALK_LOAD_STORE,
+ MCE_ECLASS_HARDWARE,
+ MCE_INITIATOR_CPU, MCE_SEV_SEVERE, true },
+{ 0x00000800, true,
+ MCE_ERROR_TYPE_ERAT, MCE_ERAT_ERROR_MULTIHIT, MCE_ECLASS_SOFT_INDETERMINATE,
+ MCE_INITIATOR_CPU, MCE_SEV_WARNING, true },
+{ 0x00000400, true,
+ MCE_ERROR_TYPE_TLB, MCE_TLB_ERROR_MULTIHIT, MCE_ECLASS_SOFT_INDETERMINATE,
+ MCE_INITIATOR_CPU, MCE_SEV_WARNING, true },
+{ 0x00000200, false,
+ MCE_ERROR_TYPE_USER, MCE_USER_ERROR_TLBIE, MCE_ECLASS_SOFTWARE,
+ MCE_INITIATOR_CPU, MCE_SEV_WARNING, true },
+{ 0x00000080, true,
+ MCE_ERROR_TYPE_SLB, MCE_SLB_ERROR_MULTIHIT, /* Before PARITY */
+ MCE_ECLASS_SOFT_INDETERMINATE,
+ MCE_INITIATOR_CPU, MCE_SEV_WARNING, true },
+{ 0x00000100, true,
+ MCE_ERROR_TYPE_SLB, MCE_SLB_ERROR_PARITY, MCE_ECLASS_HARD_INDETERMINATE,
+ MCE_INITIATOR_CPU, MCE_SEV_SEVERE, true },
+{ 0x00000040, true,
+ MCE_ERROR_TYPE_RA, MCE_RA_ERROR_LOAD, MCE_ECLASS_HARDWARE,
+ MCE_INITIATOR_CPU, MCE_SEV_SEVERE, true },
+{ 0x00000020, false,
+ MCE_ERROR_TYPE_RA, MCE_RA_ERROR_PAGE_TABLE_WALK_LOAD_STORE,
+ MCE_ECLASS_HARDWARE,
+ MCE_INITIATOR_CPU, MCE_SEV_SEVERE, true },
+{ 0x00000010, false,
+ MCE_ERROR_TYPE_RA, MCE_RA_ERROR_PAGE_TABLE_WALK_LOAD_STORE_FOREIGN,
+ MCE_ECLASS_HARDWARE,
+ MCE_INITIATOR_CPU, MCE_SEV_SEVERE, true },
+{ 0x00000008, false,
+ MCE_ERROR_TYPE_RA, MCE_RA_ERROR_LOAD_STORE_FOREIGN, MCE_ECLASS_HARDWARE,
+ MCE_INITIATOR_CPU, MCE_SEV_SEVERE, true },
+{ 0, false, 0, 0, 0, 0, 0 } };
+
static int mce_find_instr_ea_and_phys(struct pt_regs *regs, uint64_t *addr,
uint64_t *phys_addr)
{
@@ -657,3 +736,8 @@
return mce_handle_error(regs, mce_p9_derror_table, mce_p9_ierror_table);
}
+
+long __machine_check_early_realmode_p10(struct pt_regs *regs)
+{
+ return mce_handle_error(regs, mce_p10_derror_table, mce_p10_ierror_table);
+}
diff --git a/arch/powerpc/kernel/misc_64.S b/arch/powerpc/kernel/misc_64.S
index 1864605..7bb46ad 100644
--- a/arch/powerpc/kernel/misc_64.S
+++ b/arch/powerpc/kernel/misc_64.S
@@ -413,20 +413,6 @@
li r0,0
std r0,16(r1)
-BEGIN_FTR_SECTION
- /*
- * This is the best time to turn AMR/IAMR off.
- * key 0 is used in radix for supervisor<->user
- * protection, but on hash key 0 is reserved
- * ideally we want to enter with a clean state.
- * NOTE, we rely on r0 being 0 from above.
- */
- mtspr SPRN_IAMR,r0
-BEGIN_FTR_SECTION_NESTED(42)
- mtspr SPRN_AMOR,r0
-END_FTR_SECTION_NESTED_IFSET(CPU_FTR_HVMODE, 42)
-END_FTR_SECTION_IFSET(CPU_FTR_ARCH_300)
-
/* save regs for local vars on new stack.
* yes, we won't go back, but ...
*/
diff --git a/arch/powerpc/kernel/module.c b/arch/powerpc/kernel/module.c
index df649ac..a211b02 100644
--- a/arch/powerpc/kernel/module.c
+++ b/arch/powerpc/kernel/module.c
@@ -86,3 +86,14 @@
return 0;
}
+
+#ifdef MODULES_VADDR
+void *module_alloc(unsigned long size)
+{
+ BUILD_BUG_ON(TASK_SIZE > MODULES_VADDR);
+
+ return __vmalloc_node_range(size, 1, MODULES_VADDR, MODULES_END, GFP_KERNEL,
+ PAGE_KERNEL_EXEC, VM_FLUSH_RESET_PERMS, NUMA_NO_NODE,
+ __builtin_return_address(0));
+}
+#endif
diff --git a/arch/powerpc/kernel/of_platform.c b/arch/powerpc/kernel/of_platform.c
index 71a3f97..f89376f 100644
--- a/arch/powerpc/kernel/of_platform.c
+++ b/arch/powerpc/kernel/of_platform.c
@@ -62,8 +62,8 @@
/* Init pci_dn data structures */
pci_devs_phb_init_dynamic(phb);
- /* Create EEH PEs for the PHB */
- eeh_dev_phb_init_dynamic(phb);
+ /* Create EEH PE for the PHB */
+ eeh_phb_pe_create(phb);
/* Scan the bus */
pcibios_scan_phb(phb);
diff --git a/arch/powerpc/kernel/paca.c b/arch/powerpc/kernel/paca.c
index 74da65a..0ad1576 100644
--- a/arch/powerpc/kernel/paca.c
+++ b/arch/powerpc/kernel/paca.c
@@ -57,8 +57,8 @@
#define LPPACA_SIZE 0x400
-static void *__init alloc_shared_lppaca(unsigned long size, unsigned long align,
- unsigned long limit, int cpu)
+static void *__init alloc_shared_lppaca(unsigned long size, unsigned long limit,
+ int cpu)
{
size_t shared_lppaca_total_size = PAGE_ALIGN(nr_cpu_ids * LPPACA_SIZE);
static unsigned long shared_lppaca_size;
@@ -68,6 +68,13 @@
if (!shared_lppaca) {
memblock_set_bottom_up(true);
+ /*
+ * See Documentation/powerpc/ultravisor.rst for more details.
+ *
+ * UV/HV data sharing is in PAGE_SIZE granularity. In order to
+ * minimize the number of pages shared, align the allocation to
+ * PAGE_SIZE.
+ */
shared_lppaca =
memblock_alloc_try_nid(shared_lppaca_total_size,
PAGE_SIZE, MEMBLOCK_LOW_LIMIT,
@@ -122,7 +129,7 @@
return NULL;
if (is_secure_guest())
- lp = alloc_shared_lppaca(LPPACA_SIZE, 0x400, limit, cpu);
+ lp = alloc_shared_lppaca(LPPACA_SIZE, limit, cpu);
else
lp = alloc_paca_data(LPPACA_SIZE, 0x400, limit, cpu);
diff --git a/arch/powerpc/kernel/pci_dn.c b/arch/powerpc/kernel/pci_dn.c
index 4e654df..e99b7c5 100644
--- a/arch/powerpc/kernel/pci_dn.c
+++ b/arch/powerpc/kernel/pci_dn.c
@@ -124,9 +124,28 @@
return NULL;
}
+#ifdef CONFIG_EEH
+static struct eeh_dev *eeh_dev_init(struct pci_dn *pdn)
+{
+ struct eeh_dev *edev;
+
+ /* Allocate EEH device */
+ edev = kzalloc(sizeof(*edev), GFP_KERNEL);
+ if (!edev)
+ return NULL;
+
+ /* Associate EEH device with OF node */
+ pdn->edev = edev;
+ edev->pdn = pdn;
+ edev->bdfn = (pdn->busno << 8) | pdn->devfn;
+ edev->controller = pdn->phb;
+
+ return edev;
+}
+#endif /* CONFIG_EEH */
+
#ifdef CONFIG_PCI_IOV
static struct pci_dn *add_one_sriov_vf_pdn(struct pci_dn *parent,
- int vf_index,
int busno, int devfn)
{
struct pci_dn *pdn;
@@ -143,7 +162,6 @@
pdn->parent = parent;
pdn->busno = busno;
pdn->devfn = devfn;
- pdn->vf_index = vf_index;
pdn->pe_number = IODA_INVALID_PE;
INIT_LIST_HEAD(&pdn->child_list);
INIT_LIST_HEAD(&pdn->list);
@@ -174,7 +192,7 @@
for (i = 0; i < pci_sriov_get_totalvfs(pdev); i++) {
struct eeh_dev *edev __maybe_unused;
- pdn = add_one_sriov_vf_pdn(parent, i,
+ pdn = add_one_sriov_vf_pdn(parent,
pci_iov_virtfn_bus(pdev, i),
pci_iov_virtfn_devfn(pdev, i));
if (!pdn) {
@@ -187,7 +205,10 @@
/* Create the EEH device for the VF */
edev = eeh_dev_init(pdn);
BUG_ON(!edev);
+
+ /* FIXME: these should probably be populated by the EEH probe */
edev->physfn = pdev;
+ edev->vf_index = i;
#endif /* CONFIG_EEH */
}
return pci_get_pdn(pdev);
@@ -242,7 +263,7 @@
* have a configured PE.
*/
if (edev->pe)
- eeh_rmv_from_parent_pe(edev);
+ eeh_pe_tree_remove(edev);
pdn->edev = NULL;
kfree(edev);
diff --git a/arch/powerpc/kernel/process.c b/arch/powerpc/kernel/process.c
index 794b754..016bd83 100644
--- a/arch/powerpc/kernel/process.c
+++ b/arch/powerpc/kernel/process.c
@@ -471,49 +471,58 @@
#ifdef CONFIG_PPC_BOOK3S_64
#ifdef CONFIG_PPC_FPU
-static int restore_fp(struct task_struct *tsk)
+static bool should_restore_fp(void)
{
- if (tsk->thread.load_fp) {
- load_fp_state(¤t->thread.fp_state);
+ if (current->thread.load_fp) {
current->thread.load_fp++;
- return 1;
+ return true;
}
- return 0;
+ return false;
+}
+
+static void do_restore_fp(void)
+{
+ load_fp_state(¤t->thread.fp_state);
}
#else
-static int restore_fp(struct task_struct *tsk) { return 0; }
+static bool should_restore_fp(void) { return false; }
+static void do_restore_fp(void) { }
#endif /* CONFIG_PPC_FPU */
#ifdef CONFIG_ALTIVEC
-#define loadvec(thr) ((thr).load_vec)
-static int restore_altivec(struct task_struct *tsk)
+static bool should_restore_altivec(void)
{
- if (cpu_has_feature(CPU_FTR_ALTIVEC) && (tsk->thread.load_vec)) {
- load_vr_state(&tsk->thread.vr_state);
- tsk->thread.used_vr = 1;
- tsk->thread.load_vec++;
-
- return 1;
+ if (cpu_has_feature(CPU_FTR_ALTIVEC) && (current->thread.load_vec)) {
+ current->thread.load_vec++;
+ return true;
}
- return 0;
+ return false;
+}
+
+static void do_restore_altivec(void)
+{
+ load_vr_state(¤t->thread.vr_state);
+ current->thread.used_vr = 1;
}
#else
-#define loadvec(thr) 0
-static inline int restore_altivec(struct task_struct *tsk) { return 0; }
+static bool should_restore_altivec(void) { return false; }
+static void do_restore_altivec(void) { }
#endif /* CONFIG_ALTIVEC */
#ifdef CONFIG_VSX
-static int restore_vsx(struct task_struct *tsk)
+static bool should_restore_vsx(void)
{
- if (cpu_has_feature(CPU_FTR_VSX)) {
- tsk->thread.used_vsr = 1;
- return 1;
- }
-
- return 0;
+ if (cpu_has_feature(CPU_FTR_VSX))
+ return true;
+ return false;
+}
+static void do_restore_vsx(void)
+{
+ current->thread.used_vsr = 1;
}
#else
-static inline int restore_vsx(struct task_struct *tsk) { return 0; }
+static bool should_restore_vsx(void) { return false; }
+static void do_restore_vsx(void) { }
#endif /* CONFIG_VSX */
/*
@@ -529,32 +538,42 @@
void notrace restore_math(struct pt_regs *regs)
{
unsigned long msr;
-
- if (!MSR_TM_ACTIVE(regs->msr) &&
- !current->thread.load_fp && !loadvec(current->thread))
- return;
+ unsigned long new_msr = 0;
msr = regs->msr;
- msr_check_and_set(msr_all_available);
/*
- * Only reload if the bit is not set in the user MSR, the bit BEING set
- * indicates that the registers are hot
+ * new_msr tracks the facilities that are to be restored. Only reload
+ * if the bit is not set in the user MSR (if it is set, the registers
+ * are live for the user thread).
*/
- if ((!(msr & MSR_FP)) && restore_fp(current))
- msr |= MSR_FP | current->thread.fpexc_mode;
+ if ((!(msr & MSR_FP)) && should_restore_fp())
+ new_msr |= MSR_FP | current->thread.fpexc_mode;
- if ((!(msr & MSR_VEC)) && restore_altivec(current))
- msr |= MSR_VEC;
+ if ((!(msr & MSR_VEC)) && should_restore_altivec())
+ new_msr |= MSR_VEC;
- if ((msr & (MSR_FP | MSR_VEC)) == (MSR_FP | MSR_VEC) &&
- restore_vsx(current)) {
- msr |= MSR_VSX;
+ if ((!(msr & MSR_VSX)) && should_restore_vsx()) {
+ if (((msr | new_msr) & (MSR_FP | MSR_VEC)) == (MSR_FP | MSR_VEC))
+ new_msr |= MSR_VSX;
}
- msr_check_and_clear(msr_all_available);
+ if (new_msr) {
+ msr_check_and_set(new_msr);
- regs->msr = msr;
+ if (new_msr & MSR_FP)
+ do_restore_fp();
+
+ if (new_msr & MSR_VEC)
+ do_restore_altivec();
+
+ if (new_msr & MSR_VSX)
+ do_restore_vsx();
+
+ msr_check_and_clear(new_msr);
+
+ regs->msr |= new_msr;
+ }
}
#endif
@@ -1599,6 +1618,7 @@
{
struct pt_regs *childregs, *kregs;
extern void ret_from_fork(void);
+ extern void ret_from_fork_scv(void);
extern void ret_from_kernel_thread(void);
void (*f)(void);
unsigned long sp = (unsigned long)task_stack_page(p) + THREAD_SIZE;
@@ -1635,7 +1655,9 @@
if (usp)
childregs->gpr[1] = usp;
p->thread.regs = childregs;
- childregs->gpr[3] = 0; /* Result from fork() */
+ /* 64s sets this in ret_from_fork */
+ if (!IS_ENABLED(CONFIG_PPC_BOOK3S_64))
+ childregs->gpr[3] = 0; /* Result from fork() */
if (clone_flags & CLONE_SETTLS) {
if (!is_32bit_task())
childregs->gpr[13] = tls;
@@ -1643,7 +1665,10 @@
childregs->gpr[2] = tls;
}
- f = ret_from_fork;
+ if (trap_is_scv(regs))
+ f = ret_from_fork_scv;
+ else
+ f = ret_from_fork;
}
childregs->msr &= ~(MSR_FP|MSR_VEC|MSR_VSX);
sp -= STACK_FRAME_OVERHEAD;
diff --git a/arch/powerpc/kernel/prom.c b/arch/powerpc/kernel/prom.c
index 9cc49f2..d8a2fb8 100644
--- a/arch/powerpc/kernel/prom.c
+++ b/arch/powerpc/kernel/prom.c
@@ -163,7 +163,7 @@
{ .pabyte = 0, .pabit = 6, .cpu_features = CPU_FTR_NOEXECUTE },
{ .pabyte = 1, .pabit = 2, .mmu_features = MMU_FTR_CI_LARGE_PAGE },
#ifdef CONFIG_PPC_RADIX_MMU
- { .pabyte = 40, .pabit = 0, .mmu_features = MMU_FTR_TYPE_RADIX },
+ { .pabyte = 40, .pabit = 0, .mmu_features = MMU_FTR_TYPE_RADIX | MMU_FTR_GTSE },
#endif
{ .pabyte = 1, .pabit = 1, .invert = 1, .cpu_features = CPU_FTR_NODSISRALIGN },
{ .pabyte = 5, .pabit = 0, .cpu_features = CPU_FTR_REAL_LE,
@@ -175,6 +175,8 @@
*/
{ .pabyte = 22, .pabit = 0, .cpu_features = CPU_FTR_TM_COMP,
.cpu_user_ftrs2 = PPC_FEATURE2_HTM_COMP | PPC_FEATURE2_HTM_NOSC_COMP },
+
+ { .pabyte = 64, .pabit = 0, .cpu_features = CPU_FTR_DAWR1 },
};
static void __init scan_features(unsigned long node, const unsigned char *ftrs,
@@ -468,8 +470,9 @@
* This contains a list of memory blocks along with NUMA affinity
* information.
*/
-static void __init early_init_drmem_lmb(struct drmem_lmb *lmb,
- const __be32 **usm)
+static int __init early_init_drmem_lmb(struct drmem_lmb *lmb,
+ const __be32 **usm,
+ void *data)
{
u64 base, size;
int is_kexec_kdump = 0, rngs;
@@ -484,7 +487,7 @@
*/
if ((lmb->flags & DRCONF_MEM_RESERVED) ||
!(lmb->flags & DRCONF_MEM_ASSIGNED))
- return;
+ return 0;
if (*usm)
is_kexec_kdump = 1;
@@ -499,7 +502,7 @@
*/
rngs = dt_mem_next_cell(dt_root_size_cells, usm);
if (!rngs) /* there are no (base, size) duple */
- return;
+ return 0;
}
do {
@@ -524,6 +527,8 @@
if (lmb->flags & DRCONF_MEM_HOTREMOVABLE)
memblock_mark_hotplug(base, size);
} while (--rngs);
+
+ return 0;
}
#endif /* CONFIG_PPC_PSERIES */
@@ -534,7 +539,7 @@
#ifdef CONFIG_PPC_PSERIES
if (depth == 1 &&
strcmp(uname, "ibm,dynamic-reconfiguration-memory") == 0) {
- walk_drmem_lmbs_early(node, early_init_drmem_lmb);
+ walk_drmem_lmbs_early(node, NULL, early_init_drmem_lmb);
return 0;
}
#endif
@@ -815,6 +820,11 @@
/* Now try to figure out if we are running on LPAR and so on */
pseries_probe_fw_features();
+ /*
+ * Initialize pkey features and default AMR/IAMR values
+ */
+ pkey_early_init_devtree();
+
#ifdef CONFIG_PPC_PS3
/* Identify PS3 firmware */
if (of_flat_dt_is_compatible(of_get_flat_dt_root(), "sony,ps3"))
diff --git a/arch/powerpc/kernel/prom_init.c b/arch/powerpc/kernel/prom_init.c
index 90c604d..ae7ec99 100644
--- a/arch/powerpc/kernel/prom_init.c
+++ b/arch/powerpc/kernel/prom_init.c
@@ -169,6 +169,7 @@
#ifdef CONFIG_PPC_PSERIES
static bool __prombss prom_radix_disable;
+static bool __prombss prom_radix_gtse_disable;
static bool __prombss prom_xive_disable;
#endif
@@ -823,6 +824,12 @@
if (prom_radix_disable)
prom_debug("Radix disabled from cmdline\n");
+ opt = prom_strstr(prom_cmd_line, "radix_hcall_invalidate=on");
+ if (opt) {
+ prom_radix_gtse_disable = true;
+ prom_debug("Radix GTSE disabled from cmdline\n");
+ }
+
opt = prom_strstr(prom_cmd_line, "xive=off");
if (opt) {
prom_xive_disable = true;
@@ -1285,10 +1292,8 @@
prom_parse_mmu_model(val & OV5_FEAT(OV5_MMU_SUPPORT), support);
break;
case OV5_INDX(OV5_RADIX_GTSE): /* Radix Extensions */
- if (val & OV5_FEAT(OV5_RADIX_GTSE)) {
- prom_debug("Radix - GTSE supported\n");
- support->radix_gtse = true;
- }
+ if (val & OV5_FEAT(OV5_RADIX_GTSE))
+ support->radix_gtse = !prom_radix_gtse_disable;
break;
case OV5_INDX(OV5_XIVE_SUPPORT): /* Interrupt mode */
prom_parse_xive_model(val & OV5_FEAT(OV5_XIVE_SUPPORT),
@@ -1336,12 +1341,15 @@
}
}
- if (supported.radix_mmu && supported.radix_gtse &&
- IS_ENABLED(CONFIG_PPC_RADIX_MMU)) {
- /* Radix preferred - but we require GTSE for now */
- prom_debug("Asking for radix with GTSE\n");
+ if (supported.radix_mmu && IS_ENABLED(CONFIG_PPC_RADIX_MMU)) {
+ /* Radix preferred - Check if GTSE is also supported */
+ prom_debug("Asking for radix\n");
ibm_architecture_vec.vec5.mmu = OV5_FEAT(OV5_MMU_RADIX);
- ibm_architecture_vec.vec5.radix_ext = OV5_FEAT(OV5_RADIX_GTSE);
+ if (supported.radix_gtse)
+ ibm_architecture_vec.vec5.radix_ext =
+ OV5_FEAT(OV5_RADIX_GTSE);
+ else
+ prom_debug("Radix GTSE isn't supported\n");
} else if (supported.hash_mmu) {
/* Default to hash mmu (if we can) */
prom_debug("Asking for hash\n");
@@ -3262,7 +3270,7 @@
/*
* Call the Ultravisor to transfer us to secure memory if we have an ESM blob.
*/
-static void setup_secure_guest(unsigned long kbase, unsigned long fdt)
+static void __init setup_secure_guest(unsigned long kbase, unsigned long fdt)
{
int ret;
@@ -3292,7 +3300,7 @@
}
}
#else
-static void setup_secure_guest(unsigned long kbase, unsigned long fdt)
+static void __init setup_secure_guest(unsigned long kbase, unsigned long fdt)
{
}
#endif /* CONFIG_PPC_SVM */
diff --git a/arch/powerpc/kernel/ptrace/ptrace-altivec.c b/arch/powerpc/kernel/ptrace/ptrace-altivec.c
index dd8b75d..0d9bc4b 100644
--- a/arch/powerpc/kernel/ptrace/ptrace-altivec.c
+++ b/arch/powerpc/kernel/ptrace/ptrace-altivec.c
@@ -41,38 +41,25 @@
* };
*/
int vr_get(struct task_struct *target, const struct user_regset *regset,
- unsigned int pos, unsigned int count, void *kbuf, void __user *ubuf)
+ struct membuf to)
{
- int ret;
+ union {
+ elf_vrreg_t reg;
+ u32 word;
+ } vrsave;
flush_altivec_to_thread(target);
BUILD_BUG_ON(offsetof(struct thread_vr_state, vscr) !=
offsetof(struct thread_vr_state, vr[32]));
- ret = user_regset_copyout(&pos, &count, &kbuf, &ubuf,
- &target->thread.vr_state, 0,
- 33 * sizeof(vector128));
- if (!ret) {
- /*
- * Copy out only the low-order word of vrsave.
- */
- int start, end;
- union {
- elf_vrreg_t reg;
- u32 word;
- } vrsave;
- memset(&vrsave, 0, sizeof(vrsave));
-
- vrsave.word = target->thread.vrsave;
-
- start = 33 * sizeof(vector128);
- end = start + sizeof(vrsave);
- ret = user_regset_copyout(&pos, &count, &kbuf, &ubuf, &vrsave,
- start, end);
- }
-
- return ret;
+ membuf_write(&to, &target->thread.vr_state, 33 * sizeof(vector128));
+ /*
+ * Copy out only the low-order word of vrsave.
+ */
+ memset(&vrsave, 0, sizeof(vrsave));
+ vrsave.word = target->thread.vrsave;
+ return membuf_write(&to, &vrsave, sizeof(vrsave));
}
/*
diff --git a/arch/powerpc/kernel/ptrace/ptrace-decl.h b/arch/powerpc/kernel/ptrace/ptrace-decl.h
index 3c8a819..67447a6 100644
--- a/arch/powerpc/kernel/ptrace/ptrace-decl.h
+++ b/arch/powerpc/kernel/ptrace/ptrace-decl.h
@@ -63,8 +63,7 @@
/* ptrace-(no)vsx */
-int fpr_get(struct task_struct *target, const struct user_regset *regset,
- unsigned int pos, unsigned int count, void *kbuf, void __user *ubuf);
+user_regset_get2_fn fpr_get;
int fpr_set(struct task_struct *target, const struct user_regset *regset,
unsigned int pos, unsigned int count,
const void *kbuf, const void __user *ubuf);
@@ -72,8 +71,7 @@
/* ptrace-vsx */
int vsr_active(struct task_struct *target, const struct user_regset *regset);
-int vsr_get(struct task_struct *target, const struct user_regset *regset,
- unsigned int pos, unsigned int count, void *kbuf, void __user *ubuf);
+user_regset_get2_fn vsr_get;
int vsr_set(struct task_struct *target, const struct user_regset *regset,
unsigned int pos, unsigned int count,
const void *kbuf, const void __user *ubuf);
@@ -81,8 +79,7 @@
/* ptrace-altivec */
int vr_active(struct task_struct *target, const struct user_regset *regset);
-int vr_get(struct task_struct *target, const struct user_regset *regset,
- unsigned int pos, unsigned int count, void *kbuf, void __user *ubuf);
+user_regset_get2_fn vr_get;
int vr_set(struct task_struct *target, const struct user_regset *regset,
unsigned int pos, unsigned int count,
const void *kbuf, const void __user *ubuf);
@@ -90,8 +87,7 @@
/* ptrace-spe */
int evr_active(struct task_struct *target, const struct user_regset *regset);
-int evr_get(struct task_struct *target, const struct user_regset *regset,
- unsigned int pos, unsigned int count, void *kbuf, void __user *ubuf);
+user_regset_get2_fn evr_get;
int evr_set(struct task_struct *target, const struct user_regset *regset,
unsigned int pos, unsigned int count,
const void *kbuf, const void __user *ubuf);
@@ -100,9 +96,8 @@
int gpr32_get_common(struct task_struct *target,
const struct user_regset *regset,
- unsigned int pos, unsigned int count,
- void *kbuf, void __user *ubuf,
- unsigned long *regs);
+ struct membuf to,
+ unsigned long *regs);
int gpr32_set_common(struct task_struct *target,
const struct user_regset *regset,
unsigned int pos, unsigned int count,
@@ -118,55 +113,46 @@
#endif
int tm_cgpr_active(struct task_struct *target, const struct user_regset *regset);
-int tm_cgpr_get(struct task_struct *target, const struct user_regset *regset,
- unsigned int pos, unsigned int count, void *kbuf, void __user *ubuf);
+user_regset_get2_fn tm_cgpr_get;
int tm_cgpr_set(struct task_struct *target, const struct user_regset *regset,
unsigned int pos, unsigned int count,
const void *kbuf, const void __user *ubuf);
int tm_cfpr_active(struct task_struct *target, const struct user_regset *regset);
-int tm_cfpr_get(struct task_struct *target, const struct user_regset *regset,
- unsigned int pos, unsigned int count, void *kbuf, void __user *ubuf);
+user_regset_get2_fn tm_cfpr_get;
int tm_cfpr_set(struct task_struct *target, const struct user_regset *regset,
unsigned int pos, unsigned int count,
const void *kbuf, const void __user *ubuf);
int tm_cvmx_active(struct task_struct *target, const struct user_regset *regset);
-int tm_cvmx_get(struct task_struct *target, const struct user_regset *regset,
- unsigned int pos, unsigned int count, void *kbuf, void __user *ubuf);
+user_regset_get2_fn tm_cvmx_get;
int tm_cvmx_set(struct task_struct *target, const struct user_regset *regset,
unsigned int pos, unsigned int count,
const void *kbuf, const void __user *ubuf);
int tm_cvsx_active(struct task_struct *target, const struct user_regset *regset);
-int tm_cvsx_get(struct task_struct *target, const struct user_regset *regset,
- unsigned int pos, unsigned int count, void *kbuf, void __user *ubuf);
+user_regset_get2_fn tm_cvsx_get;
int tm_cvsx_set(struct task_struct *target, const struct user_regset *regset,
unsigned int pos, unsigned int count,
const void *kbuf, const void __user *ubuf);
int tm_spr_active(struct task_struct *target, const struct user_regset *regset);
-int tm_spr_get(struct task_struct *target, const struct user_regset *regset,
- unsigned int pos, unsigned int count, void *kbuf, void __user *ubuf);
+user_regset_get2_fn tm_spr_get;
int tm_spr_set(struct task_struct *target, const struct user_regset *regset,
unsigned int pos, unsigned int count,
const void *kbuf, const void __user *ubuf);
int tm_tar_active(struct task_struct *target, const struct user_regset *regset);
-int tm_tar_get(struct task_struct *target, const struct user_regset *regset,
- unsigned int pos, unsigned int count, void *kbuf, void __user *ubuf);
+user_regset_get2_fn tm_tar_get;
int tm_tar_set(struct task_struct *target, const struct user_regset *regset,
unsigned int pos, unsigned int count,
const void *kbuf, const void __user *ubuf);
int tm_ppr_active(struct task_struct *target, const struct user_regset *regset);
-int tm_ppr_get(struct task_struct *target, const struct user_regset *regset,
- unsigned int pos, unsigned int count, void *kbuf, void __user *ubuf);
+user_regset_get2_fn tm_ppr_get;
int tm_ppr_set(struct task_struct *target, const struct user_regset *regset,
unsigned int pos, unsigned int count,
const void *kbuf, const void __user *ubuf);
int tm_dscr_active(struct task_struct *target, const struct user_regset *regset);
-int tm_dscr_get(struct task_struct *target, const struct user_regset *regset,
- unsigned int pos, unsigned int count, void *kbuf, void __user *ubuf);
+user_regset_get2_fn tm_dscr_get;
int tm_dscr_set(struct task_struct *target, const struct user_regset *regset,
unsigned int pos, unsigned int count,
const void *kbuf, const void __user *ubuf);
-int tm_cgpr32_get(struct task_struct *target, const struct user_regset *regset,
- unsigned int pos, unsigned int count, void *kbuf, void __user *ubuf);
+user_regset_get2_fn tm_cgpr32_get;
int tm_cgpr32_set(struct task_struct *target, const struct user_regset *regset,
unsigned int pos, unsigned int count,
const void *kbuf, const void __user *ubuf);
diff --git a/arch/powerpc/kernel/ptrace/ptrace-novsx.c b/arch/powerpc/kernel/ptrace/ptrace-novsx.c
index b2dc4e9..b3b3683 100644
--- a/arch/powerpc/kernel/ptrace/ptrace-novsx.c
+++ b/arch/powerpc/kernel/ptrace/ptrace-novsx.c
@@ -19,15 +19,14 @@
* };
*/
int fpr_get(struct task_struct *target, const struct user_regset *regset,
- unsigned int pos, unsigned int count, void *kbuf, void __user *ubuf)
+ struct membuf to)
{
BUILD_BUG_ON(offsetof(struct thread_fp_state, fpscr) !=
offsetof(struct thread_fp_state, fpr[32]));
flush_fp_to_thread(target);
- return user_regset_copyout(&pos, &count, &kbuf, &ubuf,
- &target->thread.fp_state, 0, -1);
+ return membuf_write(&to, &target->thread.fp_state, 33 * sizeof(u64));
}
/*
diff --git a/arch/powerpc/kernel/ptrace/ptrace-spe.c b/arch/powerpc/kernel/ptrace/ptrace-spe.c
index 68b86b4..47034d0 100644
--- a/arch/powerpc/kernel/ptrace/ptrace-spe.c
+++ b/arch/powerpc/kernel/ptrace/ptrace-spe.c
@@ -23,25 +23,17 @@
}
int evr_get(struct task_struct *target, const struct user_regset *regset,
- unsigned int pos, unsigned int count, void *kbuf, void __user *ubuf)
+ struct membuf to)
{
- int ret;
-
flush_spe_to_thread(target);
- ret = user_regset_copyout(&pos, &count, &kbuf, &ubuf,
- &target->thread.evr,
- 0, sizeof(target->thread.evr));
+ membuf_write(&to, &target->thread.evr, sizeof(target->thread.evr));
BUILD_BUG_ON(offsetof(struct thread_struct, acc) + sizeof(u64) !=
offsetof(struct thread_struct, spefscr));
- if (!ret)
- ret = user_regset_copyout(&pos, &count, &kbuf, &ubuf,
- &target->thread.acc,
- sizeof(target->thread.evr), -1);
-
- return ret;
+ return membuf_write(&to, &target->thread.acc,
+ sizeof(u64) + sizeof(u32));
}
int evr_set(struct task_struct *target, const struct user_regset *regset,
diff --git a/arch/powerpc/kernel/ptrace/ptrace-tm.c b/arch/powerpc/kernel/ptrace/ptrace-tm.c
index 32d62c6..54f2d07 100644
--- a/arch/powerpc/kernel/ptrace/ptrace-tm.c
+++ b/arch/powerpc/kernel/ptrace/ptrace-tm.c
@@ -70,10 +70,7 @@
* tm_cgpr_get - get CGPR registers
* @target: The target task.
* @regset: The user regset structure.
- * @pos: The buffer position.
- * @count: Number of bytes to copy.
- * @kbuf: Kernel buffer to copy from.
- * @ubuf: User buffer to copy into.
+ * @to: Destination of copy.
*
* This function gets transaction checkpointed GPR registers.
*
@@ -87,10 +84,8 @@
* };
*/
int tm_cgpr_get(struct task_struct *target, const struct user_regset *regset,
- unsigned int pos, unsigned int count, void *kbuf, void __user *ubuf)
+ struct membuf to)
{
- int ret;
-
if (!cpu_has_feature(CPU_FTR_TM))
return -ENODEV;
@@ -101,31 +96,18 @@
flush_fp_to_thread(target);
flush_altivec_to_thread(target);
- ret = user_regset_copyout(&pos, &count, &kbuf, &ubuf,
- &target->thread.ckpt_regs,
- 0, offsetof(struct pt_regs, msr));
- if (!ret) {
- unsigned long msr = get_user_ckpt_msr(target);
-
- ret = user_regset_copyout(&pos, &count, &kbuf, &ubuf, &msr,
- offsetof(struct pt_regs, msr),
- offsetof(struct pt_regs, msr) +
- sizeof(msr));
- }
+ membuf_write(&to, &target->thread.ckpt_regs,
+ offsetof(struct pt_regs, msr));
+ membuf_store(&to, get_user_ckpt_msr(target));
BUILD_BUG_ON(offsetof(struct pt_regs, orig_gpr3) !=
offsetof(struct pt_regs, msr) + sizeof(long));
- if (!ret)
- ret = user_regset_copyout(&pos, &count, &kbuf, &ubuf,
- &target->thread.ckpt_regs.orig_gpr3,
- offsetof(struct pt_regs, orig_gpr3),
- sizeof(struct user_pt_regs));
- if (!ret)
- ret = user_regset_copyout_zero(&pos, &count, &kbuf, &ubuf,
- sizeof(struct user_pt_regs), -1);
-
- return ret;
+ membuf_write(&to, &target->thread.ckpt_regs.orig_gpr3,
+ sizeof(struct user_pt_regs) -
+ offsetof(struct pt_regs, orig_gpr3));
+ return membuf_zero(&to, ELF_NGREG * sizeof(unsigned long) -
+ sizeof(struct user_pt_regs));
}
/*
@@ -229,10 +211,7 @@
* tm_cfpr_get - get CFPR registers
* @target: The target task.
* @regset: The user regset structure.
- * @pos: The buffer position.
- * @count: Number of bytes to copy.
- * @kbuf: Kernel buffer to copy from.
- * @ubuf: User buffer to copy into.
+ * @to: Destination of copy.
*
* This function gets in transaction checkpointed FPR registers.
*
@@ -247,7 +226,7 @@
*};
*/
int tm_cfpr_get(struct task_struct *target, const struct user_regset *regset,
- unsigned int pos, unsigned int count, void *kbuf, void __user *ubuf)
+ struct membuf to)
{
u64 buf[33];
int i;
@@ -266,7 +245,7 @@
for (i = 0; i < 32 ; i++)
buf[i] = target->thread.TS_CKFPR(i);
buf[32] = target->thread.ckfp_state.fpscr;
- return user_regset_copyout(&pos, &count, &kbuf, &ubuf, buf, 0, -1);
+ return membuf_write(&to, buf, sizeof(buf));
}
/**
@@ -344,10 +323,7 @@
* tm_cvmx_get - get CMVX registers
* @target: The target task.
* @regset: The user regset structure.
- * @pos: The buffer position.
- * @count: Number of bytes to copy.
- * @kbuf: Kernel buffer to copy from.
- * @ubuf: User buffer to copy into.
+ * @to: Destination of copy.
*
* This function gets in transaction checkpointed VMX registers.
*
@@ -363,10 +339,12 @@
*};
*/
int tm_cvmx_get(struct task_struct *target, const struct user_regset *regset,
- unsigned int pos, unsigned int count, void *kbuf, void __user *ubuf)
+ struct membuf to)
{
- int ret;
-
+ union {
+ elf_vrreg_t reg;
+ u32 word;
+ } vrsave;
BUILD_BUG_ON(TVSO(vscr) != TVSO(vr[32]));
if (!cpu_has_feature(CPU_FTR_TM))
@@ -380,23 +358,13 @@
flush_fp_to_thread(target);
flush_altivec_to_thread(target);
- ret = user_regset_copyout(&pos, &count, &kbuf, &ubuf, &target->thread.ckvr_state,
- 0, 33 * sizeof(vector128));
- if (!ret) {
- /*
- * Copy out only the low-order word of vrsave.
- */
- union {
- elf_vrreg_t reg;
- u32 word;
- } vrsave;
- memset(&vrsave, 0, sizeof(vrsave));
- vrsave.word = target->thread.ckvrsave;
- ret = user_regset_copyout(&pos, &count, &kbuf, &ubuf, &vrsave,
- 33 * sizeof(vector128), -1);
- }
-
- return ret;
+ membuf_write(&to, &target->thread.ckvr_state, 33 * sizeof(vector128));
+ /*
+ * Copy out only the low-order word of vrsave.
+ */
+ memset(&vrsave, 0, sizeof(vrsave));
+ vrsave.word = target->thread.ckvrsave;
+ return membuf_write(&to, &vrsave, sizeof(vrsave));
}
/**
@@ -484,10 +452,7 @@
* tm_cvsx_get - get CVSX registers
* @target: The target task.
* @regset: The user regset structure.
- * @pos: The buffer position.
- * @count: Number of bytes to copy.
- * @kbuf: Kernel buffer to copy from.
- * @ubuf: User buffer to copy into.
+ * @to: Destination of copy.
*
* This function gets in transaction checkpointed VSX registers.
*
@@ -501,10 +466,10 @@
*};
*/
int tm_cvsx_get(struct task_struct *target, const struct user_regset *regset,
- unsigned int pos, unsigned int count, void *kbuf, void __user *ubuf)
+ struct membuf to)
{
u64 buf[32];
- int ret, i;
+ int i;
if (!cpu_has_feature(CPU_FTR_TM))
return -ENODEV;
@@ -520,10 +485,7 @@
for (i = 0; i < 32 ; i++)
buf[i] = target->thread.ckfp_state.fpr[i][TS_VSRLOWOFFSET];
- ret = user_regset_copyout(&pos, &count, &kbuf, &ubuf,
- buf, 0, 32 * sizeof(double));
-
- return ret;
+ return membuf_write(&to, buf, 32 * sizeof(double));
}
/**
@@ -597,10 +559,7 @@
* tm_spr_get - get the TM related SPR registers
* @target: The target task.
* @regset: The user regset structure.
- * @pos: The buffer position.
- * @count: Number of bytes to copy.
- * @kbuf: Kernel buffer to copy from.
- * @ubuf: User buffer to copy into.
+ * @to: Destination of copy.
*
* This function gets transactional memory related SPR registers.
* The userspace interface buffer layout is as follows.
@@ -612,10 +571,8 @@
* };
*/
int tm_spr_get(struct task_struct *target, const struct user_regset *regset,
- unsigned int pos, unsigned int count, void *kbuf, void __user *ubuf)
+ struct membuf to)
{
- int ret;
-
/* Build tests */
BUILD_BUG_ON(TSO(tm_tfhar) + sizeof(u64) != TSO(tm_texasr));
BUILD_BUG_ON(TSO(tm_texasr) + sizeof(u64) != TSO(tm_tfiar));
@@ -630,21 +587,11 @@
flush_altivec_to_thread(target);
/* TFHAR register */
- ret = user_regset_copyout(&pos, &count, &kbuf, &ubuf,
- &target->thread.tm_tfhar, 0, sizeof(u64));
-
+ membuf_write(&to, &target->thread.tm_tfhar, sizeof(u64));
/* TEXASR register */
- if (!ret)
- ret = user_regset_copyout(&pos, &count, &kbuf, &ubuf,
- &target->thread.tm_texasr, sizeof(u64),
- 2 * sizeof(u64));
-
+ membuf_write(&to, &target->thread.tm_texasr, sizeof(u64));
/* TFIAR register */
- if (!ret)
- ret = user_regset_copyout(&pos, &count, &kbuf, &ubuf,
- &target->thread.tm_tfiar,
- 2 * sizeof(u64), 3 * sizeof(u64));
- return ret;
+ return membuf_write(&to, &target->thread.tm_tfiar, sizeof(u64));
}
/**
@@ -714,19 +661,15 @@
}
int tm_tar_get(struct task_struct *target, const struct user_regset *regset,
- unsigned int pos, unsigned int count, void *kbuf, void __user *ubuf)
+ struct membuf to)
{
- int ret;
-
if (!cpu_has_feature(CPU_FTR_TM))
return -ENODEV;
if (!MSR_TM_ACTIVE(target->thread.regs->msr))
return -ENODATA;
- ret = user_regset_copyout(&pos, &count, &kbuf, &ubuf,
- &target->thread.tm_tar, 0, sizeof(u64));
- return ret;
+ return membuf_write(&to, &target->thread.tm_tar, sizeof(u64));
}
int tm_tar_set(struct task_struct *target, const struct user_regset *regset,
@@ -759,19 +702,15 @@
int tm_ppr_get(struct task_struct *target, const struct user_regset *regset,
- unsigned int pos, unsigned int count, void *kbuf, void __user *ubuf)
+ struct membuf to)
{
- int ret;
-
if (!cpu_has_feature(CPU_FTR_TM))
return -ENODEV;
if (!MSR_TM_ACTIVE(target->thread.regs->msr))
return -ENODATA;
- ret = user_regset_copyout(&pos, &count, &kbuf, &ubuf,
- &target->thread.tm_ppr, 0, sizeof(u64));
- return ret;
+ return membuf_write(&to, &target->thread.tm_ppr, sizeof(u64));
}
int tm_ppr_set(struct task_struct *target, const struct user_regset *regset,
@@ -803,19 +742,15 @@
}
int tm_dscr_get(struct task_struct *target, const struct user_regset *regset,
- unsigned int pos, unsigned int count, void *kbuf, void __user *ubuf)
+ struct membuf to)
{
- int ret;
-
if (!cpu_has_feature(CPU_FTR_TM))
return -ENODEV;
if (!MSR_TM_ACTIVE(target->thread.regs->msr))
return -ENODATA;
- ret = user_regset_copyout(&pos, &count, &kbuf, &ubuf,
- &target->thread.tm_dscr, 0, sizeof(u64));
- return ret;
+ return membuf_write(&to, &target->thread.tm_dscr, sizeof(u64));
}
int tm_dscr_set(struct task_struct *target, const struct user_regset *regset,
@@ -836,10 +771,11 @@
}
int tm_cgpr32_get(struct task_struct *target, const struct user_regset *regset,
- unsigned int pos, unsigned int count, void *kbuf, void __user *ubuf)
+ struct membuf to)
{
- return gpr32_get_common(target, regset, pos, count, kbuf, ubuf,
+ gpr32_get_common(target, regset, to,
&target->thread.ckpt_regs.gpr[0]);
+ return membuf_zero(&to, ELF_NGREG * sizeof(u32));
}
int tm_cgpr32_set(struct task_struct *target, const struct user_regset *regset,
diff --git a/arch/powerpc/kernel/ptrace/ptrace-view.c b/arch/powerpc/kernel/ptrace/ptrace-view.c
index caeb582..19823a2 100644
--- a/arch/powerpc/kernel/ptrace/ptrace-view.c
+++ b/arch/powerpc/kernel/ptrace/ptrace-view.c
@@ -215,9 +215,9 @@
}
static int gpr_get(struct task_struct *target, const struct user_regset *regset,
- unsigned int pos, unsigned int count, void *kbuf, void __user *ubuf)
+ struct membuf to)
{
- int i, ret;
+ int i;
if (target->thread.regs == NULL)
return -EIO;
@@ -228,30 +228,17 @@
target->thread.regs->gpr[i] = NV_REG_POISON;
}
- ret = user_regset_copyout(&pos, &count, &kbuf, &ubuf,
- target->thread.regs,
- 0, offsetof(struct pt_regs, msr));
- if (!ret) {
- unsigned long msr = get_user_msr(target);
- ret = user_regset_copyout(&pos, &count, &kbuf, &ubuf, &msr,
- offsetof(struct pt_regs, msr),
- offsetof(struct pt_regs, msr) +
- sizeof(msr));
- }
+ membuf_write(&to, target->thread.regs, offsetof(struct pt_regs, msr));
+ membuf_store(&to, get_user_msr(target));
BUILD_BUG_ON(offsetof(struct pt_regs, orig_gpr3) !=
offsetof(struct pt_regs, msr) + sizeof(long));
- if (!ret)
- ret = user_regset_copyout(&pos, &count, &kbuf, &ubuf,
- &target->thread.regs->orig_gpr3,
- offsetof(struct pt_regs, orig_gpr3),
- sizeof(struct user_pt_regs));
- if (!ret)
- ret = user_regset_copyout_zero(&pos, &count, &kbuf, &ubuf,
- sizeof(struct user_pt_regs), -1);
-
- return ret;
+ membuf_write(&to, &target->thread.regs->orig_gpr3,
+ sizeof(struct user_pt_regs) -
+ offsetof(struct pt_regs, orig_gpr3));
+ return membuf_zero(&to, ELF_NGREG * sizeof(unsigned long) -
+ sizeof(struct user_pt_regs));
}
static int gpr_set(struct task_struct *target, const struct user_regset *regset,
@@ -309,10 +296,9 @@
#ifdef CONFIG_PPC64
static int ppr_get(struct task_struct *target, const struct user_regset *regset,
- unsigned int pos, unsigned int count, void *kbuf, void __user *ubuf)
+ struct membuf to)
{
- return user_regset_copyout(&pos, &count, &kbuf, &ubuf,
- &target->thread.regs->ppr, 0, sizeof(u64));
+ return membuf_write(&to, &target->thread.regs->ppr, sizeof(u64));
}
static int ppr_set(struct task_struct *target, const struct user_regset *regset,
@@ -324,10 +310,9 @@
}
static int dscr_get(struct task_struct *target, const struct user_regset *regset,
- unsigned int pos, unsigned int count, void *kbuf, void __user *ubuf)
+ struct membuf to)
{
- return user_regset_copyout(&pos, &count, &kbuf, &ubuf,
- &target->thread.dscr, 0, sizeof(u64));
+ return membuf_write(&to, &target->thread.dscr, sizeof(u64));
}
static int dscr_set(struct task_struct *target, const struct user_regset *regset,
unsigned int pos, unsigned int count, const void *kbuf,
@@ -339,10 +324,9 @@
#endif
#ifdef CONFIG_PPC_BOOK3S_64
static int tar_get(struct task_struct *target, const struct user_regset *regset,
- unsigned int pos, unsigned int count, void *kbuf, void __user *ubuf)
+ struct membuf to)
{
- return user_regset_copyout(&pos, &count, &kbuf, &ubuf,
- &target->thread.tar, 0, sizeof(u64));
+ return membuf_write(&to, &target->thread.tar, sizeof(u64));
}
static int tar_set(struct task_struct *target, const struct user_regset *regset,
unsigned int pos, unsigned int count, const void *kbuf,
@@ -364,7 +348,7 @@
}
static int ebb_get(struct task_struct *target, const struct user_regset *regset,
- unsigned int pos, unsigned int count, void *kbuf, void __user *ubuf)
+ struct membuf to)
{
/* Build tests */
BUILD_BUG_ON(TSO(ebbrr) + sizeof(unsigned long) != TSO(ebbhr));
@@ -376,8 +360,7 @@
if (!target->thread.used_ebb)
return -ENODATA;
- return user_regset_copyout(&pos, &count, &kbuf, &ubuf, &target->thread.ebbrr,
- 0, 3 * sizeof(unsigned long));
+ return membuf_write(&to, &target->thread.ebbrr, 3 * sizeof(unsigned long));
}
static int ebb_set(struct task_struct *target, const struct user_regset *regset,
@@ -420,7 +403,7 @@
}
static int pmu_get(struct task_struct *target, const struct user_regset *regset,
- unsigned int pos, unsigned int count, void *kbuf, void __user *ubuf)
+ struct membuf to)
{
/* Build tests */
BUILD_BUG_ON(TSO(siar) + sizeof(unsigned long) != TSO(sdar));
@@ -431,8 +414,7 @@
if (!cpu_has_feature(CPU_FTR_ARCH_207S))
return -ENODEV;
- return user_regset_copyout(&pos, &count, &kbuf, &ubuf, &target->thread.siar,
- 0, 5 * sizeof(unsigned long));
+ return membuf_write(&to, &target->thread.siar, 5 * sizeof(unsigned long));
}
static int pmu_set(struct task_struct *target, const struct user_regset *regset,
@@ -486,16 +468,17 @@
}
static int pkey_get(struct task_struct *target, const struct user_regset *regset,
- unsigned int pos, unsigned int count, void *kbuf, void __user *ubuf)
+ struct membuf to)
{
+ int ret;
+
BUILD_BUG_ON(TSO(amr) + sizeof(unsigned long) != TSO(iamr));
- BUILD_BUG_ON(TSO(iamr) + sizeof(unsigned long) != TSO(uamor));
if (!arch_pkeys_enabled())
return -ENODEV;
- return user_regset_copyout(&pos, &count, &kbuf, &ubuf, &target->thread.amr,
- 0, ELF_NPKEY * sizeof(unsigned long));
+ membuf_write(&to, &target->thread.amr, 2 * sizeof(unsigned long));
+ return membuf_store(&to, default_uamor);
}
static int pkey_set(struct task_struct *target, const struct user_regset *regset,
@@ -517,9 +500,17 @@
if (ret)
return ret;
- /* UAMOR determines which bits of the AMR can be set from userspace. */
- target->thread.amr = (new_amr & target->thread.uamor) |
- (target->thread.amr & ~target->thread.uamor);
+ /*
+ * UAMOR determines which bits of the AMR can be set from userspace.
+ * UAMOR value 0b11 indicates that the AMR value can be modified
+ * from userspace. If the kernel is using a specific key, we avoid
+ * userspace modifying the AMR value for that key by masking them
+ * via UAMOR 0b00.
+ *
+ * Pick the AMR values for the keys that kernel is using. This
+ * will be indicated by the ~default_uamor bits.
+ */
+ target->thread.amr = (new_amr & default_uamor) | (target->thread.amr & ~default_uamor);
return 0;
}
@@ -529,110 +520,110 @@
[REGSET_GPR] = {
.core_note_type = NT_PRSTATUS, .n = ELF_NGREG,
.size = sizeof(long), .align = sizeof(long),
- .get = gpr_get, .set = gpr_set
+ .regset_get = gpr_get, .set = gpr_set
},
[REGSET_FPR] = {
.core_note_type = NT_PRFPREG, .n = ELF_NFPREG,
.size = sizeof(double), .align = sizeof(double),
- .get = fpr_get, .set = fpr_set
+ .regset_get = fpr_get, .set = fpr_set
},
#ifdef CONFIG_ALTIVEC
[REGSET_VMX] = {
.core_note_type = NT_PPC_VMX, .n = 34,
.size = sizeof(vector128), .align = sizeof(vector128),
- .active = vr_active, .get = vr_get, .set = vr_set
+ .active = vr_active, .regset_get = vr_get, .set = vr_set
},
#endif
#ifdef CONFIG_VSX
[REGSET_VSX] = {
.core_note_type = NT_PPC_VSX, .n = 32,
.size = sizeof(double), .align = sizeof(double),
- .active = vsr_active, .get = vsr_get, .set = vsr_set
+ .active = vsr_active, .regset_get = vsr_get, .set = vsr_set
},
#endif
#ifdef CONFIG_SPE
[REGSET_SPE] = {
.core_note_type = NT_PPC_SPE, .n = 35,
.size = sizeof(u32), .align = sizeof(u32),
- .active = evr_active, .get = evr_get, .set = evr_set
+ .active = evr_active, .regset_get = evr_get, .set = evr_set
},
#endif
#ifdef CONFIG_PPC_TRANSACTIONAL_MEM
[REGSET_TM_CGPR] = {
.core_note_type = NT_PPC_TM_CGPR, .n = ELF_NGREG,
.size = sizeof(long), .align = sizeof(long),
- .active = tm_cgpr_active, .get = tm_cgpr_get, .set = tm_cgpr_set
+ .active = tm_cgpr_active, .regset_get = tm_cgpr_get, .set = tm_cgpr_set
},
[REGSET_TM_CFPR] = {
.core_note_type = NT_PPC_TM_CFPR, .n = ELF_NFPREG,
.size = sizeof(double), .align = sizeof(double),
- .active = tm_cfpr_active, .get = tm_cfpr_get, .set = tm_cfpr_set
+ .active = tm_cfpr_active, .regset_get = tm_cfpr_get, .set = tm_cfpr_set
},
[REGSET_TM_CVMX] = {
.core_note_type = NT_PPC_TM_CVMX, .n = ELF_NVMX,
.size = sizeof(vector128), .align = sizeof(vector128),
- .active = tm_cvmx_active, .get = tm_cvmx_get, .set = tm_cvmx_set
+ .active = tm_cvmx_active, .regset_get = tm_cvmx_get, .set = tm_cvmx_set
},
[REGSET_TM_CVSX] = {
.core_note_type = NT_PPC_TM_CVSX, .n = ELF_NVSX,
.size = sizeof(double), .align = sizeof(double),
- .active = tm_cvsx_active, .get = tm_cvsx_get, .set = tm_cvsx_set
+ .active = tm_cvsx_active, .regset_get = tm_cvsx_get, .set = tm_cvsx_set
},
[REGSET_TM_SPR] = {
.core_note_type = NT_PPC_TM_SPR, .n = ELF_NTMSPRREG,
.size = sizeof(u64), .align = sizeof(u64),
- .active = tm_spr_active, .get = tm_spr_get, .set = tm_spr_set
+ .active = tm_spr_active, .regset_get = tm_spr_get, .set = tm_spr_set
},
[REGSET_TM_CTAR] = {
.core_note_type = NT_PPC_TM_CTAR, .n = 1,
.size = sizeof(u64), .align = sizeof(u64),
- .active = tm_tar_active, .get = tm_tar_get, .set = tm_tar_set
+ .active = tm_tar_active, .regset_get = tm_tar_get, .set = tm_tar_set
},
[REGSET_TM_CPPR] = {
.core_note_type = NT_PPC_TM_CPPR, .n = 1,
.size = sizeof(u64), .align = sizeof(u64),
- .active = tm_ppr_active, .get = tm_ppr_get, .set = tm_ppr_set
+ .active = tm_ppr_active, .regset_get = tm_ppr_get, .set = tm_ppr_set
},
[REGSET_TM_CDSCR] = {
.core_note_type = NT_PPC_TM_CDSCR, .n = 1,
.size = sizeof(u64), .align = sizeof(u64),
- .active = tm_dscr_active, .get = tm_dscr_get, .set = tm_dscr_set
+ .active = tm_dscr_active, .regset_get = tm_dscr_get, .set = tm_dscr_set
},
#endif
#ifdef CONFIG_PPC64
[REGSET_PPR] = {
.core_note_type = NT_PPC_PPR, .n = 1,
.size = sizeof(u64), .align = sizeof(u64),
- .get = ppr_get, .set = ppr_set
+ .regset_get = ppr_get, .set = ppr_set
},
[REGSET_DSCR] = {
.core_note_type = NT_PPC_DSCR, .n = 1,
.size = sizeof(u64), .align = sizeof(u64),
- .get = dscr_get, .set = dscr_set
+ .regset_get = dscr_get, .set = dscr_set
},
#endif
#ifdef CONFIG_PPC_BOOK3S_64
[REGSET_TAR] = {
.core_note_type = NT_PPC_TAR, .n = 1,
.size = sizeof(u64), .align = sizeof(u64),
- .get = tar_get, .set = tar_set
+ .regset_get = tar_get, .set = tar_set
},
[REGSET_EBB] = {
.core_note_type = NT_PPC_EBB, .n = ELF_NEBB,
.size = sizeof(u64), .align = sizeof(u64),
- .active = ebb_active, .get = ebb_get, .set = ebb_set
+ .active = ebb_active, .regset_get = ebb_get, .set = ebb_set
},
[REGSET_PMR] = {
.core_note_type = NT_PPC_PMU, .n = ELF_NPMU,
.size = sizeof(u64), .align = sizeof(u64),
- .active = pmu_active, .get = pmu_get, .set = pmu_set
+ .active = pmu_active, .regset_get = pmu_get, .set = pmu_set
},
#endif
#ifdef CONFIG_PPC_MEM_KEYS
[REGSET_PKEY] = {
.core_note_type = NT_PPC_PKEY, .n = ELF_NPKEY,
.size = sizeof(u64), .align = sizeof(u64),
- .active = pkey_active, .get = pkey_get, .set = pkey_set
+ .active = pkey_active, .regset_get = pkey_get, .set = pkey_set
},
#endif
};
@@ -646,49 +637,16 @@
int gpr32_get_common(struct task_struct *target,
const struct user_regset *regset,
- unsigned int pos, unsigned int count,
- void *kbuf, void __user *ubuf,
- unsigned long *regs)
+ struct membuf to, unsigned long *regs)
{
- compat_ulong_t *k = kbuf;
- compat_ulong_t __user *u = ubuf;
- compat_ulong_t reg;
+ int i;
- pos /= sizeof(reg);
- count /= sizeof(reg);
-
- if (kbuf)
- for (; count > 0 && pos < PT_MSR; --count)
- *k++ = regs[pos++];
- else
- for (; count > 0 && pos < PT_MSR; --count)
- if (__put_user((compat_ulong_t)regs[pos++], u++))
- return -EFAULT;
-
- if (count > 0 && pos == PT_MSR) {
- reg = get_user_msr(target);
- if (kbuf)
- *k++ = reg;
- else if (__put_user(reg, u++))
- return -EFAULT;
- ++pos;
- --count;
- }
-
- if (kbuf)
- for (; count > 0 && pos < PT_REGS_COUNT; --count)
- *k++ = regs[pos++];
- else
- for (; count > 0 && pos < PT_REGS_COUNT; --count)
- if (__put_user((compat_ulong_t)regs[pos++], u++))
- return -EFAULT;
-
- kbuf = k;
- ubuf = u;
- pos *= sizeof(reg);
- count *= sizeof(reg);
- return user_regset_copyout_zero(&pos, &count, &kbuf, &ubuf,
- PT_REGS_COUNT * sizeof(reg), -1);
+ for (i = 0; i < PT_MSR; i++)
+ membuf_store(&to, (u32)regs[i]);
+ membuf_store(&to, (u32)get_user_msr(target));
+ for (i++ ; i < PT_REGS_COUNT; i++)
+ membuf_store(&to, (u32)regs[i]);
+ return membuf_zero(&to, (ELF_NGREG - PT_REGS_COUNT) * sizeof(u32));
}
int gpr32_set_common(struct task_struct *target,
@@ -761,8 +719,7 @@
static int gpr32_get(struct task_struct *target,
const struct user_regset *regset,
- unsigned int pos, unsigned int count,
- void *kbuf, void __user *ubuf)
+ struct membuf to)
{
int i;
@@ -777,7 +734,7 @@
for (i = 14; i < 32; i++)
target->thread.regs->gpr[i] = NV_REG_POISON;
}
- return gpr32_get_common(target, regset, pos, count, kbuf, ubuf,
+ return gpr32_get_common(target, regset, to,
&target->thread.regs->gpr[0]);
}
@@ -801,25 +758,25 @@
[REGSET_GPR] = {
.core_note_type = NT_PRSTATUS, .n = ELF_NGREG,
.size = sizeof(compat_long_t), .align = sizeof(compat_long_t),
- .get = gpr32_get, .set = gpr32_set
+ .regset_get = gpr32_get, .set = gpr32_set
},
[REGSET_FPR] = {
.core_note_type = NT_PRFPREG, .n = ELF_NFPREG,
.size = sizeof(double), .align = sizeof(double),
- .get = fpr_get, .set = fpr_set
+ .regset_get = fpr_get, .set = fpr_set
},
#ifdef CONFIG_ALTIVEC
[REGSET_VMX] = {
.core_note_type = NT_PPC_VMX, .n = 34,
.size = sizeof(vector128), .align = sizeof(vector128),
- .active = vr_active, .get = vr_get, .set = vr_set
+ .active = vr_active, .regset_get = vr_get, .set = vr_set
},
#endif
#ifdef CONFIG_SPE
[REGSET_SPE] = {
.core_note_type = NT_PPC_SPE, .n = 35,
.size = sizeof(u32), .align = sizeof(u32),
- .active = evr_active, .get = evr_get, .set = evr_set
+ .active = evr_active, .regset_get = evr_get, .set = evr_set
},
#endif
#ifdef CONFIG_PPC_TRANSACTIONAL_MEM
@@ -827,66 +784,66 @@
.core_note_type = NT_PPC_TM_CGPR, .n = ELF_NGREG,
.size = sizeof(long), .align = sizeof(long),
.active = tm_cgpr_active,
- .get = tm_cgpr32_get, .set = tm_cgpr32_set
+ .regset_get = tm_cgpr32_get, .set = tm_cgpr32_set
},
[REGSET_TM_CFPR] = {
.core_note_type = NT_PPC_TM_CFPR, .n = ELF_NFPREG,
.size = sizeof(double), .align = sizeof(double),
- .active = tm_cfpr_active, .get = tm_cfpr_get, .set = tm_cfpr_set
+ .active = tm_cfpr_active, .regset_get = tm_cfpr_get, .set = tm_cfpr_set
},
[REGSET_TM_CVMX] = {
.core_note_type = NT_PPC_TM_CVMX, .n = ELF_NVMX,
.size = sizeof(vector128), .align = sizeof(vector128),
- .active = tm_cvmx_active, .get = tm_cvmx_get, .set = tm_cvmx_set
+ .active = tm_cvmx_active, .regset_get = tm_cvmx_get, .set = tm_cvmx_set
},
[REGSET_TM_CVSX] = {
.core_note_type = NT_PPC_TM_CVSX, .n = ELF_NVSX,
.size = sizeof(double), .align = sizeof(double),
- .active = tm_cvsx_active, .get = tm_cvsx_get, .set = tm_cvsx_set
+ .active = tm_cvsx_active, .regset_get = tm_cvsx_get, .set = tm_cvsx_set
},
[REGSET_TM_SPR] = {
.core_note_type = NT_PPC_TM_SPR, .n = ELF_NTMSPRREG,
.size = sizeof(u64), .align = sizeof(u64),
- .active = tm_spr_active, .get = tm_spr_get, .set = tm_spr_set
+ .active = tm_spr_active, .regset_get = tm_spr_get, .set = tm_spr_set
},
[REGSET_TM_CTAR] = {
.core_note_type = NT_PPC_TM_CTAR, .n = 1,
.size = sizeof(u64), .align = sizeof(u64),
- .active = tm_tar_active, .get = tm_tar_get, .set = tm_tar_set
+ .active = tm_tar_active, .regset_get = tm_tar_get, .set = tm_tar_set
},
[REGSET_TM_CPPR] = {
.core_note_type = NT_PPC_TM_CPPR, .n = 1,
.size = sizeof(u64), .align = sizeof(u64),
- .active = tm_ppr_active, .get = tm_ppr_get, .set = tm_ppr_set
+ .active = tm_ppr_active, .regset_get = tm_ppr_get, .set = tm_ppr_set
},
[REGSET_TM_CDSCR] = {
.core_note_type = NT_PPC_TM_CDSCR, .n = 1,
.size = sizeof(u64), .align = sizeof(u64),
- .active = tm_dscr_active, .get = tm_dscr_get, .set = tm_dscr_set
+ .active = tm_dscr_active, .regset_get = tm_dscr_get, .set = tm_dscr_set
},
#endif
#ifdef CONFIG_PPC64
[REGSET_PPR] = {
.core_note_type = NT_PPC_PPR, .n = 1,
.size = sizeof(u64), .align = sizeof(u64),
- .get = ppr_get, .set = ppr_set
+ .regset_get = ppr_get, .set = ppr_set
},
[REGSET_DSCR] = {
.core_note_type = NT_PPC_DSCR, .n = 1,
.size = sizeof(u64), .align = sizeof(u64),
- .get = dscr_get, .set = dscr_set
+ .regset_get = dscr_get, .set = dscr_set
},
#endif
#ifdef CONFIG_PPC_BOOK3S_64
[REGSET_TAR] = {
.core_note_type = NT_PPC_TAR, .n = 1,
.size = sizeof(u64), .align = sizeof(u64),
- .get = tar_get, .set = tar_set
+ .regset_get = tar_get, .set = tar_set
},
[REGSET_EBB] = {
.core_note_type = NT_PPC_EBB, .n = ELF_NEBB,
.size = sizeof(u64), .align = sizeof(u64),
- .active = ebb_active, .get = ebb_get, .set = ebb_set
+ .active = ebb_active, .regset_get = ebb_get, .set = ebb_set
},
#endif
};
diff --git a/arch/powerpc/kernel/ptrace/ptrace-vsx.c b/arch/powerpc/kernel/ptrace/ptrace-vsx.c
index d53466d..1da4303 100644
--- a/arch/powerpc/kernel/ptrace/ptrace-vsx.c
+++ b/arch/powerpc/kernel/ptrace/ptrace-vsx.c
@@ -19,7 +19,7 @@
* };
*/
int fpr_get(struct task_struct *target, const struct user_regset *regset,
- unsigned int pos, unsigned int count, void *kbuf, void __user *ubuf)
+ struct membuf to)
{
u64 buf[33];
int i;
@@ -30,7 +30,7 @@
for (i = 0; i < 32 ; i++)
buf[i] = target->thread.TS_FPR(i);
buf[32] = target->thread.fp_state.fpscr;
- return user_regset_copyout(&pos, &count, &kbuf, &ubuf, buf, 0, -1);
+ return membuf_write(&to, buf, 33 * sizeof(u64));
}
/*
@@ -95,10 +95,10 @@
* };
*/
int vsr_get(struct task_struct *target, const struct user_regset *regset,
- unsigned int pos, unsigned int count, void *kbuf, void __user *ubuf)
+ struct membuf to)
{
u64 buf[32];
- int ret, i;
+ int i;
flush_tmregs_to_thread(target);
flush_fp_to_thread(target);
@@ -108,10 +108,7 @@
for (i = 0; i < 32 ; i++)
buf[i] = target->thread.fp_state.fpr[i][TS_VSRLOWOFFSET];
- ret = user_regset_copyout(&pos, &count, &kbuf, &ubuf,
- buf, 0, 32 * sizeof(double));
-
- return ret;
+ return membuf_write(&to, buf, 32 * sizeof(double));
}
/*
diff --git a/arch/powerpc/kernel/rtas.c b/arch/powerpc/kernel/rtas.c
index a09eba0..806d554 100644
--- a/arch/powerpc/kernel/rtas.c
+++ b/arch/powerpc/kernel/rtas.c
@@ -843,96 +843,6 @@
__rtas_suspend_cpu((struct rtas_suspend_me_data *)info, 1);
}
-enum rtas_cpu_state {
- DOWN,
- UP,
-};
-
-#ifndef CONFIG_SMP
-static int rtas_cpu_state_change_mask(enum rtas_cpu_state state,
- cpumask_var_t cpus)
-{
- if (!cpumask_empty(cpus)) {
- cpumask_clear(cpus);
- return -EINVAL;
- } else
- return 0;
-}
-#else
-/* On return cpumask will be altered to indicate CPUs changed.
- * CPUs with states changed will be set in the mask,
- * CPUs with status unchanged will be unset in the mask. */
-static int rtas_cpu_state_change_mask(enum rtas_cpu_state state,
- cpumask_var_t cpus)
-{
- int cpu;
- int cpuret = 0;
- int ret = 0;
-
- if (cpumask_empty(cpus))
- return 0;
-
- for_each_cpu(cpu, cpus) {
- struct device *dev = get_cpu_device(cpu);
-
- switch (state) {
- case DOWN:
- cpuret = device_offline(dev);
- break;
- case UP:
- cpuret = device_online(dev);
- break;
- }
- if (cpuret < 0) {
- pr_debug("%s: cpu_%s for cpu#%d returned %d.\n",
- __func__,
- ((state == UP) ? "up" : "down"),
- cpu, cpuret);
- if (!ret)
- ret = cpuret;
- if (state == UP) {
- /* clear bits for unchanged cpus, return */
- cpumask_shift_right(cpus, cpus, cpu);
- cpumask_shift_left(cpus, cpus, cpu);
- break;
- } else {
- /* clear bit for unchanged cpu, continue */
- cpumask_clear_cpu(cpu, cpus);
- }
- }
- cond_resched();
- }
-
- return ret;
-}
-#endif
-
-int rtas_online_cpus_mask(cpumask_var_t cpus)
-{
- int ret;
-
- ret = rtas_cpu_state_change_mask(UP, cpus);
-
- if (ret) {
- cpumask_var_t tmp_mask;
-
- if (!alloc_cpumask_var(&tmp_mask, GFP_KERNEL))
- return ret;
-
- /* Use tmp_mask to preserve cpus mask from first failure */
- cpumask_copy(tmp_mask, cpus);
- rtas_offline_cpus_mask(tmp_mask);
- free_cpumask_var(tmp_mask);
- }
-
- return ret;
-}
-
-int rtas_offline_cpus_mask(cpumask_var_t cpus)
-{
- return rtas_cpu_state_change_mask(DOWN, cpus);
-}
-
int rtas_ibm_suspend_me(u64 handle)
{
long state;
@@ -940,8 +850,6 @@
unsigned long retbuf[PLPAR_HCALL_BUFSIZE];
struct rtas_suspend_me_data data;
DECLARE_COMPLETION_ONSTACK(done);
- cpumask_var_t offline_mask;
- int cpuret;
if (!rtas_service_present("ibm,suspend-me"))
return -ENOSYS;
@@ -962,9 +870,6 @@
return -EIO;
}
- if (!alloc_cpumask_var(&offline_mask, GFP_KERNEL))
- return -ENOMEM;
-
atomic_set(&data.working, 0);
atomic_set(&data.done, 0);
atomic_set(&data.error, 0);
@@ -973,24 +878,8 @@
lock_device_hotplug();
- /* All present CPUs must be online */
- cpumask_andnot(offline_mask, cpu_present_mask, cpu_online_mask);
- cpuret = rtas_online_cpus_mask(offline_mask);
- if (cpuret) {
- pr_err("%s: Could not bring present CPUs online.\n", __func__);
- atomic_set(&data.error, cpuret);
- goto out;
- }
-
cpu_hotplug_disable();
- /* Check if we raced with a CPU-Offline Operation */
- if (!cpumask_equal(cpu_present_mask, cpu_online_mask)) {
- pr_info("%s: Raced against a concurrent CPU-Offline\n", __func__);
- atomic_set(&data.error, -EAGAIN);
- goto out_hotplug_enable;
- }
-
/* Call function on all CPUs. One of us will make the
* rtas call
*/
@@ -1001,18 +890,11 @@
if (atomic_read(&data.error) != 0)
printk(KERN_ERR "Error doing global join\n");
-out_hotplug_enable:
+
cpu_hotplug_enable();
- /* Take down CPUs not online prior to suspend */
- cpuret = rtas_offline_cpus_mask(offline_mask);
- if (cpuret)
- pr_warn("%s: Could not restore CPUs to offline state.\n",
- __func__);
-
-out:
unlock_device_hotplug();
- free_cpumask_var(offline_mask);
+
return atomic_read(&data.error);
}
diff --git a/arch/powerpc/kernel/rtasd.c b/arch/powerpc/kernel/rtasd.c
index 89b798f..8561dfb 100644
--- a/arch/powerpc/kernel/rtasd.c
+++ b/arch/powerpc/kernel/rtasd.c
@@ -273,37 +273,15 @@
}
}
-#ifdef CONFIG_PPC_PSERIES
-static void handle_prrn_event(s32 scope)
-{
- /*
- * For PRRN, we must pass the negative of the scope value in
- * the RTAS event.
- */
- pseries_devicetree_update(-scope);
- numa_update_cpu_topology(false);
-}
-
static void handle_rtas_event(const struct rtas_error_log *log)
{
- if (rtas_error_type(log) != RTAS_TYPE_PRRN || !prrn_is_enabled())
+ if (!machine_is(pseries))
return;
- /* For PRRN Events the extended log length is used to denote
- * the scope for calling rtas update-nodes.
- */
- handle_prrn_event(rtas_error_extended_log_length(log));
+ if (rtas_error_type(log) == RTAS_TYPE_PRRN)
+ pr_info_ratelimited("Platform resource reassignment ignored.\n");
}
-#else
-
-static void handle_rtas_event(const struct rtas_error_log *log)
-{
- return;
-}
-
-#endif
-
static int rtas_log_open(struct inode * inode, struct file * file)
{
return 0;
diff --git a/arch/powerpc/kernel/secure_boot.c b/arch/powerpc/kernel/secure_boot.c
index 4b98232..f9af305 100644
--- a/arch/powerpc/kernel/secure_boot.c
+++ b/arch/powerpc/kernel/secure_boot.c
@@ -23,12 +23,19 @@
{
struct device_node *node;
bool enabled = false;
+ u32 secureboot;
node = get_ppc_fw_sb_node();
enabled = of_property_read_bool(node, "os-secureboot-enforcing");
-
of_node_put(node);
+ if (enabled)
+ goto out;
+
+ if (!of_property_read_u32(of_root, "ibm,secure-boot", &secureboot))
+ enabled = (secureboot > 1);
+
+out:
pr_info("Secure boot mode %s\n", enabled ? "enabled" : "disabled");
return enabled;
@@ -38,12 +45,19 @@
{
struct device_node *node;
bool enabled = false;
+ u32 trustedboot;
node = get_ppc_fw_sb_node();
enabled = of_property_read_bool(node, "trusted-enabled");
-
of_node_put(node);
+ if (enabled)
+ goto out;
+
+ if (!of_property_read_u32(of_root, "ibm,trusted-boot", &trustedboot))
+ enabled = (trustedboot > 0);
+
+out:
pr_info("Trusted boot mode %s\n", enabled ? "enabled" : "disabled");
return enabled;
diff --git a/arch/powerpc/kernel/security.c b/arch/powerpc/kernel/security.c
index d86701c..c9876aa 100644
--- a/arch/powerpc/kernel/security.c
+++ b/arch/powerpc/kernel/security.c
@@ -21,13 +21,13 @@
u64 powerpc_security_features __read_mostly = SEC_FTR_DEFAULT;
-enum count_cache_flush_type {
- COUNT_CACHE_FLUSH_NONE = 0x1,
- COUNT_CACHE_FLUSH_SW = 0x2,
- COUNT_CACHE_FLUSH_HW = 0x4,
+enum branch_cache_flush_type {
+ BRANCH_CACHE_FLUSH_NONE = 0x1,
+ BRANCH_CACHE_FLUSH_SW = 0x2,
+ BRANCH_CACHE_FLUSH_HW = 0x4,
};
-static enum count_cache_flush_type count_cache_flush_type = COUNT_CACHE_FLUSH_NONE;
-static bool link_stack_flush_enabled;
+static enum branch_cache_flush_type count_cache_flush_type = BRANCH_CACHE_FLUSH_NONE;
+static enum branch_cache_flush_type link_stack_flush_type = BRANCH_CACHE_FLUSH_NONE;
bool barrier_nospec_enabled;
static bool no_nospec;
@@ -219,24 +219,25 @@
if (ccd)
seq_buf_printf(&s, "Indirect branch cache disabled");
- if (link_stack_flush_enabled)
- seq_buf_printf(&s, ", Software link stack flush");
-
- } else if (count_cache_flush_type != COUNT_CACHE_FLUSH_NONE) {
+ } else if (count_cache_flush_type != BRANCH_CACHE_FLUSH_NONE) {
seq_buf_printf(&s, "Mitigation: Software count cache flush");
- if (count_cache_flush_type == COUNT_CACHE_FLUSH_HW)
+ if (count_cache_flush_type == BRANCH_CACHE_FLUSH_HW)
seq_buf_printf(&s, " (hardware accelerated)");
- if (link_stack_flush_enabled)
- seq_buf_printf(&s, ", Software link stack flush");
-
} else if (btb_flush_enabled) {
seq_buf_printf(&s, "Mitigation: Branch predictor state flush");
} else {
seq_buf_printf(&s, "Vulnerable");
}
+ if (bcs || ccd || count_cache_flush_type != BRANCH_CACHE_FLUSH_NONE) {
+ if (link_stack_flush_type != BRANCH_CACHE_FLUSH_NONE)
+ seq_buf_printf(&s, ", Software link stack flush");
+ if (link_stack_flush_type == BRANCH_CACHE_FLUSH_HW)
+ seq_buf_printf(&s, " (hardware accelerated)");
+ }
+
seq_buf_printf(&s, "\n");
return s.len;
@@ -427,61 +428,79 @@
device_initcall(stf_barrier_debugfs_init);
#endif /* CONFIG_DEBUG_FS */
-static void no_count_cache_flush(void)
+static void update_branch_cache_flush(void)
{
- count_cache_flush_type = COUNT_CACHE_FLUSH_NONE;
- pr_info("count-cache-flush: software flush disabled.\n");
-}
-
-static void toggle_count_cache_flush(bool enable)
-{
- if (!security_ftr_enabled(SEC_FTR_FLUSH_COUNT_CACHE) &&
- !security_ftr_enabled(SEC_FTR_FLUSH_LINK_STACK))
- enable = false;
-
- if (!enable) {
- patch_instruction_site(&patch__call_flush_count_cache,
- ppc_inst(PPC_INST_NOP));
#ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE
+ // This controls the branch from guest_exit_cont to kvm_flush_link_stack
+ if (link_stack_flush_type == BRANCH_CACHE_FLUSH_NONE) {
patch_instruction_site(&patch__call_kvm_flush_link_stack,
ppc_inst(PPC_INST_NOP));
-#endif
- pr_info("link-stack-flush: software flush disabled.\n");
- link_stack_flush_enabled = false;
- no_count_cache_flush();
- return;
+ } else {
+ // Could use HW flush, but that could also flush count cache
+ patch_branch_site(&patch__call_kvm_flush_link_stack,
+ (u64)&kvm_flush_link_stack, BRANCH_SET_LINK);
}
-
- // This enables the branch from _switch to flush_count_cache
- patch_branch_site(&patch__call_flush_count_cache,
- (u64)&flush_count_cache, BRANCH_SET_LINK);
-
-#ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE
- // This enables the branch from guest_exit_cont to kvm_flush_link_stack
- patch_branch_site(&patch__call_kvm_flush_link_stack,
- (u64)&kvm_flush_link_stack, BRANCH_SET_LINK);
#endif
- pr_info("link-stack-flush: software flush enabled.\n");
- link_stack_flush_enabled = true;
+ // This controls the branch from _switch to flush_branch_caches
+ if (count_cache_flush_type == BRANCH_CACHE_FLUSH_NONE &&
+ link_stack_flush_type == BRANCH_CACHE_FLUSH_NONE) {
+ patch_instruction_site(&patch__call_flush_branch_caches,
+ ppc_inst(PPC_INST_NOP));
+ } else if (count_cache_flush_type == BRANCH_CACHE_FLUSH_HW &&
+ link_stack_flush_type == BRANCH_CACHE_FLUSH_HW) {
+ patch_instruction_site(&patch__call_flush_branch_caches,
+ ppc_inst(PPC_INST_BCCTR_FLUSH));
+ } else {
+ patch_branch_site(&patch__call_flush_branch_caches,
+ (u64)&flush_branch_caches, BRANCH_SET_LINK);
- // If we just need to flush the link stack, patch an early return
- if (!security_ftr_enabled(SEC_FTR_FLUSH_COUNT_CACHE)) {
- patch_instruction_site(&patch__flush_link_stack_return,
- ppc_inst(PPC_INST_BLR));
- no_count_cache_flush();
- return;
+ // If we just need to flush the link stack, early return
+ if (count_cache_flush_type == BRANCH_CACHE_FLUSH_NONE) {
+ patch_instruction_site(&patch__flush_link_stack_return,
+ ppc_inst(PPC_INST_BLR));
+
+ // If we have flush instruction, early return
+ } else if (count_cache_flush_type == BRANCH_CACHE_FLUSH_HW) {
+ patch_instruction_site(&patch__flush_count_cache_return,
+ ppc_inst(PPC_INST_BLR));
+ }
+ }
+}
+
+static void toggle_branch_cache_flush(bool enable)
+{
+ if (!enable || !security_ftr_enabled(SEC_FTR_FLUSH_COUNT_CACHE)) {
+ if (count_cache_flush_type != BRANCH_CACHE_FLUSH_NONE)
+ count_cache_flush_type = BRANCH_CACHE_FLUSH_NONE;
+
+ pr_info("count-cache-flush: flush disabled.\n");
+ } else {
+ if (security_ftr_enabled(SEC_FTR_BCCTR_FLUSH_ASSIST)) {
+ count_cache_flush_type = BRANCH_CACHE_FLUSH_HW;
+ pr_info("count-cache-flush: hardware flush enabled.\n");
+ } else {
+ count_cache_flush_type = BRANCH_CACHE_FLUSH_SW;
+ pr_info("count-cache-flush: software flush enabled.\n");
+ }
}
- if (!security_ftr_enabled(SEC_FTR_BCCTR_FLUSH_ASSIST)) {
- count_cache_flush_type = COUNT_CACHE_FLUSH_SW;
- pr_info("count-cache-flush: full software flush sequence enabled.\n");
- return;
+ if (!enable || !security_ftr_enabled(SEC_FTR_FLUSH_LINK_STACK)) {
+ if (link_stack_flush_type != BRANCH_CACHE_FLUSH_NONE)
+ link_stack_flush_type = BRANCH_CACHE_FLUSH_NONE;
+
+ pr_info("link-stack-flush: flush disabled.\n");
+ } else {
+ if (security_ftr_enabled(SEC_FTR_BCCTR_LINK_FLUSH_ASSIST)) {
+ link_stack_flush_type = BRANCH_CACHE_FLUSH_HW;
+ pr_info("link-stack-flush: hardware flush enabled.\n");
+ } else {
+ link_stack_flush_type = BRANCH_CACHE_FLUSH_SW;
+ pr_info("link-stack-flush: software flush enabled.\n");
+ }
}
- patch_instruction_site(&patch__flush_count_cache_return, ppc_inst(PPC_INST_BLR));
- count_cache_flush_type = COUNT_CACHE_FLUSH_HW;
- pr_info("count-cache-flush: hardware assisted flush sequence enabled\n");
+ update_branch_cache_flush();
}
void setup_count_cache_flush(void)
@@ -505,7 +524,7 @@
security_ftr_enabled(SEC_FTR_FLUSH_COUNT_CACHE))
security_ftr_set(SEC_FTR_FLUSH_LINK_STACK);
- toggle_count_cache_flush(enable);
+ toggle_branch_cache_flush(enable);
}
#ifdef CONFIG_DEBUG_FS
@@ -520,14 +539,14 @@
else
return -EINVAL;
- toggle_count_cache_flush(enable);
+ toggle_branch_cache_flush(enable);
return 0;
}
static int count_cache_flush_get(void *data, u64 *val)
{
- if (count_cache_flush_type == COUNT_CACHE_FLUSH_NONE)
+ if (count_cache_flush_type == BRANCH_CACHE_FLUSH_NONE)
*val = 0;
else
*val = 1;
diff --git a/arch/powerpc/kernel/setup-common.c b/arch/powerpc/kernel/setup-common.c
index 9d3faac..b198b0f 100644
--- a/arch/powerpc/kernel/setup-common.c
+++ b/arch/powerpc/kernel/setup-common.c
@@ -928,6 +928,9 @@
/* Reserve large chunks of memory for use by CMA for KVM. */
kvm_cma_reserve();
+ /* Reserve large chunks of memory for us by CMA for hugetlb */
+ gigantic_hugetlb_cma_reserve();
+
klp_init_thread_info(&init_task);
init_mm.start_code = (unsigned long)_stext;
diff --git a/arch/powerpc/kernel/setup_64.c b/arch/powerpc/kernel/setup_64.c
index 0ba1ed7..6be4301 100644
--- a/arch/powerpc/kernel/setup_64.c
+++ b/arch/powerpc/kernel/setup_64.c
@@ -196,7 +196,10 @@
/* Under a PAPR hypervisor, we need hypercalls */
if (firmware_has_feature(FW_FEATURE_SET_MODE)) {
/* Enable AIL if possible */
- pseries_enable_reloc_on_exc();
+ if (!pseries_enable_reloc_on_exc()) {
+ init_task.thread.fscr &= ~FSCR_SCV;
+ cur_cpu_spec->cpu_user_features2 &= ~PPC_FEATURE2_SCV;
+ }
/*
* Tell the hypervisor that we want our exceptions to
diff --git a/arch/powerpc/kernel/signal.c b/arch/powerpc/kernel/signal.c
index b4143b6..d15a98c 100644
--- a/arch/powerpc/kernel/signal.c
+++ b/arch/powerpc/kernel/signal.c
@@ -205,8 +205,14 @@
return;
/* error signalled ? */
- if (!(regs->ccr & 0x10000000))
+ if (trap_is_scv(regs)) {
+ /* 32-bit compat mode sign extend? */
+ if (!IS_ERR_VALUE(ret))
+ return;
+ ret = -ret;
+ } else if (!(regs->ccr & 0x10000000)) {
return;
+ }
switch (ret) {
case ERESTART_RESTARTBLOCK:
@@ -239,9 +245,14 @@
regs->nip -= 4;
regs->result = 0;
} else {
- regs->result = -EINTR;
- regs->gpr[3] = EINTR;
- regs->ccr |= 0x10000000;
+ if (trap_is_scv(regs)) {
+ regs->result = -EINTR;
+ regs->gpr[3] = -EINTR;
+ } else {
+ regs->result = -EINTR;
+ regs->gpr[3] = EINTR;
+ regs->ccr |= 0x10000000;
+ }
}
}
diff --git a/arch/powerpc/kernel/signal_32.c b/arch/powerpc/kernel/signal_32.c
index 1415c16..96950f1 100644
--- a/arch/powerpc/kernel/signal_32.c
+++ b/arch/powerpc/kernel/signal_32.c
@@ -102,22 +102,18 @@
struct mcontext __user *frame)
{
elf_greg_t64 *gregs = (elf_greg_t64 *)regs;
- int i;
- /* Force usr to alway see softe as 1 (interrupts enabled) */
- elf_greg_t64 softe = 0x1;
+ int val, i;
WARN_ON(!FULL_REGS(regs));
for (i = 0; i <= PT_RESULT; i ++) {
- if (i == 14 && !FULL_REGS(regs))
- i = 32;
- if ( i == PT_SOFTE) {
- if(__put_user((unsigned int)softe, &frame->mc_gregs[i]))
- return -EFAULT;
- else
- continue;
- }
- if (__put_user((unsigned int)gregs[i], &frame->mc_gregs[i]))
+ /* Force usr to alway see softe as 1 (interrupts enabled) */
+ if (i == PT_SOFTE)
+ val = 1;
+ else
+ val = gregs[i];
+
+ if (__put_user(val, &frame->mc_gregs[i]))
return -EFAULT;
}
return 0;
diff --git a/arch/powerpc/kernel/signal_64.c b/arch/powerpc/kernel/signal_64.c
index 55e5f76..bfc9393 100644
--- a/arch/powerpc/kernel/signal_64.c
+++ b/arch/powerpc/kernel/signal_64.c
@@ -21,6 +21,7 @@
#include <linux/ptrace.h>
#include <linux/ratelimit.h>
#include <linux/syscalls.h>
+#include <linux/pagemap.h>
#include <asm/sigcontext.h>
#include <asm/ucontext.h>
@@ -39,8 +40,8 @@
#define GP_REGS_SIZE min(sizeof(elf_gregset_t), sizeof(struct pt_regs))
#define FP_REGS_SIZE sizeof(elf_fpregset_t)
-#define TRAMP_TRACEBACK 3
-#define TRAMP_SIZE 6
+#define TRAMP_TRACEBACK 4
+#define TRAMP_SIZE 7
/*
* When we have signals to deliver, we set up on the user stack,
@@ -600,13 +601,15 @@
int i;
long err = 0;
+ /* bctrl # call the handler */
+ err |= __put_user(PPC_INST_BCTRL, &tramp[0]);
/* addi r1, r1, __SIGNAL_FRAMESIZE # Pop the dummy stackframe */
err |= __put_user(PPC_INST_ADDI | __PPC_RT(R1) | __PPC_RA(R1) |
- (__SIGNAL_FRAMESIZE & 0xffff), &tramp[0]);
+ (__SIGNAL_FRAMESIZE & 0xffff), &tramp[1]);
/* li r0, __NR_[rt_]sigreturn| */
- err |= __put_user(PPC_INST_ADDI | (syscall & 0xffff), &tramp[1]);
+ err |= __put_user(PPC_INST_ADDI | (syscall & 0xffff), &tramp[2]);
/* sc */
- err |= __put_user(PPC_INST_SC, &tramp[2]);
+ err |= __put_user(PPC_INST_SC, &tramp[3]);
/* Minimal traceback info */
for (i=TRAMP_TRACEBACK; i < TRAMP_SIZE ;i++)
@@ -632,7 +635,6 @@
SYSCALL_DEFINE3(swapcontext, struct ucontext __user *, old_ctx,
struct ucontext __user *, new_ctx, long, ctx_size)
{
- unsigned char tmp;
sigset_t set;
unsigned long new_msr = 0;
int ctx_has_vsx_region = 0;
@@ -667,9 +669,8 @@
}
if (new_ctx == NULL)
return 0;
- if (!access_ok(new_ctx, ctx_size)
- || __get_user(tmp, (u8 __user *) new_ctx)
- || __get_user(tmp, (u8 __user *) new_ctx + ctx_size - 1))
+ if (!access_ok(new_ctx, ctx_size) ||
+ fault_in_pages_readable((u8 __user *)new_ctx, ctx_size))
return -EFAULT;
/*
@@ -864,12 +865,12 @@
/* Set up to return from userspace. */
if (vdso64_rt_sigtramp && tsk->mm->context.vdso_base) {
- regs->link = tsk->mm->context.vdso_base + vdso64_rt_sigtramp;
+ regs->nip = tsk->mm->context.vdso_base + vdso64_rt_sigtramp;
} else {
err |= setup_trampoline(__NR_rt_sigreturn, &frame->tramp[0]);
if (err)
goto badframe;
- regs->link = (unsigned long) &frame->tramp[0];
+ regs->nip = (unsigned long) &frame->tramp[0];
}
/* Allocate a dummy caller frame for the signal handler. */
@@ -878,8 +879,8 @@
/* Set up "regs" so we "return" to the signal handler. */
if (is_elf2_task()) {
- regs->nip = (unsigned long) ksig->ka.sa.sa_handler;
- regs->gpr[12] = regs->nip;
+ regs->ctr = (unsigned long) ksig->ka.sa.sa_handler;
+ regs->gpr[12] = regs->ctr;
} else {
/* Handler is *really* a pointer to the function descriptor for
* the signal routine. The first entry in the function
@@ -889,7 +890,7 @@
func_descr_t __user *funct_desc_ptr =
(func_descr_t __user *) ksig->ka.sa.sa_handler;
- err |= get_user(regs->nip, &funct_desc_ptr->entry);
+ err |= get_user(regs->ctr, &funct_desc_ptr->entry);
err |= get_user(regs->gpr[2], &funct_desc_ptr->toc);
}
diff --git a/arch/powerpc/kernel/smp.c b/arch/powerpc/kernel/smp.c
index 7319947..8261999 100644
--- a/arch/powerpc/kernel/smp.c
+++ b/arch/powerpc/kernel/smp.c
@@ -59,6 +59,7 @@
#include <asm/asm-prototypes.h>
#include <asm/cpu_has_feature.h>
#include <asm/ftrace.h>
+#include <asm/kup.h>
#ifdef DEBUG
#include <asm/udbg.h>
diff --git a/arch/powerpc/kernel/syscall_64.c b/arch/powerpc/kernel/syscall_64.c
index 79edba3..8e50818 100644
--- a/arch/powerpc/kernel/syscall_64.c
+++ b/arch/powerpc/kernel/syscall_64.c
@@ -60,6 +60,11 @@
local_irq_enable();
if (unlikely(current_thread_info()->flags & _TIF_SYSCALL_DOTRACE)) {
+ if (unlikely(regs->trap == 0x7ff0)) {
+ /* Unsupported scv vector */
+ _exception(SIGILL, regs, ILL_ILLOPC, regs->nip);
+ return regs->gpr[3];
+ }
/*
* We use the return value of do_syscall_trace_enter() as the
* syscall number. If the syscall was rejected for any reason
@@ -78,6 +83,11 @@
r8 = regs->gpr[8];
} else if (unlikely(r0 >= NR_syscalls)) {
+ if (unlikely(regs->trap == 0x7ff0)) {
+ /* Unsupported scv vector */
+ _exception(SIGILL, regs, ILL_ILLOPC, regs->nip);
+ return regs->gpr[3];
+ }
return -ENOSYS;
}
@@ -105,16 +115,20 @@
* local irqs must be disabled. Returns false if the caller must re-enable
* them, check for new work, and try again.
*/
-static notrace inline bool prep_irq_for_enabled_exit(void)
+static notrace inline bool prep_irq_for_enabled_exit(bool clear_ri)
{
/* This must be done with RI=1 because tracing may touch vmaps */
trace_hardirqs_on();
/* This pattern matches prep_irq_for_idle */
- __hard_EE_RI_disable();
+ if (clear_ri)
+ __hard_EE_RI_disable();
+ else
+ __hard_irq_disable();
if (unlikely(lazy_irq_pending_nocheck())) {
/* Took an interrupt, may have more exit work to do. */
- __hard_RI_enable();
+ if (clear_ri)
+ __hard_RI_enable();
trace_hardirqs_off();
local_paca->irq_happened |= PACA_IRQ_HARD_DIS;
@@ -136,7 +150,8 @@
* because RI=0 and soft mask state is "unreconciled", so it is marked notrace.
*/
notrace unsigned long syscall_exit_prepare(unsigned long r3,
- struct pt_regs *regs)
+ struct pt_regs *regs,
+ long scv)
{
unsigned long *ti_flagsp = ¤t_thread_info()->flags;
unsigned long ti_flags;
@@ -151,7 +166,7 @@
ti_flags = *ti_flagsp;
- if (unlikely(r3 >= (unsigned long)-MAX_ERRNO)) {
+ if (unlikely(r3 >= (unsigned long)-MAX_ERRNO) && !scv) {
if (likely(!(ti_flags & (_TIF_NOERROR | _TIF_RESTOREALL)))) {
r3 = -r3;
regs->ccr |= 0x10000000; /* Set SO bit in CR */
@@ -206,12 +221,20 @@
else if (cpu_has_feature(CPU_FTR_ALTIVEC))
mathflags |= MSR_VEC;
+ /*
+ * If userspace MSR has all available FP bits set,
+ * then they are live and no need to restore. If not,
+ * it means the regs were given up and restore_math
+ * may decide to restore them (to avoid taking an FP
+ * fault).
+ */
if ((regs->msr & mathflags) != mathflags)
restore_math(regs);
}
}
- if (unlikely(!prep_irq_for_enabled_exit())) {
+ /* scv need not set RI=0 because SRRs are not used */
+ if (unlikely(!prep_irq_for_enabled_exit(!scv))) {
local_irq_enable();
goto again;
}
@@ -277,12 +300,13 @@
else if (cpu_has_feature(CPU_FTR_ALTIVEC))
mathflags |= MSR_VEC;
+ /* See above restore_math comment */
if ((regs->msr & mathflags) != mathflags)
restore_math(regs);
}
}
- if (unlikely(!prep_irq_for_enabled_exit())) {
+ if (unlikely(!prep_irq_for_enabled_exit(true))) {
local_irq_enable();
local_irq_disable();
goto again;
@@ -345,7 +369,7 @@
}
}
- if (unlikely(!prep_irq_for_enabled_exit())) {
+ if (unlikely(!prep_irq_for_enabled_exit(true))) {
/*
* Can't local_irq_restore to replay if we were in
* interrupt context. Must replay directly.
diff --git a/arch/powerpc/kernel/sysfs.c b/arch/powerpc/kernel/sysfs.c
index 571b325..46b4ebc 100644
--- a/arch/powerpc/kernel/sysfs.c
+++ b/arch/powerpc/kernel/sysfs.c
@@ -622,8 +622,10 @@
SYSFS_PMCSETUP(pmc8, SPRN_PMC8);
SYSFS_PMCSETUP(mmcra, SPRN_MMCRA);
+SYSFS_PMCSETUP(mmcr3, SPRN_MMCR3);
static DEVICE_ATTR(mmcra, 0600, show_mmcra, store_mmcra);
+static DEVICE_ATTR(mmcr3, 0600, show_mmcr3, store_mmcr3);
#endif /* HAS_PPC_PMC56 */
@@ -886,6 +888,9 @@
#ifdef CONFIG_PMU_SYSFS
if (cpu_has_feature(CPU_FTR_MMCRA))
device_create_file(s, &dev_attr_mmcra);
+
+ if (cpu_has_feature(CPU_FTR_ARCH_31))
+ device_create_file(s, &dev_attr_mmcr3);
#endif /* CONFIG_PMU_SYSFS */
if (cpu_has_feature(CPU_FTR_PURR)) {
@@ -980,6 +985,9 @@
#ifdef CONFIG_PMU_SYSFS
if (cpu_has_feature(CPU_FTR_MMCRA))
device_remove_file(s, &dev_attr_mmcra);
+
+ if (cpu_has_feature(CPU_FTR_ARCH_31))
+ device_remove_file(s, &dev_attr_mmcr3);
#endif /* CONFIG_PMU_SYSFS */
if (cpu_has_feature(CPU_FTR_PURR)) {
diff --git a/arch/powerpc/kernel/trace/ftrace.c b/arch/powerpc/kernel/trace/ftrace.c
index c1fede6..42761eb 100644
--- a/arch/powerpc/kernel/trace/ftrace.c
+++ b/arch/powerpc/kernel/trace/ftrace.c
@@ -73,8 +73,8 @@
/* Make sure it is what we expect it to be */
if (!ppc_inst_equal(replaced, old)) {
- pr_err("%p: replaced (%#x) != old (%#x)",
- (void *)ip, ppc_inst_val(replaced), ppc_inst_val(old));
+ pr_err("%p: replaced (%s) != old (%s)",
+ (void *)ip, ppc_inst_as_str(replaced), ppc_inst_as_str(old));
return -EINVAL;
}
@@ -137,7 +137,7 @@
/* Make sure that that this is still a 24bit jump */
if (!is_bl_op(op)) {
- pr_err("Not expected bl: opcode is %x\n", ppc_inst_val(op));
+ pr_err("Not expected bl: opcode is %s\n", ppc_inst_as_str(op));
return -EINVAL;
}
@@ -172,8 +172,8 @@
/* We expect either a mflr r0, or a std r0, LRSAVE(r1) */
if (!ppc_inst_equal(op, ppc_inst(PPC_INST_MFLR)) &&
!ppc_inst_equal(op, ppc_inst(PPC_INST_STD_LR))) {
- pr_err("Unexpected instruction %08x around bl _mcount\n",
- ppc_inst_val(op));
+ pr_err("Unexpected instruction %s around bl _mcount\n",
+ ppc_inst_as_str(op));
return -EINVAL;
}
#else
@@ -203,7 +203,7 @@
}
if (!ppc_inst_equal(op, ppc_inst(PPC_INST_LD_TOC))) {
- pr_err("Expected %08x found %08x\n", PPC_INST_LD_TOC, ppc_inst_val(op));
+ pr_err("Expected %08x found %s\n", PPC_INST_LD_TOC, ppc_inst_as_str(op));
return -EINVAL;
}
#endif /* CONFIG_MPROFILE_KERNEL */
@@ -231,7 +231,7 @@
/* Make sure that that this is still a 24bit jump */
if (!is_bl_op(op)) {
- pr_err("Not expected bl: opcode is %x\n", ppc_inst_val(op));
+ pr_err("Not expected bl: opcode is %s\n", ppc_inst_as_str(op));
return -EINVAL;
}
@@ -406,7 +406,7 @@
/* Make sure that that this is still a 24bit jump */
if (!is_bl_op(op)) {
- pr_err("Not expected bl: opcode is %x\n", ppc_inst_val(op));
+ pr_err("Not expected bl: opcode is %s\n", ppc_inst_as_str(op));
return -EINVAL;
}
@@ -533,8 +533,8 @@
return -EFAULT;
if (!expected_nop_sequence(ip, op[0], op[1])) {
- pr_err("Unexpected call sequence at %p: %x %x\n",
- ip, ppc_inst_val(op[0]), ppc_inst_val(op[1]));
+ pr_err("Unexpected call sequence at %p: %s %s\n",
+ ip, ppc_inst_as_str(op[0]), ppc_inst_as_str(op[1]));
return -EINVAL;
}
@@ -597,7 +597,7 @@
/* It should be pointing to a nop */
if (!ppc_inst_equal(op, ppc_inst(PPC_INST_NOP))) {
- pr_err("Expected NOP but have %x\n", ppc_inst_val(op));
+ pr_err("Expected NOP but have %s\n", ppc_inst_as_str(op));
return -EINVAL;
}
@@ -654,7 +654,7 @@
}
if (!ppc_inst_equal(op, ppc_inst(PPC_INST_NOP))) {
- pr_err("Unexpected call sequence at %p: %x\n", ip, ppc_inst_val(op));
+ pr_err("Unexpected call sequence at %p: %s\n", ip, ppc_inst_as_str(op));
return -EINVAL;
}
@@ -733,7 +733,7 @@
/* Make sure that that this is still a 24bit jump */
if (!is_bl_op(op)) {
- pr_err("Not expected bl: opcode is %x\n", ppc_inst_val(op));
+ pr_err("Not expected bl: opcode is %s\n", ppc_inst_as_str(op));
return -EINVAL;
}
diff --git a/arch/powerpc/kernel/traps.c b/arch/powerpc/kernel/traps.c
index 97413a3..d1ebe15 100644
--- a/arch/powerpc/kernel/traps.c
+++ b/arch/powerpc/kernel/traps.c
@@ -2060,14 +2060,6 @@
NOKPROBE_SYMBOL(DebugException);
#endif /* CONFIG_PPC_ADV_DEBUG_REGS */
-#if !defined(CONFIG_TAU_INT)
-void TAUException(struct pt_regs *regs)
-{
- printk("TAU trap at PC: %lx, MSR: %lx, vector=%lx %s\n",
- regs->nip, regs->msr, regs->trap, print_tainted());
-}
-#endif /* CONFIG_INT_TAU */
-
#ifdef CONFIG_ALTIVEC
void altivec_assist_exception(struct pt_regs *regs)
{
diff --git a/arch/powerpc/kernel/vdso.c b/arch/powerpc/kernel/vdso.c
index e0f4ba4..8dad442 100644
--- a/arch/powerpc/kernel/vdso.c
+++ b/arch/powerpc/kernel/vdso.c
@@ -677,7 +677,7 @@
node = cpu_to_node(cpu);
WARN_ON_ONCE(node > 0xffff);
- val = (cpu & 0xfff) | ((node & 0xffff) << 16);
+ val = (cpu & 0xffff) | ((node & 0xffff) << 16);
mtspr(SPRN_SPRG_VDSO_WRITE, val);
get_paca()->sprg_vdso = val;
diff --git a/arch/powerpc/kernel/vdso32/Makefile b/arch/powerpc/kernel/vdso32/Makefile
index e147bbd..87ab1152 100644
--- a/arch/powerpc/kernel/vdso32/Makefile
+++ b/arch/powerpc/kernel/vdso32/Makefile
@@ -50,7 +50,7 @@
# actual build commands
quiet_cmd_vdso32ld = VDSO32L $@
- cmd_vdso32ld = $(VDSOCC) $(c_flags) $(CC32FLAGS) -o $@ -Wl,-T$(filter %.lds,$^) $(filter %.o,$^)
+ cmd_vdso32ld = $(VDSOCC) $(c_flags) $(CC32FLAGS) -o $@ $(call cc-ldoption, -Wl$(comma)--orphan-handling=warn) -Wl,-T$(filter %.lds,$^) $(filter %.o,$^)
quiet_cmd_vdso32as = VDSO32A $@
cmd_vdso32as = $(VDSOCC) $(a_flags) $(CC32FLAGS) -c -o $@ $<
diff --git a/arch/powerpc/kernel/vdso32/vdso32.lds.S b/arch/powerpc/kernel/vdso32/vdso32.lds.S
index 5206c2e..4c98546 100644
--- a/arch/powerpc/kernel/vdso32/vdso32.lds.S
+++ b/arch/powerpc/kernel/vdso32/vdso32.lds.S
@@ -111,6 +111,7 @@
*(.note.GNU-stack)
*(.data .data.* .gnu.linkonce.d.* .sdata*)
*(.bss .sbss .dynbss .dynsbss)
+ *(.glink .iplt .plt .rela*)
}
}
diff --git a/arch/powerpc/kernel/vdso64/Makefile b/arch/powerpc/kernel/vdso64/Makefile
index 32ebb352..38c317f 100644
--- a/arch/powerpc/kernel/vdso64/Makefile
+++ b/arch/powerpc/kernel/vdso64/Makefile
@@ -34,7 +34,7 @@
# actual build commands
quiet_cmd_vdso64ld = VDSO64L $@
- cmd_vdso64ld = $(CC) $(c_flags) -o $@ -Wl,-T$(filter %.lds,$^) $(filter %.o,$^)
+ cmd_vdso64ld = $(CC) $(c_flags) -o $@ -Wl,-T$(filter %.lds,$^) $(filter %.o,$^) $(call cc-ldoption, -Wl$(comma)--orphan-handling=warn)
# install commands for the unstripped file
quiet_cmd_vdso_install = INSTALL $@
diff --git a/arch/powerpc/kernel/vdso64/cacheflush.S b/arch/powerpc/kernel/vdso64/cacheflush.S
index 526f5ba..cab1432 100644
--- a/arch/powerpc/kernel/vdso64/cacheflush.S
+++ b/arch/powerpc/kernel/vdso64/cacheflush.S
@@ -8,6 +8,7 @@
#include <asm/processor.h>
#include <asm/ppc_asm.h>
#include <asm/vdso.h>
+#include <asm/vdso_datapage.h>
#include <asm/asm-offsets.h>
.text
@@ -24,14 +25,12 @@
.cfi_startproc
mflr r12
.cfi_register lr,r12
- mr r11,r3
- bl V_LOCAL_FUNC(__get_datapage)
+ get_datapage r10, r0
mtlr r12
- mr r10,r3
lwz r7,CFG_DCACHE_BLOCKSZ(r10)
addi r5,r7,-1
- andc r6,r11,r5 /* round low to line bdy */
+ andc r6,r3,r5 /* round low to line bdy */
subf r8,r6,r4 /* compute length */
add r8,r8,r5 /* ensure we get enough */
lwz r9,CFG_DCACHE_LOGBLOCKSZ(r10)
@@ -48,7 +47,7 @@
lwz r7,CFG_ICACHE_BLOCKSZ(r10)
addi r5,r7,-1
- andc r6,r11,r5 /* round low to line bdy */
+ andc r6,r3,r5 /* round low to line bdy */
subf r8,r6,r4 /* compute length */
add r8,r8,r5
lwz r9,CFG_ICACHE_LOGBLOCKSZ(r10)
diff --git a/arch/powerpc/kernel/vdso64/datapage.S b/arch/powerpc/kernel/vdso64/datapage.S
index dc84f5a..067247d 100644
--- a/arch/powerpc/kernel/vdso64/datapage.S
+++ b/arch/powerpc/kernel/vdso64/datapage.S
@@ -10,35 +10,13 @@
#include <asm/asm-offsets.h>
#include <asm/unistd.h>
#include <asm/vdso.h>
+#include <asm/vdso_datapage.h>
.text
.global __kernel_datapage_offset;
__kernel_datapage_offset:
.long 0
-V_FUNCTION_BEGIN(__get_datapage)
- .cfi_startproc
- /* We don't want that exposed or overridable as we want other objects
- * to be able to bl directly to here
- */
- .protected __get_datapage
- .hidden __get_datapage
-
- mflr r0
- .cfi_register lr,r0
-
- bcl 20,31,data_page_branch
-data_page_branch:
- mflr r3
- mtlr r0
- addi r3, r3, __kernel_datapage_offset-data_page_branch
- lwz r0,0(r3)
- .cfi_restore lr
- add r3,r0,r3
- blr
- .cfi_endproc
-V_FUNCTION_END(__get_datapage)
-
/*
* void *__kernel_get_syscall_map(unsigned int *syscall_count) ;
*
@@ -53,7 +31,7 @@
mflr r12
.cfi_register lr,r12
mr r4,r3
- bl V_LOCAL_FUNC(__get_datapage)
+ get_datapage r3, r0
mtlr r12
addi r3,r3,CFG_SYSCALL_MAP64
cmpldi cr0,r4,0
@@ -75,7 +53,7 @@
.cfi_startproc
mflr r12
.cfi_register lr,r12
- bl V_LOCAL_FUNC(__get_datapage)
+ get_datapage r3, r0
ld r3,CFG_TB_TICKS_PER_SEC(r3)
mtlr r12
crclr cr0*4+so
diff --git a/arch/powerpc/kernel/vdso64/gettimeofday.S b/arch/powerpc/kernel/vdso64/gettimeofday.S
index 1c9a047..20f8be4 100644
--- a/arch/powerpc/kernel/vdso64/gettimeofday.S
+++ b/arch/powerpc/kernel/vdso64/gettimeofday.S
@@ -9,6 +9,7 @@
#include <asm/processor.h>
#include <asm/ppc_asm.h>
#include <asm/vdso.h>
+#include <asm/vdso_datapage.h>
#include <asm/asm-offsets.h>
#include <asm/unistd.h>
@@ -26,7 +27,7 @@
mr r11,r3 /* r11 holds tv */
mr r10,r4 /* r10 holds tz */
- bl V_LOCAL_FUNC(__get_datapage) /* get data page */
+ get_datapage r3, r0
cmpldi r11,0 /* check if tv is NULL */
beq 2f
lis r7,1000000@ha /* load up USEC_PER_SEC */
@@ -71,7 +72,7 @@
mflr r12 /* r12 saves lr */
.cfi_register lr,r12
mr r11,r4 /* r11 saves tp */
- bl V_LOCAL_FUNC(__get_datapage) /* get data page */
+ get_datapage r3, r0
lis r7,NSEC_PER_SEC@h /* want nanoseconds */
ori r7,r7,NSEC_PER_SEC@l
beq cr5,70f
@@ -188,7 +189,7 @@
mflr r12
.cfi_register lr,r12
- bl V_LOCAL_FUNC(__get_datapage)
+ get_datapage r3, r0
lwz r5, CLOCK_HRTIMER_RES(r3)
mtlr r12
li r3,0
@@ -221,7 +222,7 @@
.cfi_register lr,r12
mr r11,r3 /* r11 holds t */
- bl V_LOCAL_FUNC(__get_datapage)
+ get_datapage r3, r0
ld r4,STAMP_XTIME_SEC(r3)
diff --git a/arch/powerpc/kernel/vdso64/sigtramp.S b/arch/powerpc/kernel/vdso64/sigtramp.S
index a8cc040..bbf68cd 100644
--- a/arch/powerpc/kernel/vdso64/sigtramp.S
+++ b/arch/powerpc/kernel/vdso64/sigtramp.S
@@ -6,6 +6,7 @@
* Copyright (C) 2004 Benjamin Herrenschmuidt (benh@kernel.crashing.org), IBM Corp.
* Copyright (C) 2004 Alan Modra (amodra@au.ibm.com)), IBM Corp.
*/
+#include <asm/cache.h> /* IFETCH_ALIGN_BYTES */
#include <asm/processor.h>
#include <asm/ppc_asm.h>
#include <asm/unistd.h>
@@ -14,21 +15,17 @@
.text
-/* The nop here is a hack. The dwarf2 unwind routines subtract 1 from
- the return address to get an address in the middle of the presumed
- call instruction. Since we don't have a call here, we artificially
- extend the range covered by the unwind info by padding before the
- real start. */
- nop
.balign 8
+ .balign IFETCH_ALIGN_BYTES
V_FUNCTION_BEGIN(__kernel_sigtramp_rt64)
-.Lsigrt_start = . - 4
+.Lsigrt_start:
+ bctrl /* call the handler */
addi r1, r1, __SIGNAL_FRAMESIZE
li r0,__NR_rt_sigreturn
sc
.Lsigrt_end:
V_FUNCTION_END(__kernel_sigtramp_rt64)
-/* The ".balign 8" above and the following zeros mimic the old stack
+/* The .balign 8 above and the following zeros mimic the old stack
trampoline layout. The last magic value is the ucontext pointer,
chosen in such a way that older libgcc unwind code returns a zero
for a sigcontext pointer. */
diff --git a/arch/powerpc/kernel/vdso64/vdso64.lds.S b/arch/powerpc/kernel/vdso64/vdso64.lds.S
index 256fb97..4e3a8d4 100644
--- a/arch/powerpc/kernel/vdso64/vdso64.lds.S
+++ b/arch/powerpc/kernel/vdso64/vdso64.lds.S
@@ -30,7 +30,7 @@
. = ALIGN(16);
.text : {
*(.text .stub .text.* .gnu.linkonce.t.* __ftr_alt_*)
- *(.sfpr .glink)
+ *(.sfpr)
} :text
PROVIDE(__etext = .);
PROVIDE(_etext = .);
@@ -111,6 +111,7 @@
*(.branch_lt)
*(.data .data.* .gnu.linkonce.d.* .sdata*)
*(.bss .sbss .dynbss .dynsbss)
+ *(.glink .iplt .plt .rela*)
}
}
diff --git a/arch/powerpc/kernel/vector.S b/arch/powerpc/kernel/vector.S
index efc5b52..801dc28 100644
--- a/arch/powerpc/kernel/vector.S
+++ b/arch/powerpc/kernel/vector.S
@@ -76,9 +76,7 @@
oris r12,r12,MSR_VEC@h
std r12,_MSR(r1)
#endif
- /* Don't care if r4 overflows, this is desired behaviour */
- lbz r4,THREAD_LOAD_VEC(r5)
- addi r4,r4,1
+ li r4,1
stb r4,THREAD_LOAD_VEC(r5)
addi r6,r5,THREAD_VRSTATE
li r4,1
diff --git a/arch/powerpc/kexec/Makefile b/arch/powerpc/kexec/Makefile
index 86380c6..4aff684 100644
--- a/arch/powerpc/kexec/Makefile
+++ b/arch/powerpc/kexec/Makefile
@@ -7,7 +7,7 @@
obj-$(CONFIG_PPC32) += relocate_32.o
-obj-$(CONFIG_KEXEC_FILE) += file_load.o elf_$(BITS).o
+obj-$(CONFIG_KEXEC_FILE) += file_load.o ranges.o file_load_$(BITS).o elf_$(BITS).o
ifdef CONFIG_HAVE_IMA_KEXEC
ifdef CONFIG_IMA
diff --git a/arch/powerpc/kexec/core_64.c b/arch/powerpc/kexec/core_64.c
index b41840921..8a449b2 100644
--- a/arch/powerpc/kexec/core_64.c
+++ b/arch/powerpc/kexec/core_64.c
@@ -152,6 +152,8 @@
if (ppc_md.kexec_cpu_down)
ppc_md.kexec_cpu_down(0, 1);
+ reset_sprs();
+
kexec_smp_wait();
/* NOTREACHED */
}
diff --git a/arch/powerpc/kexec/elf_64.c b/arch/powerpc/kexec/elf_64.c
index 3072fd6..d0e459b 100644
--- a/arch/powerpc/kexec/elf_64.c
+++ b/arch/powerpc/kexec/elf_64.c
@@ -35,6 +35,7 @@
void *fdt;
const void *slave_code;
struct elfhdr ehdr;
+ char *modified_cmdline = NULL;
struct kexec_elf_info elf_info;
struct kexec_buf kbuf = { .image = image, .buf_min = 0,
.buf_max = ppc64_rma_size };
@@ -46,6 +47,14 @@
if (ret)
goto out;
+ if (image->type == KEXEC_TYPE_CRASH) {
+ /* min & max buffer values for kdump case */
+ kbuf.buf_min = pbuf.buf_min = crashk_res.start;
+ kbuf.buf_max = pbuf.buf_max =
+ ((crashk_res.end < ppc64_rma_size) ?
+ crashk_res.end : (ppc64_rma_size - 1));
+ }
+
ret = kexec_elf_load(image, &ehdr, &elf_info, &kbuf, &kernel_load_addr);
if (ret)
goto out;
@@ -60,6 +69,25 @@
pr_debug("Loaded purgatory at 0x%lx\n", pbuf.mem);
+ /* Load additional segments needed for panic kernel */
+ if (image->type == KEXEC_TYPE_CRASH) {
+ ret = load_crashdump_segments_ppc64(image, &kbuf);
+ if (ret) {
+ pr_err("Failed to load kdump kernel segments\n");
+ goto out;
+ }
+
+ /* Setup cmdline for kdump kernel case */
+ modified_cmdline = setup_kdump_cmdline(image, cmdline,
+ cmdline_len);
+ if (!modified_cmdline) {
+ pr_err("Setting up cmdline for kdump kernel failed\n");
+ ret = -EINVAL;
+ goto out;
+ }
+ cmdline = modified_cmdline;
+ }
+
if (initrd != NULL) {
kbuf.buffer = initrd;
kbuf.bufsz = kbuf.memsz = initrd_len;
@@ -88,7 +116,8 @@
goto out;
}
- ret = setup_new_fdt(image, fdt, initrd_load_addr, initrd_len, cmdline);
+ ret = setup_new_fdt_ppc64(image, fdt, initrd_load_addr,
+ initrd_len, cmdline);
if (ret)
goto out;
@@ -107,12 +136,13 @@
pr_debug("Loaded device tree at 0x%lx\n", fdt_load_addr);
slave_code = elf_info.buffer + elf_info.proghdrs[0].p_offset;
- ret = setup_purgatory(image, slave_code, fdt, kernel_load_addr,
- fdt_load_addr);
+ ret = setup_purgatory_ppc64(image, slave_code, fdt, kernel_load_addr,
+ fdt_load_addr);
if (ret)
pr_err("Error setting up the purgatory.\n");
out:
+ kfree(modified_cmdline);
kexec_free_elf_info(&elf_info);
/* Make kimage_file_post_load_cleanup free the fdt buffer for us. */
diff --git a/arch/powerpc/kexec/file_load.c b/arch/powerpc/kexec/file_load.c
index 143c917..9a232bc 100644
--- a/arch/powerpc/kexec/file_load.c
+++ b/arch/powerpc/kexec/file_load.c
@@ -1,6 +1,6 @@
// SPDX-License-Identifier: GPL-2.0-only
/*
- * ppc64 code to implement the kexec_file_load syscall
+ * powerpc code to implement the kexec_file_load syscall
*
* Copyright (C) 2004 Adam Litke (agl@us.ibm.com)
* Copyright (C) 2004 IBM Corp.
@@ -18,23 +18,45 @@
#include <linux/kexec.h>
#include <linux/of_fdt.h>
#include <linux/libfdt.h>
+#include <asm/setup.h>
#include <asm/ima.h>
-#define SLAVE_CODE_SIZE 256
+#define SLAVE_CODE_SIZE 256 /* First 0x100 bytes */
-const struct kexec_file_ops * const kexec_file_loaders[] = {
- &kexec_elf64_ops,
- NULL
-};
-
-int arch_kexec_kernel_image_probe(struct kimage *image, void *buf,
- unsigned long buf_len)
+/**
+ * setup_kdump_cmdline - Prepend "elfcorehdr=<addr> " to command line
+ * of kdump kernel for exporting the core.
+ * @image: Kexec image
+ * @cmdline: Command line parameters to update.
+ * @cmdline_len: Length of the cmdline parameters.
+ *
+ * kdump segment must be setup before calling this function.
+ *
+ * Returns new cmdline buffer for kdump kernel on success, NULL otherwise.
+ */
+char *setup_kdump_cmdline(struct kimage *image, char *cmdline,
+ unsigned long cmdline_len)
{
- /* We don't support crash kernels yet. */
- if (image->type == KEXEC_TYPE_CRASH)
- return -EOPNOTSUPP;
+ int elfcorehdr_strlen;
+ char *cmdline_ptr;
- return kexec_image_probe_default(image, buf, buf_len);
+ cmdline_ptr = kzalloc(COMMAND_LINE_SIZE, GFP_KERNEL);
+ if (!cmdline_ptr)
+ return NULL;
+
+ elfcorehdr_strlen = sprintf(cmdline_ptr, "elfcorehdr=0x%lx ",
+ image->arch.elfcorehdr_addr);
+
+ if (elfcorehdr_strlen + cmdline_len > COMMAND_LINE_SIZE) {
+ pr_err("Appending elfcorehdr=<addr> exceeds cmdline size\n");
+ kfree(cmdline_ptr);
+ return NULL;
+ }
+
+ memcpy(cmdline_ptr + elfcorehdr_strlen, cmdline, cmdline_len);
+ // Ensure it's nul terminated
+ cmdline_ptr[COMMAND_LINE_SIZE - 1] = '\0';
+ return cmdline_ptr;
}
/**
@@ -236,6 +258,20 @@
}
}
+ if (image->type == KEXEC_TYPE_CRASH) {
+ /*
+ * Avoid elfcorehdr from being stomped on in kdump kernel by
+ * setting up memory reserve map.
+ */
+ ret = fdt_add_mem_rsv(fdt, image->arch.elfcorehdr_addr,
+ image->arch.elf_headers_sz);
+ if (ret) {
+ pr_err("Error reserving elfcorehdr memory: %s\n",
+ fdt_strerror(ret));
+ goto err;
+ }
+ }
+
ret = setup_ima_buffer(image, fdt, chosen_node);
if (ret) {
pr_err("Error setting up the new device tree.\n");
diff --git a/arch/powerpc/kexec/file_load_64.c b/arch/powerpc/kexec/file_load_64.c
new file mode 100644
index 0000000..53bb71e
--- /dev/null
+++ b/arch/powerpc/kexec/file_load_64.c
@@ -0,0 +1,1119 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * ppc64 code to implement the kexec_file_load syscall
+ *
+ * Copyright (C) 2004 Adam Litke (agl@us.ibm.com)
+ * Copyright (C) 2004 IBM Corp.
+ * Copyright (C) 2004,2005 Milton D Miller II, IBM Corporation
+ * Copyright (C) 2005 R Sharada (sharada@in.ibm.com)
+ * Copyright (C) 2006 Mohan Kumar M (mohan@in.ibm.com)
+ * Copyright (C) 2020 IBM Corporation
+ *
+ * Based on kexec-tools' kexec-ppc64.c, kexec-elf-rel-ppc64.c, fs2dt.c.
+ * Heavily modified for the kernel by
+ * Hari Bathini, IBM Corporation.
+ */
+
+#include <linux/kexec.h>
+#include <linux/of_fdt.h>
+#include <linux/libfdt.h>
+#include <linux/of_device.h>
+#include <linux/memblock.h>
+#include <linux/slab.h>
+#include <linux/vmalloc.h>
+#include <asm/drmem.h>
+#include <asm/kexec_ranges.h>
+#include <asm/crashdump-ppc64.h>
+
+struct umem_info {
+ u64 *buf; /* data buffer for usable-memory property */
+ u32 size; /* size allocated for the data buffer */
+ u32 max_entries; /* maximum no. of entries */
+ u32 idx; /* index of current entry */
+
+ /* usable memory ranges to look up */
+ unsigned int nr_ranges;
+ const struct crash_mem_range *ranges;
+};
+
+const struct kexec_file_ops * const kexec_file_loaders[] = {
+ &kexec_elf64_ops,
+ NULL
+};
+
+/**
+ * get_exclude_memory_ranges - Get exclude memory ranges. This list includes
+ * regions like opal/rtas, tce-table, initrd,
+ * kernel, htab which should be avoided while
+ * setting up kexec load segments.
+ * @mem_ranges: Range list to add the memory ranges to.
+ *
+ * Returns 0 on success, negative errno on error.
+ */
+static int get_exclude_memory_ranges(struct crash_mem **mem_ranges)
+{
+ int ret;
+
+ ret = add_tce_mem_ranges(mem_ranges);
+ if (ret)
+ goto out;
+
+ ret = add_initrd_mem_range(mem_ranges);
+ if (ret)
+ goto out;
+
+ ret = add_htab_mem_range(mem_ranges);
+ if (ret)
+ goto out;
+
+ ret = add_kernel_mem_range(mem_ranges);
+ if (ret)
+ goto out;
+
+ ret = add_rtas_mem_range(mem_ranges);
+ if (ret)
+ goto out;
+
+ ret = add_opal_mem_range(mem_ranges);
+ if (ret)
+ goto out;
+
+ ret = add_reserved_mem_ranges(mem_ranges);
+ if (ret)
+ goto out;
+
+ /* exclude memory ranges should be sorted for easy lookup */
+ sort_memory_ranges(*mem_ranges, true);
+out:
+ if (ret)
+ pr_err("Failed to setup exclude memory ranges\n");
+ return ret;
+}
+
+/**
+ * get_usable_memory_ranges - Get usable memory ranges. This list includes
+ * regions like crashkernel, opal/rtas & tce-table,
+ * that kdump kernel could use.
+ * @mem_ranges: Range list to add the memory ranges to.
+ *
+ * Returns 0 on success, negative errno on error.
+ */
+static int get_usable_memory_ranges(struct crash_mem **mem_ranges)
+{
+ int ret;
+
+ /*
+ * Early boot failure observed on guests when low memory (first memory
+ * block?) is not added to usable memory. So, add [0, crashk_res.end]
+ * instead of [crashk_res.start, crashk_res.end] to workaround it.
+ * Also, crashed kernel's memory must be added to reserve map to
+ * avoid kdump kernel from using it.
+ */
+ ret = add_mem_range(mem_ranges, 0, crashk_res.end + 1);
+ if (ret)
+ goto out;
+
+ ret = add_rtas_mem_range(mem_ranges);
+ if (ret)
+ goto out;
+
+ ret = add_opal_mem_range(mem_ranges);
+ if (ret)
+ goto out;
+
+ ret = add_tce_mem_ranges(mem_ranges);
+out:
+ if (ret)
+ pr_err("Failed to setup usable memory ranges\n");
+ return ret;
+}
+
+/**
+ * get_crash_memory_ranges - Get crash memory ranges. This list includes
+ * first/crashing kernel's memory regions that
+ * would be exported via an elfcore.
+ * @mem_ranges: Range list to add the memory ranges to.
+ *
+ * Returns 0 on success, negative errno on error.
+ */
+static int get_crash_memory_ranges(struct crash_mem **mem_ranges)
+{
+ struct memblock_region *reg;
+ struct crash_mem *tmem;
+ int ret;
+
+ for_each_memblock(memory, reg) {
+ u64 base, size;
+
+ base = (u64)reg->base;
+ size = (u64)reg->size;
+
+ /* Skip backup memory region, which needs a separate entry */
+ if (base == BACKUP_SRC_START) {
+ if (size > BACKUP_SRC_SIZE) {
+ base = BACKUP_SRC_END + 1;
+ size -= BACKUP_SRC_SIZE;
+ } else
+ continue;
+ }
+
+ ret = add_mem_range(mem_ranges, base, size);
+ if (ret)
+ goto out;
+
+ /* Try merging adjacent ranges before reallocation attempt */
+ if ((*mem_ranges)->nr_ranges == (*mem_ranges)->max_nr_ranges)
+ sort_memory_ranges(*mem_ranges, true);
+ }
+
+ /* Reallocate memory ranges if there is no space to split ranges */
+ tmem = *mem_ranges;
+ if (tmem && (tmem->nr_ranges == tmem->max_nr_ranges)) {
+ tmem = realloc_mem_ranges(mem_ranges);
+ if (!tmem)
+ goto out;
+ }
+
+ /* Exclude crashkernel region */
+ ret = crash_exclude_mem_range(tmem, crashk_res.start, crashk_res.end);
+ if (ret)
+ goto out;
+
+ /*
+ * FIXME: For now, stay in parity with kexec-tools but if RTAS/OPAL
+ * regions are exported to save their context at the time of
+ * crash, they should actually be backed up just like the
+ * first 64K bytes of memory.
+ */
+ ret = add_rtas_mem_range(mem_ranges);
+ if (ret)
+ goto out;
+
+ ret = add_opal_mem_range(mem_ranges);
+ if (ret)
+ goto out;
+
+ /* create a separate program header for the backup region */
+ ret = add_mem_range(mem_ranges, BACKUP_SRC_START, BACKUP_SRC_SIZE);
+ if (ret)
+ goto out;
+
+ sort_memory_ranges(*mem_ranges, false);
+out:
+ if (ret)
+ pr_err("Failed to setup crash memory ranges\n");
+ return ret;
+}
+
+/**
+ * get_reserved_memory_ranges - Get reserve memory ranges. This list includes
+ * memory regions that should be added to the
+ * memory reserve map to ensure the region is
+ * protected from any mischief.
+ * @mem_ranges: Range list to add the memory ranges to.
+ *
+ * Returns 0 on success, negative errno on error.
+ */
+static int get_reserved_memory_ranges(struct crash_mem **mem_ranges)
+{
+ int ret;
+
+ ret = add_rtas_mem_range(mem_ranges);
+ if (ret)
+ goto out;
+
+ ret = add_tce_mem_ranges(mem_ranges);
+ if (ret)
+ goto out;
+
+ ret = add_reserved_mem_ranges(mem_ranges);
+out:
+ if (ret)
+ pr_err("Failed to setup reserved memory ranges\n");
+ return ret;
+}
+
+/**
+ * __locate_mem_hole_top_down - Looks top down for a large enough memory hole
+ * in the memory regions between buf_min & buf_max
+ * for the buffer. If found, sets kbuf->mem.
+ * @kbuf: Buffer contents and memory parameters.
+ * @buf_min: Minimum address for the buffer.
+ * @buf_max: Maximum address for the buffer.
+ *
+ * Returns 0 on success, negative errno on error.
+ */
+static int __locate_mem_hole_top_down(struct kexec_buf *kbuf,
+ u64 buf_min, u64 buf_max)
+{
+ int ret = -EADDRNOTAVAIL;
+ phys_addr_t start, end;
+ u64 i;
+
+ for_each_mem_range_rev(i, &memblock.memory, NULL, NUMA_NO_NODE,
+ MEMBLOCK_NONE, &start, &end, NULL) {
+ /*
+ * memblock uses [start, end) convention while it is
+ * [start, end] here. Fix the off-by-one to have the
+ * same convention.
+ */
+ end -= 1;
+
+ if (start > buf_max)
+ continue;
+
+ /* Memory hole not found */
+ if (end < buf_min)
+ break;
+
+ /* Adjust memory region based on the given range */
+ if (start < buf_min)
+ start = buf_min;
+ if (end > buf_max)
+ end = buf_max;
+
+ start = ALIGN(start, kbuf->buf_align);
+ if (start < end && (end - start + 1) >= kbuf->memsz) {
+ /* Suitable memory range found. Set kbuf->mem */
+ kbuf->mem = ALIGN_DOWN(end - kbuf->memsz + 1,
+ kbuf->buf_align);
+ ret = 0;
+ break;
+ }
+ }
+
+ return ret;
+}
+
+/**
+ * locate_mem_hole_top_down_ppc64 - Skip special memory regions to find a
+ * suitable buffer with top down approach.
+ * @kbuf: Buffer contents and memory parameters.
+ * @buf_min: Minimum address for the buffer.
+ * @buf_max: Maximum address for the buffer.
+ * @emem: Exclude memory ranges.
+ *
+ * Returns 0 on success, negative errno on error.
+ */
+static int locate_mem_hole_top_down_ppc64(struct kexec_buf *kbuf,
+ u64 buf_min, u64 buf_max,
+ const struct crash_mem *emem)
+{
+ int i, ret = 0, err = -EADDRNOTAVAIL;
+ u64 start, end, tmin, tmax;
+
+ tmax = buf_max;
+ for (i = (emem->nr_ranges - 1); i >= 0; i--) {
+ start = emem->ranges[i].start;
+ end = emem->ranges[i].end;
+
+ if (start > tmax)
+ continue;
+
+ if (end < tmax) {
+ tmin = (end < buf_min ? buf_min : end + 1);
+ ret = __locate_mem_hole_top_down(kbuf, tmin, tmax);
+ if (!ret)
+ return 0;
+ }
+
+ tmax = start - 1;
+
+ if (tmax < buf_min) {
+ ret = err;
+ break;
+ }
+ ret = 0;
+ }
+
+ if (!ret) {
+ tmin = buf_min;
+ ret = __locate_mem_hole_top_down(kbuf, tmin, tmax);
+ }
+ return ret;
+}
+
+/**
+ * __locate_mem_hole_bottom_up - Looks bottom up for a large enough memory hole
+ * in the memory regions between buf_min & buf_max
+ * for the buffer. If found, sets kbuf->mem.
+ * @kbuf: Buffer contents and memory parameters.
+ * @buf_min: Minimum address for the buffer.
+ * @buf_max: Maximum address for the buffer.
+ *
+ * Returns 0 on success, negative errno on error.
+ */
+static int __locate_mem_hole_bottom_up(struct kexec_buf *kbuf,
+ u64 buf_min, u64 buf_max)
+{
+ int ret = -EADDRNOTAVAIL;
+ phys_addr_t start, end;
+ u64 i;
+
+ for_each_mem_range(i, &memblock.memory, NULL, NUMA_NO_NODE,
+ MEMBLOCK_NONE, &start, &end, NULL) {
+ /*
+ * memblock uses [start, end) convention while it is
+ * [start, end] here. Fix the off-by-one to have the
+ * same convention.
+ */
+ end -= 1;
+
+ if (end < buf_min)
+ continue;
+
+ /* Memory hole not found */
+ if (start > buf_max)
+ break;
+
+ /* Adjust memory region based on the given range */
+ if (start < buf_min)
+ start = buf_min;
+ if (end > buf_max)
+ end = buf_max;
+
+ start = ALIGN(start, kbuf->buf_align);
+ if (start < end && (end - start + 1) >= kbuf->memsz) {
+ /* Suitable memory range found. Set kbuf->mem */
+ kbuf->mem = start;
+ ret = 0;
+ break;
+ }
+ }
+
+ return ret;
+}
+
+/**
+ * locate_mem_hole_bottom_up_ppc64 - Skip special memory regions to find a
+ * suitable buffer with bottom up approach.
+ * @kbuf: Buffer contents and memory parameters.
+ * @buf_min: Minimum address for the buffer.
+ * @buf_max: Maximum address for the buffer.
+ * @emem: Exclude memory ranges.
+ *
+ * Returns 0 on success, negative errno on error.
+ */
+static int locate_mem_hole_bottom_up_ppc64(struct kexec_buf *kbuf,
+ u64 buf_min, u64 buf_max,
+ const struct crash_mem *emem)
+{
+ int i, ret = 0, err = -EADDRNOTAVAIL;
+ u64 start, end, tmin, tmax;
+
+ tmin = buf_min;
+ for (i = 0; i < emem->nr_ranges; i++) {
+ start = emem->ranges[i].start;
+ end = emem->ranges[i].end;
+
+ if (end < tmin)
+ continue;
+
+ if (start > tmin) {
+ tmax = (start > buf_max ? buf_max : start - 1);
+ ret = __locate_mem_hole_bottom_up(kbuf, tmin, tmax);
+ if (!ret)
+ return 0;
+ }
+
+ tmin = end + 1;
+
+ if (tmin > buf_max) {
+ ret = err;
+ break;
+ }
+ ret = 0;
+ }
+
+ if (!ret) {
+ tmax = buf_max;
+ ret = __locate_mem_hole_bottom_up(kbuf, tmin, tmax);
+ }
+ return ret;
+}
+
+/**
+ * check_realloc_usable_mem - Reallocate buffer if it can't accommodate entries
+ * @um_info: Usable memory buffer and ranges info.
+ * @cnt: No. of entries to accommodate.
+ *
+ * Frees up the old buffer if memory reallocation fails.
+ *
+ * Returns buffer on success, NULL on error.
+ */
+static u64 *check_realloc_usable_mem(struct umem_info *um_info, int cnt)
+{
+ u32 new_size;
+ u64 *tbuf;
+
+ if ((um_info->idx + cnt) <= um_info->max_entries)
+ return um_info->buf;
+
+ new_size = um_info->size + MEM_RANGE_CHUNK_SZ;
+ tbuf = krealloc(um_info->buf, new_size, GFP_KERNEL);
+ if (tbuf) {
+ um_info->buf = tbuf;
+ um_info->size = new_size;
+ um_info->max_entries = (um_info->size / sizeof(u64));
+ }
+
+ return tbuf;
+}
+
+/**
+ * add_usable_mem - Add the usable memory ranges within the given memory range
+ * to the buffer
+ * @um_info: Usable memory buffer and ranges info.
+ * @base: Base address of memory range to look for.
+ * @end: End address of memory range to look for.
+ *
+ * Returns 0 on success, negative errno on error.
+ */
+static int add_usable_mem(struct umem_info *um_info, u64 base, u64 end)
+{
+ u64 loc_base, loc_end;
+ bool add;
+ int i;
+
+ for (i = 0; i < um_info->nr_ranges; i++) {
+ add = false;
+ loc_base = um_info->ranges[i].start;
+ loc_end = um_info->ranges[i].end;
+ if (loc_base >= base && loc_end <= end)
+ add = true;
+ else if (base < loc_end && end > loc_base) {
+ if (loc_base < base)
+ loc_base = base;
+ if (loc_end > end)
+ loc_end = end;
+ add = true;
+ }
+
+ if (add) {
+ if (!check_realloc_usable_mem(um_info, 2))
+ return -ENOMEM;
+
+ um_info->buf[um_info->idx++] = cpu_to_be64(loc_base);
+ um_info->buf[um_info->idx++] =
+ cpu_to_be64(loc_end - loc_base + 1);
+ }
+ }
+
+ return 0;
+}
+
+/**
+ * kdump_setup_usable_lmb - This is a callback function that gets called by
+ * walk_drmem_lmbs for every LMB to set its
+ * usable memory ranges.
+ * @lmb: LMB info.
+ * @usm: linux,drconf-usable-memory property value.
+ * @data: Pointer to usable memory buffer and ranges info.
+ *
+ * Returns 0 on success, negative errno on error.
+ */
+static int kdump_setup_usable_lmb(struct drmem_lmb *lmb, const __be32 **usm,
+ void *data)
+{
+ struct umem_info *um_info;
+ int tmp_idx, ret;
+ u64 base, end;
+
+ /*
+ * kdump load isn't supported on kernels already booted with
+ * linux,drconf-usable-memory property.
+ */
+ if (*usm) {
+ pr_err("linux,drconf-usable-memory property already exists!");
+ return -EINVAL;
+ }
+
+ um_info = data;
+ tmp_idx = um_info->idx;
+ if (!check_realloc_usable_mem(um_info, 1))
+ return -ENOMEM;
+
+ um_info->idx++;
+ base = lmb->base_addr;
+ end = base + drmem_lmb_size() - 1;
+ ret = add_usable_mem(um_info, base, end);
+ if (!ret) {
+ /*
+ * Update the no. of ranges added. Two entries (base & size)
+ * for every range added.
+ */
+ um_info->buf[tmp_idx] =
+ cpu_to_be64((um_info->idx - tmp_idx - 1) / 2);
+ }
+
+ return ret;
+}
+
+#define NODE_PATH_LEN 256
+/**
+ * add_usable_mem_property - Add usable memory property for the given
+ * memory node.
+ * @fdt: Flattened device tree for the kdump kernel.
+ * @dn: Memory node.
+ * @um_info: Usable memory buffer and ranges info.
+ *
+ * Returns 0 on success, negative errno on error.
+ */
+static int add_usable_mem_property(void *fdt, struct device_node *dn,
+ struct umem_info *um_info)
+{
+ int n_mem_addr_cells, n_mem_size_cells, node;
+ char path[NODE_PATH_LEN];
+ int i, len, ranges, ret;
+ const __be32 *prop;
+ u64 base, end;
+
+ of_node_get(dn);
+
+ if (snprintf(path, NODE_PATH_LEN, "%pOF", dn) > (NODE_PATH_LEN - 1)) {
+ pr_err("Buffer (%d) too small for memory node: %pOF\n",
+ NODE_PATH_LEN, dn);
+ return -EOVERFLOW;
+ }
+ pr_debug("Memory node path: %s\n", path);
+
+ /* Now that we know the path, find its offset in kdump kernel's fdt */
+ node = fdt_path_offset(fdt, path);
+ if (node < 0) {
+ pr_err("Malformed device tree: error reading %s\n", path);
+ ret = -EINVAL;
+ goto out;
+ }
+
+ /* Get the address & size cells */
+ n_mem_addr_cells = of_n_addr_cells(dn);
+ n_mem_size_cells = of_n_size_cells(dn);
+ pr_debug("address cells: %d, size cells: %d\n", n_mem_addr_cells,
+ n_mem_size_cells);
+
+ um_info->idx = 0;
+ if (!check_realloc_usable_mem(um_info, 2)) {
+ ret = -ENOMEM;
+ goto out;
+ }
+
+ prop = of_get_property(dn, "reg", &len);
+ if (!prop || len <= 0) {
+ ret = 0;
+ goto out;
+ }
+
+ /*
+ * "reg" property represents sequence of (addr,size) tuples
+ * each representing a memory range.
+ */
+ ranges = (len >> 2) / (n_mem_addr_cells + n_mem_size_cells);
+
+ for (i = 0; i < ranges; i++) {
+ base = of_read_number(prop, n_mem_addr_cells);
+ prop += n_mem_addr_cells;
+ end = base + of_read_number(prop, n_mem_size_cells) - 1;
+ prop += n_mem_size_cells;
+
+ ret = add_usable_mem(um_info, base, end);
+ if (ret)
+ goto out;
+ }
+
+ /*
+ * No kdump kernel usable memory found in this memory node.
+ * Write (0,0) tuple in linux,usable-memory property for
+ * this region to be ignored.
+ */
+ if (um_info->idx == 0) {
+ um_info->buf[0] = 0;
+ um_info->buf[1] = 0;
+ um_info->idx = 2;
+ }
+
+ ret = fdt_setprop(fdt, node, "linux,usable-memory", um_info->buf,
+ (um_info->idx * sizeof(u64)));
+
+out:
+ of_node_put(dn);
+ return ret;
+}
+
+
+/**
+ * update_usable_mem_fdt - Updates kdump kernel's fdt with linux,usable-memory
+ * and linux,drconf-usable-memory DT properties as
+ * appropriate to restrict its memory usage.
+ * @fdt: Flattened device tree for the kdump kernel.
+ * @usable_mem: Usable memory ranges for kdump kernel.
+ *
+ * Returns 0 on success, negative errno on error.
+ */
+static int update_usable_mem_fdt(void *fdt, struct crash_mem *usable_mem)
+{
+ struct umem_info um_info;
+ struct device_node *dn;
+ int node, ret = 0;
+
+ if (!usable_mem) {
+ pr_err("Usable memory ranges for kdump kernel not found\n");
+ return -ENOENT;
+ }
+
+ node = fdt_path_offset(fdt, "/ibm,dynamic-reconfiguration-memory");
+ if (node == -FDT_ERR_NOTFOUND)
+ pr_debug("No dynamic reconfiguration memory found\n");
+ else if (node < 0) {
+ pr_err("Malformed device tree: error reading /ibm,dynamic-reconfiguration-memory.\n");
+ return -EINVAL;
+ }
+
+ um_info.buf = NULL;
+ um_info.size = 0;
+ um_info.max_entries = 0;
+ um_info.idx = 0;
+ /* Memory ranges to look up */
+ um_info.ranges = &(usable_mem->ranges[0]);
+ um_info.nr_ranges = usable_mem->nr_ranges;
+
+ dn = of_find_node_by_path("/ibm,dynamic-reconfiguration-memory");
+ if (dn) {
+ ret = walk_drmem_lmbs(dn, &um_info, kdump_setup_usable_lmb);
+ of_node_put(dn);
+
+ if (ret) {
+ pr_err("Could not setup linux,drconf-usable-memory property for kdump\n");
+ goto out;
+ }
+
+ ret = fdt_setprop(fdt, node, "linux,drconf-usable-memory",
+ um_info.buf, (um_info.idx * sizeof(u64)));
+ if (ret) {
+ pr_err("Failed to update fdt with linux,drconf-usable-memory property");
+ goto out;
+ }
+ }
+
+ /*
+ * Walk through each memory node and set linux,usable-memory property
+ * for the corresponding node in kdump kernel's fdt.
+ */
+ for_each_node_by_type(dn, "memory") {
+ ret = add_usable_mem_property(fdt, dn, &um_info);
+ if (ret) {
+ pr_err("Failed to set linux,usable-memory property for %s node",
+ dn->full_name);
+ goto out;
+ }
+ }
+
+out:
+ kfree(um_info.buf);
+ return ret;
+}
+
+/**
+ * load_backup_segment - Locate a memory hole to place the backup region.
+ * @image: Kexec image.
+ * @kbuf: Buffer contents and memory parameters.
+ *
+ * Returns 0 on success, negative errno on error.
+ */
+static int load_backup_segment(struct kimage *image, struct kexec_buf *kbuf)
+{
+ void *buf;
+ int ret;
+
+ /*
+ * Setup a source buffer for backup segment.
+ *
+ * A source buffer has no meaning for backup region as data will
+ * be copied from backup source, after crash, in the purgatory.
+ * But as load segment code doesn't recognize such segments,
+ * setup a dummy source buffer to keep it happy for now.
+ */
+ buf = vzalloc(BACKUP_SRC_SIZE);
+ if (!buf)
+ return -ENOMEM;
+
+ kbuf->buffer = buf;
+ kbuf->mem = KEXEC_BUF_MEM_UNKNOWN;
+ kbuf->bufsz = kbuf->memsz = BACKUP_SRC_SIZE;
+ kbuf->top_down = false;
+
+ ret = kexec_add_buffer(kbuf);
+ if (ret) {
+ vfree(buf);
+ return ret;
+ }
+
+ image->arch.backup_buf = buf;
+ image->arch.backup_start = kbuf->mem;
+ return 0;
+}
+
+/**
+ * update_backup_region_phdr - Update backup region's offset for the core to
+ * export the region appropriately.
+ * @image: Kexec image.
+ * @ehdr: ELF core header.
+ *
+ * Assumes an exclusive program header is setup for the backup region
+ * in the ELF headers
+ *
+ * Returns nothing.
+ */
+static void update_backup_region_phdr(struct kimage *image, Elf64_Ehdr *ehdr)
+{
+ Elf64_Phdr *phdr;
+ unsigned int i;
+
+ phdr = (Elf64_Phdr *)(ehdr + 1);
+ for (i = 0; i < ehdr->e_phnum; i++) {
+ if (phdr->p_paddr == BACKUP_SRC_START) {
+ phdr->p_offset = image->arch.backup_start;
+ pr_debug("Backup region offset updated to 0x%lx\n",
+ image->arch.backup_start);
+ return;
+ }
+ }
+}
+
+/**
+ * load_elfcorehdr_segment - Setup crash memory ranges and initialize elfcorehdr
+ * segment needed to load kdump kernel.
+ * @image: Kexec image.
+ * @kbuf: Buffer contents and memory parameters.
+ *
+ * Returns 0 on success, negative errno on error.
+ */
+static int load_elfcorehdr_segment(struct kimage *image, struct kexec_buf *kbuf)
+{
+ struct crash_mem *cmem = NULL;
+ unsigned long headers_sz;
+ void *headers = NULL;
+ int ret;
+
+ ret = get_crash_memory_ranges(&cmem);
+ if (ret)
+ goto out;
+
+ /* Setup elfcorehdr segment */
+ ret = crash_prepare_elf64_headers(cmem, false, &headers, &headers_sz);
+ if (ret) {
+ pr_err("Failed to prepare elf headers for the core\n");
+ goto out;
+ }
+
+ /* Fix the offset for backup region in the ELF header */
+ update_backup_region_phdr(image, headers);
+
+ kbuf->buffer = headers;
+ kbuf->mem = KEXEC_BUF_MEM_UNKNOWN;
+ kbuf->bufsz = kbuf->memsz = headers_sz;
+ kbuf->top_down = false;
+
+ ret = kexec_add_buffer(kbuf);
+ if (ret) {
+ vfree(headers);
+ goto out;
+ }
+
+ image->arch.elfcorehdr_addr = kbuf->mem;
+ image->arch.elf_headers_sz = headers_sz;
+ image->arch.elf_headers = headers;
+out:
+ kfree(cmem);
+ return ret;
+}
+
+/**
+ * load_crashdump_segments_ppc64 - Initialize the additional segements needed
+ * to load kdump kernel.
+ * @image: Kexec image.
+ * @kbuf: Buffer contents and memory parameters.
+ *
+ * Returns 0 on success, negative errno on error.
+ */
+int load_crashdump_segments_ppc64(struct kimage *image,
+ struct kexec_buf *kbuf)
+{
+ int ret;
+
+ /* Load backup segment - first 64K bytes of the crashing kernel */
+ ret = load_backup_segment(image, kbuf);
+ if (ret) {
+ pr_err("Failed to load backup segment\n");
+ return ret;
+ }
+ pr_debug("Loaded the backup region at 0x%lx\n", kbuf->mem);
+
+ /* Load elfcorehdr segment - to export crashing kernel's vmcore */
+ ret = load_elfcorehdr_segment(image, kbuf);
+ if (ret) {
+ pr_err("Failed to load elfcorehdr segment\n");
+ return ret;
+ }
+ pr_debug("Loaded elf core header at 0x%lx, bufsz=0x%lx memsz=0x%lx\n",
+ image->arch.elfcorehdr_addr, kbuf->bufsz, kbuf->memsz);
+
+ return 0;
+}
+
+/**
+ * setup_purgatory_ppc64 - initialize PPC64 specific purgatory's global
+ * variables and call setup_purgatory() to initialize
+ * common global variable.
+ * @image: kexec image.
+ * @slave_code: Slave code for the purgatory.
+ * @fdt: Flattened device tree for the next kernel.
+ * @kernel_load_addr: Address where the kernel is loaded.
+ * @fdt_load_addr: Address where the flattened device tree is loaded.
+ *
+ * Returns 0 on success, negative errno on error.
+ */
+int setup_purgatory_ppc64(struct kimage *image, const void *slave_code,
+ const void *fdt, unsigned long kernel_load_addr,
+ unsigned long fdt_load_addr)
+{
+ struct device_node *dn = NULL;
+ int ret;
+
+ ret = setup_purgatory(image, slave_code, fdt, kernel_load_addr,
+ fdt_load_addr);
+ if (ret)
+ goto out;
+
+ if (image->type == KEXEC_TYPE_CRASH) {
+ u32 my_run_at_load = 1;
+
+ /*
+ * Tell relocatable kernel to run at load address
+ * via the word meant for that at 0x5c.
+ */
+ ret = kexec_purgatory_get_set_symbol(image, "run_at_load",
+ &my_run_at_load,
+ sizeof(my_run_at_load),
+ false);
+ if (ret)
+ goto out;
+ }
+
+ /* Tell purgatory where to look for backup region */
+ ret = kexec_purgatory_get_set_symbol(image, "backup_start",
+ &image->arch.backup_start,
+ sizeof(image->arch.backup_start),
+ false);
+ if (ret)
+ goto out;
+
+ /* Setup OPAL base & entry values */
+ dn = of_find_node_by_path("/ibm,opal");
+ if (dn) {
+ u64 val;
+
+ of_property_read_u64(dn, "opal-base-address", &val);
+ ret = kexec_purgatory_get_set_symbol(image, "opal_base", &val,
+ sizeof(val), false);
+ if (ret)
+ goto out;
+
+ of_property_read_u64(dn, "opal-entry-address", &val);
+ ret = kexec_purgatory_get_set_symbol(image, "opal_entry", &val,
+ sizeof(val), false);
+ }
+out:
+ if (ret)
+ pr_err("Failed to setup purgatory symbols");
+ of_node_put(dn);
+ return ret;
+}
+
+/**
+ * setup_new_fdt_ppc64 - Update the flattend device-tree of the kernel
+ * being loaded.
+ * @image: kexec image being loaded.
+ * @fdt: Flattened device tree for the next kernel.
+ * @initrd_load_addr: Address where the next initrd will be loaded.
+ * @initrd_len: Size of the next initrd, or 0 if there will be none.
+ * @cmdline: Command line for the next kernel, or NULL if there will
+ * be none.
+ *
+ * Returns 0 on success, negative errno on error.
+ */
+int setup_new_fdt_ppc64(const struct kimage *image, void *fdt,
+ unsigned long initrd_load_addr,
+ unsigned long initrd_len, const char *cmdline)
+{
+ struct crash_mem *umem = NULL, *rmem = NULL;
+ int i, nr_ranges, ret;
+
+ ret = setup_new_fdt(image, fdt, initrd_load_addr, initrd_len, cmdline);
+ if (ret)
+ goto out;
+
+ /*
+ * Restrict memory usage for kdump kernel by setting up
+ * usable memory ranges and memory reserve map.
+ */
+ if (image->type == KEXEC_TYPE_CRASH) {
+ ret = get_usable_memory_ranges(&umem);
+ if (ret)
+ goto out;
+
+ ret = update_usable_mem_fdt(fdt, umem);
+ if (ret) {
+ pr_err("Error setting up usable-memory property for kdump kernel\n");
+ goto out;
+ }
+
+ /*
+ * Ensure we don't touch crashed kernel's memory except the
+ * first 64K of RAM, which will be backed up.
+ */
+ ret = fdt_add_mem_rsv(fdt, BACKUP_SRC_END + 1,
+ crashk_res.start - BACKUP_SRC_SIZE);
+ if (ret) {
+ pr_err("Error reserving crash memory: %s\n",
+ fdt_strerror(ret));
+ goto out;
+ }
+
+ /* Ensure backup region is not used by kdump/capture kernel */
+ ret = fdt_add_mem_rsv(fdt, image->arch.backup_start,
+ BACKUP_SRC_SIZE);
+ if (ret) {
+ pr_err("Error reserving memory for backup: %s\n",
+ fdt_strerror(ret));
+ goto out;
+ }
+ }
+
+ /* Update memory reserve map */
+ ret = get_reserved_memory_ranges(&rmem);
+ if (ret)
+ goto out;
+
+ nr_ranges = rmem ? rmem->nr_ranges : 0;
+ for (i = 0; i < nr_ranges; i++) {
+ u64 base, size;
+
+ base = rmem->ranges[i].start;
+ size = rmem->ranges[i].end - base + 1;
+ ret = fdt_add_mem_rsv(fdt, base, size);
+ if (ret) {
+ pr_err("Error updating memory reserve map: %s\n",
+ fdt_strerror(ret));
+ goto out;
+ }
+ }
+
+out:
+ kfree(rmem);
+ kfree(umem);
+ return ret;
+}
+
+/**
+ * arch_kexec_locate_mem_hole - Skip special memory regions like rtas, opal,
+ * tce-table, reserved-ranges & such (exclude
+ * memory ranges) as they can't be used for kexec
+ * segment buffer. Sets kbuf->mem when a suitable
+ * memory hole is found.
+ * @kbuf: Buffer contents and memory parameters.
+ *
+ * Assumes minimum of PAGE_SIZE alignment for kbuf->memsz & kbuf->buf_align.
+ *
+ * Returns 0 on success, negative errno on error.
+ */
+int arch_kexec_locate_mem_hole(struct kexec_buf *kbuf)
+{
+ struct crash_mem **emem;
+ u64 buf_min, buf_max;
+ int ret;
+
+ /* Look up the exclude ranges list while locating the memory hole */
+ emem = &(kbuf->image->arch.exclude_ranges);
+ if (!(*emem) || ((*emem)->nr_ranges == 0)) {
+ pr_warn("No exclude range list. Using the default locate mem hole method\n");
+ return kexec_locate_mem_hole(kbuf);
+ }
+
+ buf_min = kbuf->buf_min;
+ buf_max = kbuf->buf_max;
+ /* Segments for kdump kernel should be within crashkernel region */
+ if (kbuf->image->type == KEXEC_TYPE_CRASH) {
+ buf_min = (buf_min < crashk_res.start ?
+ crashk_res.start : buf_min);
+ buf_max = (buf_max > crashk_res.end ?
+ crashk_res.end : buf_max);
+ }
+
+ if (buf_min > buf_max) {
+ pr_err("Invalid buffer min and/or max values\n");
+ return -EINVAL;
+ }
+
+ if (kbuf->top_down)
+ ret = locate_mem_hole_top_down_ppc64(kbuf, buf_min, buf_max,
+ *emem);
+ else
+ ret = locate_mem_hole_bottom_up_ppc64(kbuf, buf_min, buf_max,
+ *emem);
+
+ /* Add the buffer allocated to the exclude list for the next lookup */
+ if (!ret) {
+ add_mem_range(emem, kbuf->mem, kbuf->memsz);
+ sort_memory_ranges(*emem, true);
+ } else {
+ pr_err("Failed to locate memory buffer of size %lu\n",
+ kbuf->memsz);
+ }
+ return ret;
+}
+
+/**
+ * arch_kexec_kernel_image_probe - Does additional handling needed to setup
+ * kexec segments.
+ * @image: kexec image being loaded.
+ * @buf: Buffer pointing to elf data.
+ * @buf_len: Length of the buffer.
+ *
+ * Returns 0 on success, negative errno on error.
+ */
+int arch_kexec_kernel_image_probe(struct kimage *image, void *buf,
+ unsigned long buf_len)
+{
+ int ret;
+
+ /* Get exclude memory ranges needed for setting up kexec segments */
+ ret = get_exclude_memory_ranges(&(image->arch.exclude_ranges));
+ if (ret) {
+ pr_err("Failed to setup exclude memory ranges for buffer lookup\n");
+ return ret;
+ }
+
+ return kexec_image_probe_default(image, buf, buf_len);
+}
+
+/**
+ * arch_kimage_file_post_load_cleanup - Frees up all the allocations done
+ * while loading the image.
+ * @image: kexec image being loaded.
+ *
+ * Returns 0 on success, negative errno on error.
+ */
+int arch_kimage_file_post_load_cleanup(struct kimage *image)
+{
+ kfree(image->arch.exclude_ranges);
+ image->arch.exclude_ranges = NULL;
+
+ vfree(image->arch.backup_buf);
+ image->arch.backup_buf = NULL;
+
+ vfree(image->arch.elf_headers);
+ image->arch.elf_headers = NULL;
+ image->arch.elf_headers_sz = 0;
+
+ return kexec_image_post_load_cleanup_default(image);
+}
diff --git a/arch/powerpc/kexec/ranges.c b/arch/powerpc/kexec/ranges.c
new file mode 100644
index 0000000..6b81c85
--- /dev/null
+++ b/arch/powerpc/kexec/ranges.c
@@ -0,0 +1,412 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * powerpc code to implement the kexec_file_load syscall
+ *
+ * Copyright (C) 2004 Adam Litke (agl@us.ibm.com)
+ * Copyright (C) 2004 IBM Corp.
+ * Copyright (C) 2004,2005 Milton D Miller II, IBM Corporation
+ * Copyright (C) 2005 R Sharada (sharada@in.ibm.com)
+ * Copyright (C) 2006 Mohan Kumar M (mohan@in.ibm.com)
+ * Copyright (C) 2020 IBM Corporation
+ *
+ * Based on kexec-tools' kexec-ppc64.c, fs2dt.c.
+ * Heavily modified for the kernel by
+ * Hari Bathini, IBM Corporation.
+ */
+
+#define pr_fmt(fmt) "kexec ranges: " fmt
+
+#include <linux/sort.h>
+#include <linux/kexec.h>
+#include <linux/of_device.h>
+#include <linux/slab.h>
+#include <asm/sections.h>
+#include <asm/kexec_ranges.h>
+
+/**
+ * get_max_nr_ranges - Get the max no. of ranges crash_mem structure
+ * could hold, given the size allocated for it.
+ * @size: Allocation size of crash_mem structure.
+ *
+ * Returns the maximum no. of ranges.
+ */
+static inline unsigned int get_max_nr_ranges(size_t size)
+{
+ return ((size - sizeof(struct crash_mem)) /
+ sizeof(struct crash_mem_range));
+}
+
+/**
+ * get_mem_rngs_size - Get the allocated size of mem_rngs based on
+ * max_nr_ranges and chunk size.
+ * @mem_rngs: Memory ranges.
+ *
+ * Returns the maximum size of @mem_rngs.
+ */
+static inline size_t get_mem_rngs_size(struct crash_mem *mem_rngs)
+{
+ size_t size;
+
+ if (!mem_rngs)
+ return 0;
+
+ size = (sizeof(struct crash_mem) +
+ (mem_rngs->max_nr_ranges * sizeof(struct crash_mem_range)));
+
+ /*
+ * Memory is allocated in size multiple of MEM_RANGE_CHUNK_SZ.
+ * So, align to get the actual length.
+ */
+ return ALIGN(size, MEM_RANGE_CHUNK_SZ);
+}
+
+/**
+ * __add_mem_range - add a memory range to memory ranges list.
+ * @mem_ranges: Range list to add the memory range to.
+ * @base: Base address of the range to add.
+ * @size: Size of the memory range to add.
+ *
+ * (Re)allocates memory, if needed.
+ *
+ * Returns 0 on success, negative errno on error.
+ */
+static int __add_mem_range(struct crash_mem **mem_ranges, u64 base, u64 size)
+{
+ struct crash_mem *mem_rngs = *mem_ranges;
+
+ if (!mem_rngs || (mem_rngs->nr_ranges == mem_rngs->max_nr_ranges)) {
+ mem_rngs = realloc_mem_ranges(mem_ranges);
+ if (!mem_rngs)
+ return -ENOMEM;
+ }
+
+ mem_rngs->ranges[mem_rngs->nr_ranges].start = base;
+ mem_rngs->ranges[mem_rngs->nr_ranges].end = base + size - 1;
+ pr_debug("Added memory range [%#016llx - %#016llx] at index %d\n",
+ base, base + size - 1, mem_rngs->nr_ranges);
+ mem_rngs->nr_ranges++;
+ return 0;
+}
+
+/**
+ * __merge_memory_ranges - Merges the given memory ranges list.
+ * @mem_rngs: Range list to merge.
+ *
+ * Assumes a sorted range list.
+ *
+ * Returns nothing.
+ */
+static void __merge_memory_ranges(struct crash_mem *mem_rngs)
+{
+ struct crash_mem_range *ranges;
+ int i, idx;
+
+ if (!mem_rngs)
+ return;
+
+ idx = 0;
+ ranges = &(mem_rngs->ranges[0]);
+ for (i = 1; i < mem_rngs->nr_ranges; i++) {
+ if (ranges[i].start <= (ranges[i-1].end + 1))
+ ranges[idx].end = ranges[i].end;
+ else {
+ idx++;
+ if (i == idx)
+ continue;
+
+ ranges[idx] = ranges[i];
+ }
+ }
+ mem_rngs->nr_ranges = idx + 1;
+}
+
+/* cmp_func_t callback to sort ranges with sort() */
+static int rngcmp(const void *_x, const void *_y)
+{
+ const struct crash_mem_range *x = _x, *y = _y;
+
+ if (x->start > y->start)
+ return 1;
+ if (x->start < y->start)
+ return -1;
+ return 0;
+}
+
+/**
+ * sort_memory_ranges - Sorts the given memory ranges list.
+ * @mem_rngs: Range list to sort.
+ * @merge: If true, merge the list after sorting.
+ *
+ * Returns nothing.
+ */
+void sort_memory_ranges(struct crash_mem *mem_rngs, bool merge)
+{
+ int i;
+
+ if (!mem_rngs)
+ return;
+
+ /* Sort the ranges in-place */
+ sort(&(mem_rngs->ranges[0]), mem_rngs->nr_ranges,
+ sizeof(mem_rngs->ranges[0]), rngcmp, NULL);
+
+ if (merge)
+ __merge_memory_ranges(mem_rngs);
+
+ /* For debugging purpose */
+ pr_debug("Memory ranges:\n");
+ for (i = 0; i < mem_rngs->nr_ranges; i++) {
+ pr_debug("\t[%03d][%#016llx - %#016llx]\n", i,
+ mem_rngs->ranges[i].start,
+ mem_rngs->ranges[i].end);
+ }
+}
+
+/**
+ * realloc_mem_ranges - reallocate mem_ranges with size incremented
+ * by MEM_RANGE_CHUNK_SZ. Frees up the old memory,
+ * if memory allocation fails.
+ * @mem_ranges: Memory ranges to reallocate.
+ *
+ * Returns pointer to reallocated memory on success, NULL otherwise.
+ */
+struct crash_mem *realloc_mem_ranges(struct crash_mem **mem_ranges)
+{
+ struct crash_mem *mem_rngs = *mem_ranges;
+ unsigned int nr_ranges;
+ size_t size;
+
+ size = get_mem_rngs_size(mem_rngs);
+ nr_ranges = mem_rngs ? mem_rngs->nr_ranges : 0;
+
+ size += MEM_RANGE_CHUNK_SZ;
+ mem_rngs = krealloc(*mem_ranges, size, GFP_KERNEL);
+ if (!mem_rngs) {
+ kfree(*mem_ranges);
+ *mem_ranges = NULL;
+ return NULL;
+ }
+
+ mem_rngs->nr_ranges = nr_ranges;
+ mem_rngs->max_nr_ranges = get_max_nr_ranges(size);
+ *mem_ranges = mem_rngs;
+
+ return mem_rngs;
+}
+
+/**
+ * add_mem_range - Updates existing memory range, if there is an overlap.
+ * Else, adds a new memory range.
+ * @mem_ranges: Range list to add the memory range to.
+ * @base: Base address of the range to add.
+ * @size: Size of the memory range to add.
+ *
+ * (Re)allocates memory, if needed.
+ *
+ * Returns 0 on success, negative errno on error.
+ */
+int add_mem_range(struct crash_mem **mem_ranges, u64 base, u64 size)
+{
+ struct crash_mem *mem_rngs = *mem_ranges;
+ u64 mstart, mend, end;
+ unsigned int i;
+
+ if (!size)
+ return 0;
+
+ end = base + size - 1;
+
+ if (!mem_rngs || !(mem_rngs->nr_ranges))
+ return __add_mem_range(mem_ranges, base, size);
+
+ for (i = 0; i < mem_rngs->nr_ranges; i++) {
+ mstart = mem_rngs->ranges[i].start;
+ mend = mem_rngs->ranges[i].end;
+ if (base < mend && end > mstart) {
+ if (base < mstart)
+ mem_rngs->ranges[i].start = base;
+ if (end > mend)
+ mem_rngs->ranges[i].end = end;
+ return 0;
+ }
+ }
+
+ return __add_mem_range(mem_ranges, base, size);
+}
+
+/**
+ * add_tce_mem_ranges - Adds tce-table range to the given memory ranges list.
+ * @mem_ranges: Range list to add the memory range(s) to.
+ *
+ * Returns 0 on success, negative errno on error.
+ */
+int add_tce_mem_ranges(struct crash_mem **mem_ranges)
+{
+ struct device_node *dn = NULL;
+ int ret = 0;
+
+ for_each_node_by_type(dn, "pci") {
+ u64 base;
+ u32 size;
+
+ ret = of_property_read_u64(dn, "linux,tce-base", &base);
+ ret |= of_property_read_u32(dn, "linux,tce-size", &size);
+ if (ret) {
+ /*
+ * It is ok to have pci nodes without tce. So, ignore
+ * property does not exist error.
+ */
+ if (ret == -EINVAL) {
+ ret = 0;
+ continue;
+ }
+ break;
+ }
+
+ ret = add_mem_range(mem_ranges, base, size);
+ if (ret)
+ break;
+ }
+
+ of_node_put(dn);
+ return ret;
+}
+
+/**
+ * add_initrd_mem_range - Adds initrd range to the given memory ranges list,
+ * if the initrd was retained.
+ * @mem_ranges: Range list to add the memory range to.
+ *
+ * Returns 0 on success, negative errno on error.
+ */
+int add_initrd_mem_range(struct crash_mem **mem_ranges)
+{
+ u64 base, end;
+ int ret;
+
+ /* This range means something, only if initrd was retained */
+ if (!strstr(saved_command_line, "retain_initrd"))
+ return 0;
+
+ ret = of_property_read_u64(of_chosen, "linux,initrd-start", &base);
+ ret |= of_property_read_u64(of_chosen, "linux,initrd-end", &end);
+ if (!ret)
+ ret = add_mem_range(mem_ranges, base, end - base + 1);
+
+ return ret;
+}
+
+#ifdef CONFIG_PPC_BOOK3S_64
+/**
+ * add_htab_mem_range - Adds htab range to the given memory ranges list,
+ * if it exists
+ * @mem_ranges: Range list to add the memory range to.
+ *
+ * Returns 0 on success, negative errno on error.
+ */
+int add_htab_mem_range(struct crash_mem **mem_ranges)
+{
+ if (!htab_address)
+ return 0;
+
+ return add_mem_range(mem_ranges, __pa(htab_address), htab_size_bytes);
+}
+#endif
+
+/**
+ * add_kernel_mem_range - Adds kernel text region to the given
+ * memory ranges list.
+ * @mem_ranges: Range list to add the memory range to.
+ *
+ * Returns 0 on success, negative errno on error.
+ */
+int add_kernel_mem_range(struct crash_mem **mem_ranges)
+{
+ return add_mem_range(mem_ranges, 0, __pa(_end));
+}
+
+/**
+ * add_rtas_mem_range - Adds RTAS region to the given memory ranges list.
+ * @mem_ranges: Range list to add the memory range to.
+ *
+ * Returns 0 on success, negative errno on error.
+ */
+int add_rtas_mem_range(struct crash_mem **mem_ranges)
+{
+ struct device_node *dn;
+ u32 base, size;
+ int ret = 0;
+
+ dn = of_find_node_by_path("/rtas");
+ if (!dn)
+ return 0;
+
+ ret = of_property_read_u32(dn, "linux,rtas-base", &base);
+ ret |= of_property_read_u32(dn, "rtas-size", &size);
+ if (!ret)
+ ret = add_mem_range(mem_ranges, base, size);
+
+ of_node_put(dn);
+ return ret;
+}
+
+/**
+ * add_opal_mem_range - Adds OPAL region to the given memory ranges list.
+ * @mem_ranges: Range list to add the memory range to.
+ *
+ * Returns 0 on success, negative errno on error.
+ */
+int add_opal_mem_range(struct crash_mem **mem_ranges)
+{
+ struct device_node *dn;
+ u64 base, size;
+ int ret;
+
+ dn = of_find_node_by_path("/ibm,opal");
+ if (!dn)
+ return 0;
+
+ ret = of_property_read_u64(dn, "opal-base-address", &base);
+ ret |= of_property_read_u64(dn, "opal-runtime-size", &size);
+ if (!ret)
+ ret = add_mem_range(mem_ranges, base, size);
+
+ of_node_put(dn);
+ return ret;
+}
+
+/**
+ * add_reserved_mem_ranges - Adds "/reserved-ranges" regions exported by f/w
+ * to the given memory ranges list.
+ * @mem_ranges: Range list to add the memory ranges to.
+ *
+ * Returns 0 on success, negative errno on error.
+ */
+int add_reserved_mem_ranges(struct crash_mem **mem_ranges)
+{
+ int n_mem_addr_cells, n_mem_size_cells, i, len, cells, ret = 0;
+ const __be32 *prop;
+
+ prop = of_get_property(of_root, "reserved-ranges", &len);
+ if (!prop)
+ return 0;
+
+ n_mem_addr_cells = of_n_addr_cells(of_root);
+ n_mem_size_cells = of_n_size_cells(of_root);
+ cells = n_mem_addr_cells + n_mem_size_cells;
+
+ /* Each reserved range is an (address,size) pair */
+ for (i = 0; i < (len / (sizeof(u32) * cells)); i++) {
+ u64 base, size;
+
+ base = of_read_number(prop + (i * cells), n_mem_addr_cells);
+ size = of_read_number(prop + (i * cells) + n_mem_addr_cells,
+ n_mem_size_cells);
+
+ ret = add_mem_range(mem_ranges, base, size);
+ if (ret)
+ break;
+ }
+
+ return ret;
+}
diff --git a/arch/powerpc/kvm/book3s_hv.c b/arch/powerpc/kvm/book3s_hv.c
index ebb04f3..0f83f39 100644
--- a/arch/powerpc/kvm/book3s_hv.c
+++ b/arch/powerpc/kvm/book3s_hv.c
@@ -765,7 +765,7 @@
return H_P3;
vcpu->arch.ciabr = value1;
return H_SUCCESS;
- case H_SET_MODE_RESOURCE_SET_DAWR:
+ case H_SET_MODE_RESOURCE_SET_DAWR0:
if (!kvmppc_power8_compatible(vcpu))
return H_P2;
if (!ppc_breakpoint_available())
@@ -1680,10 +1680,22 @@
case KVM_REG_PPC_UAMOR:
*val = get_reg_val(id, vcpu->arch.uamor);
break;
- case KVM_REG_PPC_MMCR0 ... KVM_REG_PPC_MMCRS:
+ case KVM_REG_PPC_MMCR0 ... KVM_REG_PPC_MMCR1:
i = id - KVM_REG_PPC_MMCR0;
*val = get_reg_val(id, vcpu->arch.mmcr[i]);
break;
+ case KVM_REG_PPC_MMCR2:
+ *val = get_reg_val(id, vcpu->arch.mmcr[2]);
+ break;
+ case KVM_REG_PPC_MMCRA:
+ *val = get_reg_val(id, vcpu->arch.mmcra);
+ break;
+ case KVM_REG_PPC_MMCRS:
+ *val = get_reg_val(id, vcpu->arch.mmcrs);
+ break;
+ case KVM_REG_PPC_MMCR3:
+ *val = get_reg_val(id, vcpu->arch.mmcr[3]);
+ break;
case KVM_REG_PPC_PMC1 ... KVM_REG_PPC_PMC8:
i = id - KVM_REG_PPC_PMC1;
*val = get_reg_val(id, vcpu->arch.pmc[i]);
@@ -1699,7 +1711,13 @@
*val = get_reg_val(id, vcpu->arch.sdar);
break;
case KVM_REG_PPC_SIER:
- *val = get_reg_val(id, vcpu->arch.sier);
+ *val = get_reg_val(id, vcpu->arch.sier[0]);
+ break;
+ case KVM_REG_PPC_SIER2:
+ *val = get_reg_val(id, vcpu->arch.sier[1]);
+ break;
+ case KVM_REG_PPC_SIER3:
+ *val = get_reg_val(id, vcpu->arch.sier[2]);
break;
case KVM_REG_PPC_IAMR:
*val = get_reg_val(id, vcpu->arch.iamr);
@@ -1901,10 +1919,22 @@
case KVM_REG_PPC_UAMOR:
vcpu->arch.uamor = set_reg_val(id, *val);
break;
- case KVM_REG_PPC_MMCR0 ... KVM_REG_PPC_MMCRS:
+ case KVM_REG_PPC_MMCR0 ... KVM_REG_PPC_MMCR1:
i = id - KVM_REG_PPC_MMCR0;
vcpu->arch.mmcr[i] = set_reg_val(id, *val);
break;
+ case KVM_REG_PPC_MMCR2:
+ vcpu->arch.mmcr[2] = set_reg_val(id, *val);
+ break;
+ case KVM_REG_PPC_MMCRA:
+ vcpu->arch.mmcra = set_reg_val(id, *val);
+ break;
+ case KVM_REG_PPC_MMCRS:
+ vcpu->arch.mmcrs = set_reg_val(id, *val);
+ break;
+ case KVM_REG_PPC_MMCR3:
+ *val = get_reg_val(id, vcpu->arch.mmcr[3]);
+ break;
case KVM_REG_PPC_PMC1 ... KVM_REG_PPC_PMC8:
i = id - KVM_REG_PPC_PMC1;
vcpu->arch.pmc[i] = set_reg_val(id, *val);
@@ -1920,7 +1950,13 @@
vcpu->arch.sdar = set_reg_val(id, *val);
break;
case KVM_REG_PPC_SIER:
- vcpu->arch.sier = set_reg_val(id, *val);
+ vcpu->arch.sier[0] = set_reg_val(id, *val);
+ break;
+ case KVM_REG_PPC_SIER2:
+ vcpu->arch.sier[1] = set_reg_val(id, *val);
+ break;
+ case KVM_REG_PPC_SIER3:
+ vcpu->arch.sier[2] = set_reg_val(id, *val);
break;
case KVM_REG_PPC_IAMR:
vcpu->arch.iamr = set_reg_val(id, *val);
diff --git a/arch/powerpc/kvm/book3s_hv_builtin.c b/arch/powerpc/kvm/book3s_hv_builtin.c
index 7cd3cf3d..073617c 100644
--- a/arch/powerpc/kvm/book3s_hv_builtin.c
+++ b/arch/powerpc/kvm/book3s_hv_builtin.c
@@ -113,7 +113,7 @@
selected_size = (selected_size * kvm_cma_resv_ratio / 100) << PAGE_SHIFT;
if (selected_size) {
- pr_debug("%s: reserving %ld MiB for global area\n", __func__,
+ pr_info("%s: reserving %ld MiB for global area\n", __func__,
(unsigned long)selected_size / SZ_1M);
align_size = HPT_ALIGN_PAGES << PAGE_SHIFT;
cma_declare_contiguous(0, selected_size, 0, align_size,
diff --git a/arch/powerpc/kvm/book3s_hv_interrupts.S b/arch/powerpc/kvm/book3s_hv_interrupts.S
index 63fd81f..59822cb 100644
--- a/arch/powerpc/kvm/book3s_hv_interrupts.S
+++ b/arch/powerpc/kvm/book3s_hv_interrupts.S
@@ -140,6 +140,14 @@
std r8, HSTATE_MMCR2(r13)
std r9, HSTATE_SIER(r13)
END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S)
+BEGIN_FTR_SECTION
+ mfspr r5, SPRN_MMCR3
+ mfspr r6, SPRN_SIER2
+ mfspr r7, SPRN_SIER3
+ std r5, HSTATE_MMCR3(r13)
+ std r6, HSTATE_SIER2(r13)
+ std r7, HSTATE_SIER3(r13)
+END_FTR_SECTION_IFSET(CPU_FTR_ARCH_31)
mfspr r3, SPRN_PMC1
mfspr r5, SPRN_PMC2
mfspr r6, SPRN_PMC3
diff --git a/arch/powerpc/kvm/book3s_hv_ras.c b/arch/powerpc/kvm/book3s_hv_ras.c
index 79f7d07..6028628 100644
--- a/arch/powerpc/kvm/book3s_hv_ras.c
+++ b/arch/powerpc/kvm/book3s_hv_ras.c
@@ -244,7 +244,7 @@
{
bool resync_req;
- __this_cpu_inc(irq_stat.hmi_exceptions);
+ local_paca->hmi_irqs++;
if (hmi_handle_debugtrig(NULL) >= 0)
return 1;
diff --git a/arch/powerpc/kvm/book3s_hv_rmhandlers.S b/arch/powerpc/kvm/book3s_hv_rmhandlers.S
index 7194389..799d6d0 100644
--- a/arch/powerpc/kvm/book3s_hv_rmhandlers.S
+++ b/arch/powerpc/kvm/book3s_hv_rmhandlers.S
@@ -3428,7 +3428,7 @@
mtspr SPRN_PMC6, r9
ld r3, VCPU_MMCR(r4)
ld r5, VCPU_MMCR + 8(r4)
- ld r6, VCPU_MMCR + 16(r4)
+ ld r6, VCPU_MMCRA(r4)
ld r7, VCPU_SIAR(r4)
ld r8, VCPU_SDAR(r4)
mtspr SPRN_MMCR1, r5
@@ -3436,14 +3436,22 @@
mtspr SPRN_SIAR, r7
mtspr SPRN_SDAR, r8
BEGIN_FTR_SECTION
- ld r5, VCPU_MMCR + 24(r4)
+ ld r5, VCPU_MMCR + 24(r4)
+ ld r6, VCPU_SIER + 8(r4)
+ ld r7, VCPU_SIER + 16(r4)
+ mtspr SPRN_MMCR3, r5
+ mtspr SPRN_SIER2, r6
+ mtspr SPRN_SIER3, r7
+END_FTR_SECTION_IFSET(CPU_FTR_ARCH_31)
+BEGIN_FTR_SECTION
+ ld r5, VCPU_MMCR + 16(r4)
ld r6, VCPU_SIER(r4)
mtspr SPRN_MMCR2, r5
mtspr SPRN_SIER, r6
BEGIN_FTR_SECTION_NESTED(96)
lwz r7, VCPU_PMC + 24(r4)
lwz r8, VCPU_PMC + 28(r4)
- ld r9, VCPU_MMCR + 32(r4)
+ ld r9, VCPU_MMCRS(r4)
mtspr SPRN_SPMC1, r7
mtspr SPRN_SPMC2, r8
mtspr SPRN_MMCRS, r9
@@ -3496,6 +3504,14 @@
mtspr SPRN_MMCR2, r8
mtspr SPRN_SIER, r9
END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S)
+BEGIN_FTR_SECTION
+ ld r5, HSTATE_MMCR3(r13)
+ ld r6, HSTATE_SIER2(r13)
+ ld r7, HSTATE_SIER3(r13)
+ mtspr SPRN_MMCR3, r5
+ mtspr SPRN_SIER2, r6
+ mtspr SPRN_SIER3, r7
+END_FTR_SECTION_IFSET(CPU_FTR_ARCH_31)
mtspr SPRN_MMCR0, r3
isync
mtlr r0
@@ -3551,10 +3567,18 @@
mfspr r8, SPRN_SDAR
std r4, VCPU_MMCR(r9)
std r5, VCPU_MMCR + 8(r9)
- std r6, VCPU_MMCR + 16(r9)
+ std r6, VCPU_MMCRA(r9)
BEGIN_FTR_SECTION
- std r10, VCPU_MMCR + 24(r9)
+ std r10, VCPU_MMCR + 16(r9)
END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S)
+BEGIN_FTR_SECTION
+ mfspr r5, SPRN_MMCR3
+ mfspr r6, SPRN_SIER2
+ mfspr r7, SPRN_SIER3
+ std r5, VCPU_MMCR + 24(r9)
+ std r6, VCPU_SIER + 8(r9)
+ std r7, VCPU_SIER + 16(r9)
+END_FTR_SECTION_IFSET(CPU_FTR_ARCH_31)
std r7, VCPU_SIAR(r9)
std r8, VCPU_SDAR(r9)
mfspr r3, SPRN_PMC1
@@ -3578,7 +3602,7 @@
mfspr r8, SPRN_MMCRS
stw r6, VCPU_PMC + 24(r9)
stw r7, VCPU_PMC + 28(r9)
- std r8, VCPU_MMCR + 32(r9)
+ std r8, VCPU_MMCRS(r9)
lis r4, 0x8000
mtspr SPRN_MMCRS, r4
END_FTR_SECTION_NESTED(CPU_FTR_ARCH_300, 0, 96)
diff --git a/arch/powerpc/kvm/book3s_interrupts.S b/arch/powerpc/kvm/book3s_interrupts.S
index f7ad99d..607a9b9 100644
--- a/arch/powerpc/kvm/book3s_interrupts.S
+++ b/arch/powerpc/kvm/book3s_interrupts.S
@@ -26,7 +26,7 @@
#define FUNC(name) name
#define GET_SHADOW_VCPU(reg) lwz reg, (THREAD + THREAD_KVM_SVCPU)(r2)
-#endif /* CONFIG_PPC_BOOK3S_XX */
+#endif /* CONFIG_PPC_BOOK3S_64 */
#define VCPU_LOAD_NVGPRS(vcpu) \
PPC_LL r14, VCPU_GPR(R14)(vcpu); \
diff --git a/arch/powerpc/kvm/trace_hv.h b/arch/powerpc/kvm/trace_hv.h
index 4a61a97..830a126 100644
--- a/arch/powerpc/kvm/trace_hv.h
+++ b/arch/powerpc/kvm/trace_hv.h
@@ -89,7 +89,7 @@
{H_CREATE_RPT, "H_CREATE_RPT"}, \
{H_REMOVE_RPT, "H_REMOVE_RPT"}, \
{H_REGISTER_RPAGES, "H_REGISTER_RPAGES"}, \
- {H_DISABLE_AND_GETC, "H_DISABLE_AND_GETC"}, \
+ {H_DISABLE_AND_GET, "H_DISABLE_AND_GET"}, \
{H_ERROR_DATA, "H_ERROR_DATA"}, \
{H_GET_HCA_INFO, "H_GET_HCA_INFO"}, \
{H_GET_PERF_COUNT, "H_GET_PERF_COUNT"}, \
diff --git a/arch/powerpc/lib/Makefile b/arch/powerpc/lib/Makefile
index 5e994cd..d66a645 100644
--- a/arch/powerpc/lib/Makefile
+++ b/arch/powerpc/lib/Makefile
@@ -41,7 +41,10 @@
obj64-y += copypage_64.o copyuser_64.o mem_64.o hweight_64.o \
memcpy_64.o memcpy_mcsafe_64.o
+ifndef CONFIG_PPC_QUEUED_SPINLOCKS
obj64-$(CONFIG_SMP) += locks.o
+endif
+
obj64-$(CONFIG_ALTIVEC) += vmx-helper.o
obj64-$(CONFIG_KPROBES_SANITY_TEST) += test_emulate_step.o \
test_emulate_step_exec_instr.o
diff --git a/arch/powerpc/lib/code-patching.c b/arch/powerpc/lib/code-patching.c
index 0a051df..8c3934e 100644
--- a/arch/powerpc/lib/code-patching.c
+++ b/arch/powerpc/lib/code-patching.c
@@ -93,7 +93,7 @@
unsigned long pfn;
int err;
- if (is_vmalloc_addr(addr))
+ if (is_vmalloc_or_module_addr(addr))
pfn = vmalloc_to_pfn(addr);
else
pfn = __pa_symbol(addr) >> PAGE_SHIFT;
diff --git a/arch/powerpc/lib/locks.c b/arch/powerpc/lib/locks.c
index 6440d5943..04165b7 100644
--- a/arch/powerpc/lib/locks.c
+++ b/arch/powerpc/lib/locks.c
@@ -27,14 +27,14 @@
return;
holder_cpu = lock_value & 0xffff;
BUG_ON(holder_cpu >= NR_CPUS);
- yield_count = be32_to_cpu(lppaca_of(holder_cpu).yield_count);
+
+ yield_count = yield_count_of(holder_cpu);
if ((yield_count & 1) == 0)
return; /* virtual cpu is currently running */
rmb();
if (lock->slock != lock_value)
return; /* something has changed */
- plpar_hcall_norets(H_CONFER,
- get_hard_smp_processor_id(holder_cpu), yield_count);
+ yield_to_preempted(holder_cpu, yield_count);
}
EXPORT_SYMBOL_GPL(splpar_spin_yield);
@@ -53,13 +53,13 @@
return; /* no write lock at present */
holder_cpu = lock_value & 0xffff;
BUG_ON(holder_cpu >= NR_CPUS);
- yield_count = be32_to_cpu(lppaca_of(holder_cpu).yield_count);
+
+ yield_count = yield_count_of(holder_cpu);
if ((yield_count & 1) == 0)
return; /* virtual cpu is currently running */
rmb();
if (rw->lock != lock_value)
return; /* something has changed */
- plpar_hcall_norets(H_CONFER,
- get_hard_smp_processor_id(holder_cpu), yield_count);
+ yield_to_preempted(holder_cpu, yield_count);
}
#endif
diff --git a/arch/powerpc/lib/pmem.c b/arch/powerpc/lib/pmem.c
index 0666a8d2..1550e0d 100644
--- a/arch/powerpc/lib/pmem.c
+++ b/arch/powerpc/lib/pmem.c
@@ -9,20 +9,56 @@
#include <asm/cacheflush.h>
+static inline void __clean_pmem_range(unsigned long start, unsigned long stop)
+{
+ unsigned long shift = l1_dcache_shift();
+ unsigned long bytes = l1_dcache_bytes();
+ void *addr = (void *)(start & ~(bytes - 1));
+ unsigned long size = stop - (unsigned long)addr + (bytes - 1);
+ unsigned long i;
+
+ for (i = 0; i < size >> shift; i++, addr += bytes)
+ asm volatile(PPC_DCBSTPS(%0, %1): :"i"(0), "r"(addr): "memory");
+}
+
+static inline void __flush_pmem_range(unsigned long start, unsigned long stop)
+{
+ unsigned long shift = l1_dcache_shift();
+ unsigned long bytes = l1_dcache_bytes();
+ void *addr = (void *)(start & ~(bytes - 1));
+ unsigned long size = stop - (unsigned long)addr + (bytes - 1);
+ unsigned long i;
+
+ for (i = 0; i < size >> shift; i++, addr += bytes)
+ asm volatile(PPC_DCBFPS(%0, %1): :"i"(0), "r"(addr): "memory");
+}
+
+static inline void clean_pmem_range(unsigned long start, unsigned long stop)
+{
+ if (cpu_has_feature(CPU_FTR_ARCH_207S))
+ return __clean_pmem_range(start, stop);
+}
+
+static inline void flush_pmem_range(unsigned long start, unsigned long stop)
+{
+ if (cpu_has_feature(CPU_FTR_ARCH_207S))
+ return __flush_pmem_range(start, stop);
+}
+
/*
* CONFIG_ARCH_HAS_PMEM_API symbols
*/
void arch_wb_cache_pmem(void *addr, size_t size)
{
unsigned long start = (unsigned long) addr;
- flush_dcache_range(start, start + size);
+ clean_pmem_range(start, start + size);
}
EXPORT_SYMBOL_GPL(arch_wb_cache_pmem);
void arch_invalidate_pmem(void *addr, size_t size)
{
unsigned long start = (unsigned long) addr;
- flush_dcache_range(start, start + size);
+ flush_pmem_range(start, start + size);
}
EXPORT_SYMBOL_GPL(arch_invalidate_pmem);
@@ -35,19 +71,17 @@
unsigned long copied, start = (unsigned long) dest;
copied = __copy_from_user(dest, src, size);
- flush_dcache_range(start, start + size);
+ clean_pmem_range(start, start + size);
return copied;
}
-void *memcpy_flushcache(void *dest, const void *src, size_t size)
+void memcpy_flushcache(void *dest, const void *src, size_t size)
{
unsigned long start = (unsigned long) dest;
memcpy(dest, src, size);
- flush_dcache_range(start, start + size);
-
- return dest;
+ clean_pmem_range(start, start + size);
}
EXPORT_SYMBOL(memcpy_flushcache);
diff --git a/arch/powerpc/lib/sstep.c b/arch/powerpc/lib/sstep.c
index 5abe982..caee8cc 100644
--- a/arch/powerpc/lib/sstep.c
+++ b/arch/powerpc/lib/sstep.c
@@ -16,6 +16,7 @@
#include <asm/disassemble.h>
extern char system_call_common[];
+extern char system_call_vectored_emulate[];
#ifdef CONFIG_PPC64
/* Bits in SRR1 that are copied from MSR */
@@ -200,8 +201,8 @@
unsigned int dd;
unsigned long ea, d0, d1, d;
- prefix_r = instr & (1ul << 20);
- ra = (suffix >> 16) & 0x1f;
+ prefix_r = GET_PREFIX_R(instr);
+ ra = GET_PREFIX_RA(suffix);
d0 = instr & 0x3ffff;
d1 = suffix & 0xffff;
@@ -1236,6 +1237,9 @@
case 17: /* sc */
if ((word & 0xfe2) == 2)
op->type = SYSCALL;
+ else if (IS_ENABLED(CONFIG_PPC_BOOK3S_64) &&
+ (word & 0xfe3) == 1)
+ op->type = SYSCALL_VECTORED_0;
else
op->type = UNKNOWN;
return 0;
@@ -1339,8 +1343,8 @@
switch (opcode) {
#ifdef __powerpc64__
case 1:
- prefix_r = word & (1ul << 20);
- ra = (suffix >> 16) & 0x1f;
+ prefix_r = GET_PREFIX_R(word);
+ ra = GET_PREFIX_RA(suffix);
rd = (suffix >> 21) & 0x1f;
op->reg = rd;
op->val = regs->gpr[rd];
@@ -1802,7 +1806,18 @@
op->val = (int) regs->gpr[ra] /
(int) regs->gpr[rb];
goto arith_done;
-
+#ifdef __powerpc64__
+ case 425: /* divde[.] */
+ asm volatile(PPC_DIVDE(%0, %1, %2) :
+ "=r" (op->val) : "r" (regs->gpr[ra]),
+ "r" (regs->gpr[rb]));
+ goto arith_done;
+ case 393: /* divdeu[.] */
+ asm volatile(PPC_DIVDEU(%0, %1, %2) :
+ "=r" (op->val) : "r" (regs->gpr[ra]),
+ "r" (regs->gpr[rb]));
+ goto arith_done;
+#endif
case 755: /* darn */
if (!cpu_has_feature(CPU_FTR_ARCH_300))
return -1;
@@ -2715,8 +2730,8 @@
}
break;
case 1: /* Prefixed instructions */
- prefix_r = word & (1ul << 20);
- ra = (suffix >> 16) & 0x1f;
+ prefix_r = GET_PREFIX_R(word);
+ ra = GET_PREFIX_RA(suffix);
op->update_reg = ra;
rd = (suffix >> 21) & 0x1f;
op->reg = rd;
@@ -3378,6 +3393,18 @@
regs->msr = MSR_KERNEL;
return 1;
+#ifdef CONFIG_PPC_BOOK3S_64
+ case SYSCALL_VECTORED_0: /* scv 0 */
+ regs->gpr[9] = regs->gpr[13];
+ regs->gpr[10] = MSR_KERNEL;
+ regs->gpr[11] = regs->nip + 4;
+ regs->gpr[12] = regs->msr & MSR_MASK;
+ regs->gpr[13] = (unsigned long) get_paca();
+ regs->nip = (unsigned long) &system_call_vectored_emulate;
+ regs->msr = MSR_KERNEL;
+ return 1;
+#endif
+
case RFI:
return -1;
#endif
diff --git a/arch/powerpc/lib/test_emulate_step.c b/arch/powerpc/lib/test_emulate_step.c
index 46af802..0a201b7 100644
--- a/arch/powerpc/lib/test_emulate_step.c
+++ b/arch/powerpc/lib/test_emulate_step.c
@@ -8,59 +8,51 @@
#define pr_fmt(fmt) "emulate_step_test: " fmt
#include <linux/ptrace.h>
+#include <asm/cpu_has_feature.h>
#include <asm/sstep.h>
#include <asm/ppc-opcode.h>
#include <asm/code-patching.h>
#include <asm/inst.h>
-#define IMM_L(i) ((uintptr_t)(i) & 0xffff)
-#define IMM_DS(i) ((uintptr_t)(i) & 0xfffc)
-
-/*
- * Defined with TEST_ prefix so it does not conflict with other
- * definitions.
- */
-#define TEST_LD(r, base, i) ppc_inst(PPC_INST_LD | ___PPC_RT(r) | \
- ___PPC_RA(base) | IMM_DS(i))
-#define TEST_LWZ(r, base, i) ppc_inst(PPC_INST_LWZ | ___PPC_RT(r) | \
- ___PPC_RA(base) | IMM_L(i))
-#define TEST_LWZX(t, a, b) ppc_inst(PPC_INST_LWZX | ___PPC_RT(t) | \
- ___PPC_RA(a) | ___PPC_RB(b))
-#define TEST_STD(r, base, i) ppc_inst(PPC_INST_STD | ___PPC_RS(r) | \
- ___PPC_RA(base) | IMM_DS(i))
-#define TEST_LDARX(t, a, b, eh) ppc_inst(PPC_INST_LDARX | ___PPC_RT(t) | \
- ___PPC_RA(a) | ___PPC_RB(b) | \
- __PPC_EH(eh))
-#define TEST_STDCX(s, a, b) ppc_inst(PPC_INST_STDCX | ___PPC_RS(s) | \
- ___PPC_RA(a) | ___PPC_RB(b))
-#define TEST_LFSX(t, a, b) ppc_inst(PPC_INST_LFSX | ___PPC_RT(t) | \
- ___PPC_RA(a) | ___PPC_RB(b))
-#define TEST_STFSX(s, a, b) ppc_inst(PPC_INST_STFSX | ___PPC_RS(s) | \
- ___PPC_RA(a) | ___PPC_RB(b))
-#define TEST_LFDX(t, a, b) ppc_inst(PPC_INST_LFDX | ___PPC_RT(t) | \
- ___PPC_RA(a) | ___PPC_RB(b))
-#define TEST_STFDX(s, a, b) ppc_inst(PPC_INST_STFDX | ___PPC_RS(s) | \
- ___PPC_RA(a) | ___PPC_RB(b))
-#define TEST_LVX(t, a, b) ppc_inst(PPC_INST_LVX | ___PPC_RT(t) | \
- ___PPC_RA(a) | ___PPC_RB(b))
-#define TEST_STVX(s, a, b) ppc_inst(PPC_INST_STVX | ___PPC_RS(s) | \
- ___PPC_RA(a) | ___PPC_RB(b))
-#define TEST_LXVD2X(s, a, b) ppc_inst(PPC_INST_LXVD2X | VSX_XX1((s), R##a, R##b))
-#define TEST_STXVD2X(s, a, b) ppc_inst(PPC_INST_STXVD2X | VSX_XX1((s), R##a, R##b))
-#define TEST_ADD(t, a, b) ppc_inst(PPC_INST_ADD | ___PPC_RT(t) | \
- ___PPC_RA(a) | ___PPC_RB(b))
-#define TEST_ADD_DOT(t, a, b) ppc_inst(PPC_INST_ADD | ___PPC_RT(t) | \
- ___PPC_RA(a) | ___PPC_RB(b) | 0x1)
-#define TEST_ADDC(t, a, b) ppc_inst(PPC_INST_ADDC | ___PPC_RT(t) | \
- ___PPC_RA(a) | ___PPC_RB(b))
-#define TEST_ADDC_DOT(t, a, b) ppc_inst(PPC_INST_ADDC | ___PPC_RT(t) | \
- ___PPC_RA(a) | ___PPC_RB(b) | 0x1)
-
#define MAX_SUBTESTS 16
#define IGNORE_GPR(n) (0x1UL << (n))
#define IGNORE_XER (0x1UL << 32)
#define IGNORE_CCR (0x1UL << 33)
+#define NEGATIVE_TEST (0x1UL << 63)
+
+#define TEST_PLD(r, base, i, pr) \
+ ppc_inst_prefix(PPC_PREFIX_8LS | __PPC_PRFX_R(pr) | IMM_H(i), \
+ PPC_INST_PLD | ___PPC_RT(r) | ___PPC_RA(base) | IMM_L(i))
+
+#define TEST_PLWZ(r, base, i, pr) \
+ ppc_inst_prefix(PPC_PREFIX_MLS | __PPC_PRFX_R(pr) | IMM_H(i), \
+ PPC_RAW_LWZ(r, base, i))
+
+#define TEST_PSTD(r, base, i, pr) \
+ ppc_inst_prefix(PPC_PREFIX_8LS | __PPC_PRFX_R(pr) | IMM_H(i), \
+ PPC_INST_PSTD | ___PPC_RT(r) | ___PPC_RA(base) | IMM_L(i))
+
+#define TEST_PLFS(r, base, i, pr) \
+ ppc_inst_prefix(PPC_PREFIX_MLS | __PPC_PRFX_R(pr) | IMM_H(i), \
+ PPC_INST_LFS | ___PPC_RT(r) | ___PPC_RA(base) | IMM_L(i))
+
+#define TEST_PSTFS(r, base, i, pr) \
+ ppc_inst_prefix(PPC_PREFIX_MLS | __PPC_PRFX_R(pr) | IMM_H(i), \
+ PPC_INST_STFS | ___PPC_RT(r) | ___PPC_RA(base) | IMM_L(i))
+
+#define TEST_PLFD(r, base, i, pr) \
+ ppc_inst_prefix(PPC_PREFIX_MLS | __PPC_PRFX_R(pr) | IMM_H(i), \
+ PPC_INST_LFD | ___PPC_RT(r) | ___PPC_RA(base) | IMM_L(i))
+
+#define TEST_PSTFD(r, base, i, pr) \
+ ppc_inst_prefix(PPC_PREFIX_MLS | __PPC_PRFX_R(pr) | IMM_H(i), \
+ PPC_INST_STFD | ___PPC_RT(r) | ___PPC_RA(base) | IMM_L(i))
+
+#define TEST_PADDI(t, a, i, pr) \
+ ppc_inst_prefix(PPC_PREFIX_MLS | __PPC_PRFX_R(pr) | IMM_H(i), \
+ PPC_RAW_ADDI(t, a, i))
+
static void __init init_pt_regs(struct pt_regs *regs)
{
@@ -105,7 +97,7 @@
regs.gpr[3] = (unsigned long) &a;
/* ld r5, 0(r3) */
- stepped = emulate_step(®s, TEST_LD(5, 3, 0));
+ stepped = emulate_step(®s, ppc_inst(PPC_RAW_LD(5, 3, 0)));
if (stepped == 1 && regs.gpr[5] == a)
show_result("ld", "PASS");
@@ -113,6 +105,29 @@
show_result("ld", "FAIL");
}
+static void __init test_pld(void)
+{
+ struct pt_regs regs;
+ unsigned long a = 0x23;
+ int stepped = -1;
+
+ if (!cpu_has_feature(CPU_FTR_ARCH_31)) {
+ show_result("pld", "SKIP (!CPU_FTR_ARCH_31)");
+ return;
+ }
+
+ init_pt_regs(®s);
+ regs.gpr[3] = (unsigned long)&a;
+
+ /* pld r5, 0(r3), 0 */
+ stepped = emulate_step(®s, TEST_PLD(5, 3, 0, 0));
+
+ if (stepped == 1 && regs.gpr[5] == a)
+ show_result("pld", "PASS");
+ else
+ show_result("pld", "FAIL");
+}
+
static void __init test_lwz(void)
{
struct pt_regs regs;
@@ -123,7 +138,7 @@
regs.gpr[3] = (unsigned long) &a;
/* lwz r5, 0(r3) */
- stepped = emulate_step(®s, TEST_LWZ(5, 3, 0));
+ stepped = emulate_step(®s, ppc_inst(PPC_RAW_LWZ(5, 3, 0)));
if (stepped == 1 && regs.gpr[5] == a)
show_result("lwz", "PASS");
@@ -131,6 +146,30 @@
show_result("lwz", "FAIL");
}
+static void __init test_plwz(void)
+{
+ struct pt_regs regs;
+ unsigned int a = 0x4545;
+ int stepped = -1;
+
+ if (!cpu_has_feature(CPU_FTR_ARCH_31)) {
+ show_result("plwz", "SKIP (!CPU_FTR_ARCH_31)");
+ return;
+ }
+
+ init_pt_regs(®s);
+ regs.gpr[3] = (unsigned long)&a;
+
+ /* plwz r5, 0(r3), 0 */
+
+ stepped = emulate_step(®s, TEST_PLWZ(5, 3, 0, 0));
+
+ if (stepped == 1 && regs.gpr[5] == a)
+ show_result("plwz", "PASS");
+ else
+ show_result("plwz", "FAIL");
+}
+
static void __init test_lwzx(void)
{
struct pt_regs regs;
@@ -143,7 +182,7 @@
regs.gpr[5] = 0x8765;
/* lwzx r5, r3, r4 */
- stepped = emulate_step(®s, TEST_LWZX(5, 3, 4));
+ stepped = emulate_step(®s, ppc_inst(PPC_RAW_LWZX(5, 3, 4)));
if (stepped == 1 && regs.gpr[5] == a[2])
show_result("lwzx", "PASS");
else
@@ -161,13 +200,36 @@
regs.gpr[5] = 0x5678;
/* std r5, 0(r3) */
- stepped = emulate_step(®s, TEST_STD(5, 3, 0));
+ stepped = emulate_step(®s, ppc_inst(PPC_RAW_STD(5, 3, 0)));
if (stepped == 1 && regs.gpr[5] == a)
show_result("std", "PASS");
else
show_result("std", "FAIL");
}
+static void __init test_pstd(void)
+{
+ struct pt_regs regs;
+ unsigned long a = 0x1234;
+ int stepped = -1;
+
+ if (!cpu_has_feature(CPU_FTR_ARCH_31)) {
+ show_result("pstd", "SKIP (!CPU_FTR_ARCH_31)");
+ return;
+ }
+
+ init_pt_regs(®s);
+ regs.gpr[3] = (unsigned long)&a;
+ regs.gpr[5] = 0x5678;
+
+ /* pstd r5, 0(r3), 0 */
+ stepped = emulate_step(®s, TEST_PSTD(5, 3, 0, 0));
+ if (stepped == 1 || regs.gpr[5] == a)
+ show_result("pstd", "PASS");
+ else
+ show_result("pstd", "FAIL");
+}
+
static void __init test_ldarx_stdcx(void)
{
struct pt_regs regs;
@@ -186,7 +248,7 @@
regs.gpr[5] = 0x5678;
/* ldarx r5, r3, r4, 0 */
- stepped = emulate_step(®s, TEST_LDARX(5, 3, 4, 0));
+ stepped = emulate_step(®s, ppc_inst(PPC_RAW_LDARX(5, 3, 4, 0)));
/*
* Don't touch 'a' here. Touching 'a' can do Load/store
@@ -204,7 +266,7 @@
regs.gpr[5] = 0x9ABC;
/* stdcx. r5, r3, r4 */
- stepped = emulate_step(®s, TEST_STDCX(5, 3, 4));
+ stepped = emulate_step(®s, ppc_inst(PPC_RAW_STDCX(5, 3, 4)));
/*
* Two possible scenarios that indicates successful emulation
@@ -244,7 +306,7 @@
regs.gpr[4] = 0;
/* lfsx frt10, r3, r4 */
- stepped = emulate_step(®s, TEST_LFSX(10, 3, 4));
+ stepped = emulate_step(®s, ppc_inst(PPC_RAW_LFSX(10, 3, 4)));
if (stepped == 1)
show_result("lfsx", "PASS");
@@ -257,7 +319,7 @@
c.a = 678.91;
/* stfsx frs10, r3, r4 */
- stepped = emulate_step(®s, TEST_STFSX(10, 3, 4));
+ stepped = emulate_step(®s, ppc_inst(PPC_RAW_STFSX(10, 3, 4)));
if (stepped == 1 && c.b == cached_b)
show_result("stfsx", "PASS");
@@ -265,6 +327,53 @@
show_result("stfsx", "FAIL");
}
+static void __init test_plfs_pstfs(void)
+{
+ struct pt_regs regs;
+ union {
+ float a;
+ int b;
+ } c;
+ int cached_b;
+ int stepped = -1;
+
+ if (!cpu_has_feature(CPU_FTR_ARCH_31)) {
+ show_result("pld", "SKIP (!CPU_FTR_ARCH_31)");
+ return;
+ }
+
+ init_pt_regs(®s);
+
+
+ /*** plfs ***/
+
+ c.a = 123.45;
+ cached_b = c.b;
+
+ regs.gpr[3] = (unsigned long)&c.a;
+
+ /* plfs frt10, 0(r3), 0 */
+ stepped = emulate_step(®s, TEST_PLFS(10, 3, 0, 0));
+
+ if (stepped == 1)
+ show_result("plfs", "PASS");
+ else
+ show_result("plfs", "FAIL");
+
+
+ /*** pstfs ***/
+
+ c.a = 678.91;
+
+ /* pstfs frs10, 0(r3), 0 */
+ stepped = emulate_step(®s, TEST_PSTFS(10, 3, 0, 0));
+
+ if (stepped == 1 && c.b == cached_b)
+ show_result("pstfs", "PASS");
+ else
+ show_result("pstfs", "FAIL");
+}
+
static void __init test_lfdx_stfdx(void)
{
struct pt_regs regs;
@@ -287,7 +396,7 @@
regs.gpr[4] = 0;
/* lfdx frt10, r3, r4 */
- stepped = emulate_step(®s, TEST_LFDX(10, 3, 4));
+ stepped = emulate_step(®s, ppc_inst(PPC_RAW_LFDX(10, 3, 4)));
if (stepped == 1)
show_result("lfdx", "PASS");
@@ -300,13 +409,60 @@
c.a = 987654.32;
/* stfdx frs10, r3, r4 */
- stepped = emulate_step(®s, TEST_STFDX(10, 3, 4));
+ stepped = emulate_step(®s, ppc_inst(PPC_RAW_STFDX(10, 3, 4)));
if (stepped == 1 && c.b == cached_b)
show_result("stfdx", "PASS");
else
show_result("stfdx", "FAIL");
}
+
+static void __init test_plfd_pstfd(void)
+{
+ struct pt_regs regs;
+ union {
+ double a;
+ long b;
+ } c;
+ long cached_b;
+ int stepped = -1;
+
+ if (!cpu_has_feature(CPU_FTR_ARCH_31)) {
+ show_result("pld", "SKIP (!CPU_FTR_ARCH_31)");
+ return;
+ }
+
+ init_pt_regs(®s);
+
+
+ /*** plfd ***/
+
+ c.a = 123456.78;
+ cached_b = c.b;
+
+ regs.gpr[3] = (unsigned long)&c.a;
+
+ /* plfd frt10, 0(r3), 0 */
+ stepped = emulate_step(®s, TEST_PLFD(10, 3, 0, 0));
+
+ if (stepped == 1)
+ show_result("plfd", "PASS");
+ else
+ show_result("plfd", "FAIL");
+
+
+ /*** pstfd ***/
+
+ c.a = 987654.32;
+
+ /* pstfd frs10, 0(r3), 0 */
+ stepped = emulate_step(®s, TEST_PSTFD(10, 3, 0, 0));
+
+ if (stepped == 1 && c.b == cached_b)
+ show_result("pstfd", "PASS");
+ else
+ show_result("pstfd", "FAIL");
+}
#else
static void __init test_lfsx_stfsx(void)
{
@@ -314,11 +470,23 @@
show_result("stfsx", "SKIP (CONFIG_PPC_FPU is not set)");
}
+static void __init test_plfs_pstfs(void)
+{
+ show_result("plfs", "SKIP (CONFIG_PPC_FPU is not set)");
+ show_result("pstfs", "SKIP (CONFIG_PPC_FPU is not set)");
+}
+
static void __init test_lfdx_stfdx(void)
{
show_result("lfdx", "SKIP (CONFIG_PPC_FPU is not set)");
show_result("stfdx", "SKIP (CONFIG_PPC_FPU is not set)");
}
+
+static void __init test_plfd_pstfd(void)
+{
+ show_result("plfd", "SKIP (CONFIG_PPC_FPU is not set)");
+ show_result("pstfd", "SKIP (CONFIG_PPC_FPU is not set)");
+}
#endif /* CONFIG_PPC_FPU */
#ifdef CONFIG_ALTIVEC
@@ -346,7 +514,7 @@
regs.gpr[4] = 0;
/* lvx vrt10, r3, r4 */
- stepped = emulate_step(®s, TEST_LVX(10, 3, 4));
+ stepped = emulate_step(®s, ppc_inst(PPC_RAW_LVX(10, 3, 4)));
if (stepped == 1)
show_result("lvx", "PASS");
@@ -362,7 +530,7 @@
c.b[3] = 498532;
/* stvx vrs10, r3, r4 */
- stepped = emulate_step(®s, TEST_STVX(10, 3, 4));
+ stepped = emulate_step(®s, ppc_inst(PPC_RAW_STVX(10, 3, 4)));
if (stepped == 1 && cached_b[0] == c.b[0] && cached_b[1] == c.b[1] &&
cached_b[2] == c.b[2] && cached_b[3] == c.b[3])
@@ -403,7 +571,7 @@
regs.gpr[4] = 0;
/* lxvd2x vsr39, r3, r4 */
- stepped = emulate_step(®s, TEST_LXVD2X(39, 3, 4));
+ stepped = emulate_step(®s, ppc_inst(PPC_RAW_LXVD2X(39, R3, R4)));
if (stepped == 1 && cpu_has_feature(CPU_FTR_VSX)) {
show_result("lxvd2x", "PASS");
@@ -423,7 +591,7 @@
c.b[3] = 4;
/* stxvd2x vsr39, r3, r4 */
- stepped = emulate_step(®s, TEST_STXVD2X(39, 3, 4));
+ stepped = emulate_step(®s, ppc_inst(PPC_RAW_STXVD2X(39, R3, R4)));
if (stepped == 1 && cached_b[0] == c.b[0] && cached_b[1] == c.b[1] &&
cached_b[2] == c.b[2] && cached_b[3] == c.b[3] &&
@@ -447,18 +615,24 @@
static void __init run_tests_load_store(void)
{
test_ld();
+ test_pld();
test_lwz();
+ test_plwz();
test_lwzx();
test_std();
+ test_pstd();
test_ldarx_stdcx();
test_lfsx_stfsx();
+ test_plfs_pstfs();
test_lfdx_stfdx();
+ test_plfd_pstfd();
test_lvx_stvx();
test_lxvd2x_stxvd2x();
}
struct compute_test {
char *mnemonic;
+ unsigned long cpu_feature;
struct {
char *descr;
unsigned long flags;
@@ -467,6 +641,11 @@
} subtests[MAX_SUBTESTS + 1];
};
+/* Extreme values for si0||si1 (the MLS:D-form 34 bit immediate field) */
+#define SI_MIN BIT(33)
+#define SI_MAX (BIT(33) - 1)
+#define SI_UMAX (BIT(34) - 1)
+
static struct compute_test compute_tests[] = {
{
.mnemonic = "nop",
@@ -485,7 +664,7 @@
.subtests = {
{
.descr = "RA = LONG_MIN, RB = LONG_MIN",
- .instr = TEST_ADD(20, 21, 22),
+ .instr = ppc_inst(PPC_RAW_ADD(20, 21, 22)),
.regs = {
.gpr[21] = LONG_MIN,
.gpr[22] = LONG_MIN,
@@ -493,7 +672,7 @@
},
{
.descr = "RA = LONG_MIN, RB = LONG_MAX",
- .instr = TEST_ADD(20, 21, 22),
+ .instr = ppc_inst(PPC_RAW_ADD(20, 21, 22)),
.regs = {
.gpr[21] = LONG_MIN,
.gpr[22] = LONG_MAX,
@@ -501,7 +680,7 @@
},
{
.descr = "RA = LONG_MAX, RB = LONG_MAX",
- .instr = TEST_ADD(20, 21, 22),
+ .instr = ppc_inst(PPC_RAW_ADD(20, 21, 22)),
.regs = {
.gpr[21] = LONG_MAX,
.gpr[22] = LONG_MAX,
@@ -509,7 +688,7 @@
},
{
.descr = "RA = ULONG_MAX, RB = ULONG_MAX",
- .instr = TEST_ADD(20, 21, 22),
+ .instr = ppc_inst(PPC_RAW_ADD(20, 21, 22)),
.regs = {
.gpr[21] = ULONG_MAX,
.gpr[22] = ULONG_MAX,
@@ -517,7 +696,7 @@
},
{
.descr = "RA = ULONG_MAX, RB = 0x1",
- .instr = TEST_ADD(20, 21, 22),
+ .instr = ppc_inst(PPC_RAW_ADD(20, 21, 22)),
.regs = {
.gpr[21] = ULONG_MAX,
.gpr[22] = 0x1,
@@ -525,7 +704,7 @@
},
{
.descr = "RA = INT_MIN, RB = INT_MIN",
- .instr = TEST_ADD(20, 21, 22),
+ .instr = ppc_inst(PPC_RAW_ADD(20, 21, 22)),
.regs = {
.gpr[21] = INT_MIN,
.gpr[22] = INT_MIN,
@@ -533,7 +712,7 @@
},
{
.descr = "RA = INT_MIN, RB = INT_MAX",
- .instr = TEST_ADD(20, 21, 22),
+ .instr = ppc_inst(PPC_RAW_ADD(20, 21, 22)),
.regs = {
.gpr[21] = INT_MIN,
.gpr[22] = INT_MAX,
@@ -541,7 +720,7 @@
},
{
.descr = "RA = INT_MAX, RB = INT_MAX",
- .instr = TEST_ADD(20, 21, 22),
+ .instr = ppc_inst(PPC_RAW_ADD(20, 21, 22)),
.regs = {
.gpr[21] = INT_MAX,
.gpr[22] = INT_MAX,
@@ -549,7 +728,7 @@
},
{
.descr = "RA = UINT_MAX, RB = UINT_MAX",
- .instr = TEST_ADD(20, 21, 22),
+ .instr = ppc_inst(PPC_RAW_ADD(20, 21, 22)),
.regs = {
.gpr[21] = UINT_MAX,
.gpr[22] = UINT_MAX,
@@ -557,7 +736,7 @@
},
{
.descr = "RA = UINT_MAX, RB = 0x1",
- .instr = TEST_ADD(20, 21, 22),
+ .instr = ppc_inst(PPC_RAW_ADD(20, 21, 22)),
.regs = {
.gpr[21] = UINT_MAX,
.gpr[22] = 0x1,
@@ -571,7 +750,7 @@
{
.descr = "RA = LONG_MIN, RB = LONG_MIN",
.flags = IGNORE_CCR,
- .instr = TEST_ADD_DOT(20, 21, 22),
+ .instr = ppc_inst(PPC_RAW_ADD_DOT(20, 21, 22)),
.regs = {
.gpr[21] = LONG_MIN,
.gpr[22] = LONG_MIN,
@@ -579,7 +758,7 @@
},
{
.descr = "RA = LONG_MIN, RB = LONG_MAX",
- .instr = TEST_ADD_DOT(20, 21, 22),
+ .instr = ppc_inst(PPC_RAW_ADD_DOT(20, 21, 22)),
.regs = {
.gpr[21] = LONG_MIN,
.gpr[22] = LONG_MAX,
@@ -588,7 +767,7 @@
{
.descr = "RA = LONG_MAX, RB = LONG_MAX",
.flags = IGNORE_CCR,
- .instr = TEST_ADD_DOT(20, 21, 22),
+ .instr = ppc_inst(PPC_RAW_ADD_DOT(20, 21, 22)),
.regs = {
.gpr[21] = LONG_MAX,
.gpr[22] = LONG_MAX,
@@ -596,7 +775,7 @@
},
{
.descr = "RA = ULONG_MAX, RB = ULONG_MAX",
- .instr = TEST_ADD_DOT(20, 21, 22),
+ .instr = ppc_inst(PPC_RAW_ADD_DOT(20, 21, 22)),
.regs = {
.gpr[21] = ULONG_MAX,
.gpr[22] = ULONG_MAX,
@@ -604,7 +783,7 @@
},
{
.descr = "RA = ULONG_MAX, RB = 0x1",
- .instr = TEST_ADD_DOT(20, 21, 22),
+ .instr = ppc_inst(PPC_RAW_ADD_DOT(20, 21, 22)),
.regs = {
.gpr[21] = ULONG_MAX,
.gpr[22] = 0x1,
@@ -612,7 +791,7 @@
},
{
.descr = "RA = INT_MIN, RB = INT_MIN",
- .instr = TEST_ADD_DOT(20, 21, 22),
+ .instr = ppc_inst(PPC_RAW_ADD_DOT(20, 21, 22)),
.regs = {
.gpr[21] = INT_MIN,
.gpr[22] = INT_MIN,
@@ -620,7 +799,7 @@
},
{
.descr = "RA = INT_MIN, RB = INT_MAX",
- .instr = TEST_ADD_DOT(20, 21, 22),
+ .instr = ppc_inst(PPC_RAW_ADD_DOT(20, 21, 22)),
.regs = {
.gpr[21] = INT_MIN,
.gpr[22] = INT_MAX,
@@ -628,7 +807,7 @@
},
{
.descr = "RA = INT_MAX, RB = INT_MAX",
- .instr = TEST_ADD_DOT(20, 21, 22),
+ .instr = ppc_inst(PPC_RAW_ADD_DOT(20, 21, 22)),
.regs = {
.gpr[21] = INT_MAX,
.gpr[22] = INT_MAX,
@@ -636,7 +815,7 @@
},
{
.descr = "RA = UINT_MAX, RB = UINT_MAX",
- .instr = TEST_ADD_DOT(20, 21, 22),
+ .instr = ppc_inst(PPC_RAW_ADD_DOT(20, 21, 22)),
.regs = {
.gpr[21] = UINT_MAX,
.gpr[22] = UINT_MAX,
@@ -644,7 +823,7 @@
},
{
.descr = "RA = UINT_MAX, RB = 0x1",
- .instr = TEST_ADD_DOT(20, 21, 22),
+ .instr = ppc_inst(PPC_RAW_ADD_DOT(20, 21, 22)),
.regs = {
.gpr[21] = UINT_MAX,
.gpr[22] = 0x1,
@@ -657,7 +836,7 @@
.subtests = {
{
.descr = "RA = LONG_MIN, RB = LONG_MIN",
- .instr = TEST_ADDC(20, 21, 22),
+ .instr = ppc_inst(PPC_RAW_ADDC(20, 21, 22)),
.regs = {
.gpr[21] = LONG_MIN,
.gpr[22] = LONG_MIN,
@@ -665,7 +844,7 @@
},
{
.descr = "RA = LONG_MIN, RB = LONG_MAX",
- .instr = TEST_ADDC(20, 21, 22),
+ .instr = ppc_inst(PPC_RAW_ADDC(20, 21, 22)),
.regs = {
.gpr[21] = LONG_MIN,
.gpr[22] = LONG_MAX,
@@ -673,7 +852,7 @@
},
{
.descr = "RA = LONG_MAX, RB = LONG_MAX",
- .instr = TEST_ADDC(20, 21, 22),
+ .instr = ppc_inst(PPC_RAW_ADDC(20, 21, 22)),
.regs = {
.gpr[21] = LONG_MAX,
.gpr[22] = LONG_MAX,
@@ -681,7 +860,7 @@
},
{
.descr = "RA = ULONG_MAX, RB = ULONG_MAX",
- .instr = TEST_ADDC(20, 21, 22),
+ .instr = ppc_inst(PPC_RAW_ADDC(20, 21, 22)),
.regs = {
.gpr[21] = ULONG_MAX,
.gpr[22] = ULONG_MAX,
@@ -689,7 +868,7 @@
},
{
.descr = "RA = ULONG_MAX, RB = 0x1",
- .instr = TEST_ADDC(20, 21, 22),
+ .instr = ppc_inst(PPC_RAW_ADDC(20, 21, 22)),
.regs = {
.gpr[21] = ULONG_MAX,
.gpr[22] = 0x1,
@@ -697,7 +876,7 @@
},
{
.descr = "RA = INT_MIN, RB = INT_MIN",
- .instr = TEST_ADDC(20, 21, 22),
+ .instr = ppc_inst(PPC_RAW_ADDC(20, 21, 22)),
.regs = {
.gpr[21] = INT_MIN,
.gpr[22] = INT_MIN,
@@ -705,7 +884,7 @@
},
{
.descr = "RA = INT_MIN, RB = INT_MAX",
- .instr = TEST_ADDC(20, 21, 22),
+ .instr = ppc_inst(PPC_RAW_ADDC(20, 21, 22)),
.regs = {
.gpr[21] = INT_MIN,
.gpr[22] = INT_MAX,
@@ -713,7 +892,7 @@
},
{
.descr = "RA = INT_MAX, RB = INT_MAX",
- .instr = TEST_ADDC(20, 21, 22),
+ .instr = ppc_inst(PPC_RAW_ADDC(20, 21, 22)),
.regs = {
.gpr[21] = INT_MAX,
.gpr[22] = INT_MAX,
@@ -721,7 +900,7 @@
},
{
.descr = "RA = UINT_MAX, RB = UINT_MAX",
- .instr = TEST_ADDC(20, 21, 22),
+ .instr = ppc_inst(PPC_RAW_ADDC(20, 21, 22)),
.regs = {
.gpr[21] = UINT_MAX,
.gpr[22] = UINT_MAX,
@@ -729,7 +908,7 @@
},
{
.descr = "RA = UINT_MAX, RB = 0x1",
- .instr = TEST_ADDC(20, 21, 22),
+ .instr = ppc_inst(PPC_RAW_ADDC(20, 21, 22)),
.regs = {
.gpr[21] = UINT_MAX,
.gpr[22] = 0x1,
@@ -737,7 +916,7 @@
},
{
.descr = "RA = LONG_MIN | INT_MIN, RB = LONG_MIN | INT_MIN",
- .instr = TEST_ADDC(20, 21, 22),
+ .instr = ppc_inst(PPC_RAW_ADDC(20, 21, 22)),
.regs = {
.gpr[21] = LONG_MIN | (uint)INT_MIN,
.gpr[22] = LONG_MIN | (uint)INT_MIN,
@@ -751,7 +930,7 @@
{
.descr = "RA = LONG_MIN, RB = LONG_MIN",
.flags = IGNORE_CCR,
- .instr = TEST_ADDC_DOT(20, 21, 22),
+ .instr = ppc_inst(PPC_RAW_ADDC_DOT(20, 21, 22)),
.regs = {
.gpr[21] = LONG_MIN,
.gpr[22] = LONG_MIN,
@@ -759,7 +938,7 @@
},
{
.descr = "RA = LONG_MIN, RB = LONG_MAX",
- .instr = TEST_ADDC_DOT(20, 21, 22),
+ .instr = ppc_inst(PPC_RAW_ADDC_DOT(20, 21, 22)),
.regs = {
.gpr[21] = LONG_MIN,
.gpr[22] = LONG_MAX,
@@ -768,7 +947,7 @@
{
.descr = "RA = LONG_MAX, RB = LONG_MAX",
.flags = IGNORE_CCR,
- .instr = TEST_ADDC_DOT(20, 21, 22),
+ .instr = ppc_inst(PPC_RAW_ADDC_DOT(20, 21, 22)),
.regs = {
.gpr[21] = LONG_MAX,
.gpr[22] = LONG_MAX,
@@ -776,7 +955,7 @@
},
{
.descr = "RA = ULONG_MAX, RB = ULONG_MAX",
- .instr = TEST_ADDC_DOT(20, 21, 22),
+ .instr = ppc_inst(PPC_RAW_ADDC_DOT(20, 21, 22)),
.regs = {
.gpr[21] = ULONG_MAX,
.gpr[22] = ULONG_MAX,
@@ -784,7 +963,7 @@
},
{
.descr = "RA = ULONG_MAX, RB = 0x1",
- .instr = TEST_ADDC_DOT(20, 21, 22),
+ .instr = ppc_inst(PPC_RAW_ADDC_DOT(20, 21, 22)),
.regs = {
.gpr[21] = ULONG_MAX,
.gpr[22] = 0x1,
@@ -792,7 +971,7 @@
},
{
.descr = "RA = INT_MIN, RB = INT_MIN",
- .instr = TEST_ADDC_DOT(20, 21, 22),
+ .instr = ppc_inst(PPC_RAW_ADDC_DOT(20, 21, 22)),
.regs = {
.gpr[21] = INT_MIN,
.gpr[22] = INT_MIN,
@@ -800,7 +979,7 @@
},
{
.descr = "RA = INT_MIN, RB = INT_MAX",
- .instr = TEST_ADDC_DOT(20, 21, 22),
+ .instr = ppc_inst(PPC_RAW_ADDC_DOT(20, 21, 22)),
.regs = {
.gpr[21] = INT_MIN,
.gpr[22] = INT_MAX,
@@ -808,7 +987,7 @@
},
{
.descr = "RA = INT_MAX, RB = INT_MAX",
- .instr = TEST_ADDC_DOT(20, 21, 22),
+ .instr = ppc_inst(PPC_RAW_ADDC_DOT(20, 21, 22)),
.regs = {
.gpr[21] = INT_MAX,
.gpr[22] = INT_MAX,
@@ -816,7 +995,7 @@
},
{
.descr = "RA = UINT_MAX, RB = UINT_MAX",
- .instr = TEST_ADDC_DOT(20, 21, 22),
+ .instr = ppc_inst(PPC_RAW_ADDC_DOT(20, 21, 22)),
.regs = {
.gpr[21] = UINT_MAX,
.gpr[22] = UINT_MAX,
@@ -824,7 +1003,7 @@
},
{
.descr = "RA = UINT_MAX, RB = 0x1",
- .instr = TEST_ADDC_DOT(20, 21, 22),
+ .instr = ppc_inst(PPC_RAW_ADDC_DOT(20, 21, 22)),
.regs = {
.gpr[21] = UINT_MAX,
.gpr[22] = 0x1,
@@ -832,31 +1011,320 @@
},
{
.descr = "RA = LONG_MIN | INT_MIN, RB = LONG_MIN | INT_MIN",
- .instr = TEST_ADDC_DOT(20, 21, 22),
+ .instr = ppc_inst(PPC_RAW_ADDC_DOT(20, 21, 22)),
.regs = {
.gpr[21] = LONG_MIN | (uint)INT_MIN,
.gpr[22] = LONG_MIN | (uint)INT_MIN,
}
}
}
+ },
+ {
+ .mnemonic = "divde",
+ .subtests = {
+ {
+ .descr = "RA = LONG_MIN, RB = LONG_MIN",
+ .instr = ppc_inst(PPC_RAW_DIVDE(20, 21, 22)),
+ .regs = {
+ .gpr[21] = LONG_MIN,
+ .gpr[22] = LONG_MIN,
+ }
+ },
+ {
+ .descr = "RA = 1L, RB = 0",
+ .instr = ppc_inst(PPC_RAW_DIVDE(20, 21, 22)),
+ .flags = IGNORE_GPR(20),
+ .regs = {
+ .gpr[21] = 1L,
+ .gpr[22] = 0,
+ }
+ },
+ {
+ .descr = "RA = LONG_MIN, RB = LONG_MAX",
+ .instr = ppc_inst(PPC_RAW_DIVDE(20, 21, 22)),
+ .regs = {
+ .gpr[21] = LONG_MIN,
+ .gpr[22] = LONG_MAX,
+ }
+ }
+ }
+ },
+ {
+ .mnemonic = "divde.",
+ .subtests = {
+ {
+ .descr = "RA = LONG_MIN, RB = LONG_MIN",
+ .instr = ppc_inst(PPC_RAW_DIVDE_DOT(20, 21, 22)),
+ .regs = {
+ .gpr[21] = LONG_MIN,
+ .gpr[22] = LONG_MIN,
+ }
+ },
+ {
+ .descr = "RA = 1L, RB = 0",
+ .instr = ppc_inst(PPC_RAW_DIVDE_DOT(20, 21, 22)),
+ .flags = IGNORE_GPR(20),
+ .regs = {
+ .gpr[21] = 1L,
+ .gpr[22] = 0,
+ }
+ },
+ {
+ .descr = "RA = LONG_MIN, RB = LONG_MAX",
+ .instr = ppc_inst(PPC_RAW_DIVDE_DOT(20, 21, 22)),
+ .regs = {
+ .gpr[21] = LONG_MIN,
+ .gpr[22] = LONG_MAX,
+ }
+ }
+ }
+ },
+ {
+ .mnemonic = "divdeu",
+ .subtests = {
+ {
+ .descr = "RA = LONG_MIN, RB = LONG_MIN",
+ .instr = ppc_inst(PPC_RAW_DIVDEU(20, 21, 22)),
+ .flags = IGNORE_GPR(20),
+ .regs = {
+ .gpr[21] = LONG_MIN,
+ .gpr[22] = LONG_MIN,
+ }
+ },
+ {
+ .descr = "RA = 1L, RB = 0",
+ .instr = ppc_inst(PPC_RAW_DIVDEU(20, 21, 22)),
+ .flags = IGNORE_GPR(20),
+ .regs = {
+ .gpr[21] = 1L,
+ .gpr[22] = 0,
+ }
+ },
+ {
+ .descr = "RA = LONG_MIN, RB = LONG_MAX",
+ .instr = ppc_inst(PPC_RAW_DIVDEU(20, 21, 22)),
+ .regs = {
+ .gpr[21] = LONG_MIN,
+ .gpr[22] = LONG_MAX,
+ }
+ },
+ {
+ .descr = "RA = LONG_MAX - 1, RB = LONG_MAX",
+ .instr = ppc_inst(PPC_RAW_DIVDEU(20, 21, 22)),
+ .regs = {
+ .gpr[21] = LONG_MAX - 1,
+ .gpr[22] = LONG_MAX,
+ }
+ },
+ {
+ .descr = "RA = LONG_MIN + 1, RB = LONG_MIN",
+ .instr = ppc_inst(PPC_RAW_DIVDEU(20, 21, 22)),
+ .flags = IGNORE_GPR(20),
+ .regs = {
+ .gpr[21] = LONG_MIN + 1,
+ .gpr[22] = LONG_MIN,
+ }
+ }
+ }
+ },
+ {
+ .mnemonic = "divdeu.",
+ .subtests = {
+ {
+ .descr = "RA = LONG_MIN, RB = LONG_MIN",
+ .instr = ppc_inst(PPC_RAW_DIVDEU_DOT(20, 21, 22)),
+ .flags = IGNORE_GPR(20),
+ .regs = {
+ .gpr[21] = LONG_MIN,
+ .gpr[22] = LONG_MIN,
+ }
+ },
+ {
+ .descr = "RA = 1L, RB = 0",
+ .instr = ppc_inst(PPC_RAW_DIVDEU_DOT(20, 21, 22)),
+ .flags = IGNORE_GPR(20),
+ .regs = {
+ .gpr[21] = 1L,
+ .gpr[22] = 0,
+ }
+ },
+ {
+ .descr = "RA = LONG_MIN, RB = LONG_MAX",
+ .instr = ppc_inst(PPC_RAW_DIVDEU_DOT(20, 21, 22)),
+ .regs = {
+ .gpr[21] = LONG_MIN,
+ .gpr[22] = LONG_MAX,
+ }
+ },
+ {
+ .descr = "RA = LONG_MAX - 1, RB = LONG_MAX",
+ .instr = ppc_inst(PPC_RAW_DIVDEU_DOT(20, 21, 22)),
+ .regs = {
+ .gpr[21] = LONG_MAX - 1,
+ .gpr[22] = LONG_MAX,
+ }
+ },
+ {
+ .descr = "RA = LONG_MIN + 1, RB = LONG_MIN",
+ .instr = ppc_inst(PPC_RAW_DIVDEU_DOT(20, 21, 22)),
+ .flags = IGNORE_GPR(20),
+ .regs = {
+ .gpr[21] = LONG_MIN + 1,
+ .gpr[22] = LONG_MIN,
+ }
+ }
+ }
+ },
+ {
+ .mnemonic = "paddi",
+ .cpu_feature = CPU_FTR_ARCH_31,
+ .subtests = {
+ {
+ .descr = "RA = LONG_MIN, SI = SI_MIN, R = 0",
+ .instr = TEST_PADDI(21, 22, SI_MIN, 0),
+ .regs = {
+ .gpr[21] = 0,
+ .gpr[22] = LONG_MIN,
+ }
+ },
+ {
+ .descr = "RA = LONG_MIN, SI = SI_MAX, R = 0",
+ .instr = TEST_PADDI(21, 22, SI_MAX, 0),
+ .regs = {
+ .gpr[21] = 0,
+ .gpr[22] = LONG_MIN,
+ }
+ },
+ {
+ .descr = "RA = LONG_MAX, SI = SI_MAX, R = 0",
+ .instr = TEST_PADDI(21, 22, SI_MAX, 0),
+ .regs = {
+ .gpr[21] = 0,
+ .gpr[22] = LONG_MAX,
+ }
+ },
+ {
+ .descr = "RA = ULONG_MAX, SI = SI_UMAX, R = 0",
+ .instr = TEST_PADDI(21, 22, SI_UMAX, 0),
+ .regs = {
+ .gpr[21] = 0,
+ .gpr[22] = ULONG_MAX,
+ }
+ },
+ {
+ .descr = "RA = ULONG_MAX, SI = 0x1, R = 0",
+ .instr = TEST_PADDI(21, 22, 0x1, 0),
+ .regs = {
+ .gpr[21] = 0,
+ .gpr[22] = ULONG_MAX,
+ }
+ },
+ {
+ .descr = "RA = INT_MIN, SI = SI_MIN, R = 0",
+ .instr = TEST_PADDI(21, 22, SI_MIN, 0),
+ .regs = {
+ .gpr[21] = 0,
+ .gpr[22] = INT_MIN,
+ }
+ },
+ {
+ .descr = "RA = INT_MIN, SI = SI_MAX, R = 0",
+ .instr = TEST_PADDI(21, 22, SI_MAX, 0),
+ .regs = {
+ .gpr[21] = 0,
+ .gpr[22] = INT_MIN,
+ }
+ },
+ {
+ .descr = "RA = INT_MAX, SI = SI_MAX, R = 0",
+ .instr = TEST_PADDI(21, 22, SI_MAX, 0),
+ .regs = {
+ .gpr[21] = 0,
+ .gpr[22] = INT_MAX,
+ }
+ },
+ {
+ .descr = "RA = UINT_MAX, SI = 0x1, R = 0",
+ .instr = TEST_PADDI(21, 22, 0x1, 0),
+ .regs = {
+ .gpr[21] = 0,
+ .gpr[22] = UINT_MAX,
+ }
+ },
+ {
+ .descr = "RA = UINT_MAX, SI = SI_MAX, R = 0",
+ .instr = TEST_PADDI(21, 22, SI_MAX, 0),
+ .regs = {
+ .gpr[21] = 0,
+ .gpr[22] = UINT_MAX,
+ }
+ },
+ {
+ .descr = "RA is r0, SI = SI_MIN, R = 0",
+ .instr = TEST_PADDI(21, 0, SI_MIN, 0),
+ .regs = {
+ .gpr[21] = 0x0,
+ }
+ },
+ {
+ .descr = "RA = 0, SI = SI_MIN, R = 0",
+ .instr = TEST_PADDI(21, 22, SI_MIN, 0),
+ .regs = {
+ .gpr[21] = 0x0,
+ .gpr[22] = 0x0,
+ }
+ },
+ {
+ .descr = "RA is r0, SI = 0, R = 1",
+ .instr = TEST_PADDI(21, 0, 0, 1),
+ .regs = {
+ .gpr[21] = 0,
+ }
+ },
+ {
+ .descr = "RA is r0, SI = SI_MIN, R = 1",
+ .instr = TEST_PADDI(21, 0, SI_MIN, 1),
+ .regs = {
+ .gpr[21] = 0,
+ }
+ },
+ /* Invalid instruction form with R = 1 and RA != 0 */
+ {
+ .descr = "RA = R22(0), SI = 0, R = 1",
+ .instr = TEST_PADDI(21, 22, 0, 1),
+ .flags = NEGATIVE_TEST,
+ .regs = {
+ .gpr[21] = 0,
+ .gpr[22] = 0,
+ }
+ }
+ }
}
};
static int __init emulate_compute_instr(struct pt_regs *regs,
- struct ppc_inst instr)
+ struct ppc_inst instr,
+ bool negative)
{
+ int analysed;
struct instruction_op op;
if (!regs || !ppc_inst_val(instr))
return -EINVAL;
- if (analyse_instr(&op, regs, instr) != 1 ||
- GETTYPE(op.type) != COMPUTE) {
- pr_info("emulation failed, instruction = 0x%08x\n", ppc_inst_val(instr));
+ regs->nip = patch_site_addr(&patch__exec_instr);
+
+ analysed = analyse_instr(&op, regs, instr);
+ if (analysed != 1 || GETTYPE(op.type) != COMPUTE) {
+ if (negative)
+ return -EFAULT;
+ pr_info("emulation failed, instruction = %s\n", ppc_inst_as_str(instr));
return -EFAULT;
}
-
- emulate_update_regs(regs, &op);
+ if (analysed == 1 && negative)
+ pr_info("negative test failed, instruction = %s\n", ppc_inst_as_str(instr));
+ if (!negative)
+ emulate_update_regs(regs, &op);
return 0;
}
@@ -864,7 +1332,6 @@
struct ppc_inst instr)
{
extern int exec_instr(struct pt_regs *regs);
- extern s32 patch__exec_instr;
if (!regs || !ppc_inst_val(instr))
return -EINVAL;
@@ -872,7 +1339,7 @@
/* Patch the NOP with the actual instruction */
patch_instruction_site(&patch__exec_instr, instr);
if (exec_instr(regs)) {
- pr_info("execution failed, instruction = 0x%08x\n", ppc_inst_val(instr));
+ pr_info("execution failed, instruction = %s\n", ppc_inst_as_str(instr));
return -EFAULT;
}
@@ -894,15 +1361,21 @@
struct pt_regs *regs, exp, got;
unsigned int i, j, k;
struct ppc_inst instr;
- bool ignore_gpr, ignore_xer, ignore_ccr, passed;
+ bool ignore_gpr, ignore_xer, ignore_ccr, passed, rc, negative;
for (i = 0; i < ARRAY_SIZE(compute_tests); i++) {
test = &compute_tests[i];
+ if (test->cpu_feature && !early_cpu_has_feature(test->cpu_feature)) {
+ show_result(test->mnemonic, "SKIP (!CPU_FTR)");
+ continue;
+ }
+
for (j = 0; j < MAX_SUBTESTS && test->subtests[j].descr; j++) {
instr = test->subtests[j].instr;
flags = test->subtests[j].flags;
regs = &test->subtests[j].regs;
+ negative = flags & NEGATIVE_TEST;
ignore_xer = flags & IGNORE_XER;
ignore_ccr = flags & IGNORE_CCR;
passed = true;
@@ -917,8 +1390,12 @@
exp.msr = MSR_KERNEL;
got.msr = MSR_KERNEL;
- if (emulate_compute_instr(&got, instr) ||
- execute_compute_instr(&exp, instr)) {
+ rc = emulate_compute_instr(&got, instr, negative) != 0;
+ if (negative) {
+ /* skip executing instruction */
+ passed = rc;
+ goto print;
+ } else if (rc || execute_compute_instr(&exp, instr)) {
passed = false;
goto print;
}
diff --git a/arch/powerpc/lib/test_emulate_step_exec_instr.S b/arch/powerpc/lib/test_emulate_step_exec_instr.S
index 1580f34..9ef941d 100644
--- a/arch/powerpc/lib/test_emulate_step_exec_instr.S
+++ b/arch/powerpc/lib/test_emulate_step_exec_instr.S
@@ -80,7 +80,9 @@
REST_NVGPRS(r31)
/* Placeholder for the test instruction */
+ .balign 64
1: nop
+ nop
patch_site 1b patch__exec_instr
/*
diff --git a/arch/powerpc/mm/book3s32/hash_low.S b/arch/powerpc/mm/book3s32/hash_low.S
index 923ad8f..1690d36 100644
--- a/arch/powerpc/mm/book3s32/hash_low.S
+++ b/arch/powerpc/mm/book3s32/hash_low.S
@@ -62,7 +62,7 @@
isync
#endif
/* Get PTE (linux-style) and check access */
- lis r0,KERNELBASE@h /* check if kernel address */
+ lis r0, TASK_SIZE@h /* check if kernel address */
cmplw 0,r4,r0
ori r3,r3,_PAGE_USER|_PAGE_PRESENT /* test low addresses as user */
mfspr r5, SPRN_SPRG_PGDIR /* phys page-table root */
diff --git a/arch/powerpc/mm/book3s32/mmu.c b/arch/powerpc/mm/book3s32/mmu.c
index 03b6ba5..c016291 100644
--- a/arch/powerpc/mm/book3s32/mmu.c
+++ b/arch/powerpc/mm/book3s32/mmu.c
@@ -187,6 +187,17 @@
return __mmu_mapin_ram(border, top);
}
+static bool is_module_segment(unsigned long addr)
+{
+ if (!IS_ENABLED(CONFIG_MODULES))
+ return false;
+ if (addr < ALIGN_DOWN(VMALLOC_START, SZ_256M))
+ return false;
+ if (addr >= ALIGN(VMALLOC_END, SZ_256M))
+ return false;
+ return true;
+}
+
void mmu_mark_initmem_nx(void)
{
int nb = mmu_has_feature(MMU_FTR_USE_HIGH_BATS) ? 8 : 4;
@@ -223,9 +234,9 @@
for (i = TASK_SIZE >> 28; i < 16; i++) {
/* Do not set NX on VM space for modules */
- if (IS_ENABLED(CONFIG_MODULES) &&
- (VMALLOC_START & 0xf0000000) == i << 28)
- break;
+ if (is_module_segment(i << 28))
+ continue;
+
mtsrin(mfsrin(i << 28) | 0x10000000, i << 28);
}
}
diff --git a/arch/powerpc/mm/book3s64/hash_utils.c b/arch/powerpc/mm/book3s64/hash_utils.c
index 9b9f92a..1478fce 100644
--- a/arch/powerpc/mm/book3s64/hash_utils.c
+++ b/arch/powerpc/mm/book3s64/hash_utils.c
@@ -232,8 +232,6 @@
rflags |= HPTE_R_I;
else if ((pteflags & _PAGE_CACHE_CTL) == _PAGE_NON_IDEMPOTENT)
rflags |= (HPTE_R_I | HPTE_R_G);
- else if ((pteflags & _PAGE_CACHE_CTL) == _PAGE_SAO)
- rflags |= (HPTE_R_W | HPTE_R_I | HPTE_R_M);
else
/*
* Add memory coherence if cache inhibited is not set
@@ -596,7 +594,7 @@
}
#ifdef CONFIG_HUGETLB_PAGE
- if (!hugetlb_disabled) {
+ if (!hugetlb_disabled && !early_radix_enabled() ) {
/* Reserve 16G huge page memory sections for huge pages */
of_scan_flat_dt(htab_dt_scan_hugepage_blocks, NULL);
}
@@ -663,11 +661,10 @@
* Pick a size for the linear mapping. Currently, we only
* support 16M, 1M and 4K which is the default
*/
- if (IS_ENABLED(STRICT_KERNEL_RWX) &&
+ if (IS_ENABLED(CONFIG_STRICT_KERNEL_RWX) &&
(unsigned long)_stext % 0x1000000) {
if (mmu_psize_defs[MMU_PAGE_16M].shift)
- pr_warn("Kernel not 16M aligned, "
- "disabling 16M linear map alignment");
+ pr_warn("Kernel not 16M aligned, disabling 16M linear map alignment\n");
aligned = false;
}
@@ -788,7 +785,7 @@
}
#ifdef CONFIG_MEMORY_HOTPLUG
-int resize_hpt_for_hotplug(unsigned long new_mem_size)
+static int resize_hpt_for_hotplug(unsigned long new_mem_size)
{
unsigned target_hpt_shift;
@@ -822,6 +819,8 @@
return -1;
}
+ resize_hpt_for_hotplug(memblock_phys_mem_size());
+
rc = htab_bolt_mapping(start, end, __pa(start),
pgprot_val(prot), mmu_linear_psize,
mmu_kernel_ssize);
@@ -839,6 +838,10 @@
int rc = htab_remove_mapping(start, end, mmu_linear_psize,
mmu_kernel_ssize);
WARN_ON(rc < 0);
+
+ if (resize_hpt_for_hotplug(memblock_phys_mem_size()) == -ENOSPC)
+ pr_warn("Hash collision while resizing HPT\n");
+
return rc;
}
#endif /* CONFIG_MEMORY_HOTPLUG */
@@ -1111,6 +1114,10 @@
if (cpu_has_feature(CPU_FTR_ARCH_206)
&& cpu_has_feature(CPU_FTR_HVMODE))
tlbiel_all();
+
+#ifdef CONFIG_PPC_MEM_KEYS
+ mtspr(SPRN_UAMOR, default_uamor);
+#endif
}
#endif /* CONFIG_SMP */
@@ -1731,10 +1738,6 @@
return gslot;
}
-/*
- * WARNING: This is called from hash_low_64.S, if you change this prototype,
- * do not forget to update the assembly call site !
- */
void flush_hash_page(unsigned long vpn, real_pte_t pte, int psize, int ssize,
unsigned long flags)
{
diff --git a/arch/powerpc/mm/book3s64/pgtable.c b/arch/powerpc/mm/book3s64/pgtable.c
index c58ad10..e18ae50 100644
--- a/arch/powerpc/mm/book3s64/pgtable.c
+++ b/arch/powerpc/mm/book3s64/pgtable.c
@@ -15,6 +15,7 @@
#include <asm/powernv.h>
#include <asm/firmware.h>
#include <asm/ultravisor.h>
+#include <asm/kexec.h>
#include <mm/mmu_decl.h>
#include <trace/events/thp.h>
@@ -165,6 +166,8 @@
radix__mmu_cleanup_all();
else if (mmu_hash_ops.hpte_clear_all)
mmu_hash_ops.hpte_clear_all();
+
+ reset_sprs();
}
#ifdef CONFIG_MEMORY_HOTPLUG
@@ -339,6 +342,9 @@
{
struct page *page = virt_to_page(pmd);
+ if (PageReserved(page))
+ return free_reserved_page(page);
+
BUG_ON(atomic_read(&page->pt_frag_refcount) <= 0);
if (atomic_dec_and_test(&page->pt_frag_refcount)) {
pgtable_pmd_page_dtor(page);
@@ -356,7 +362,7 @@
pmd_fragment_free(table);
break;
case PUD_INDEX:
- kmem_cache_free(PGT_CACHE(PUD_CACHE_INDEX), table);
+ __pud_free(table);
break;
#if defined(CONFIG_PPC_4K_PAGES) && defined(CONFIG_HUGETLB_PAGE)
/* 16M hugepd directory at pud level */
diff --git a/arch/powerpc/mm/book3s64/pkeys.c b/arch/powerpc/mm/book3s64/pkeys.c
index d174106..69a6b87 100644
--- a/arch/powerpc/mm/book3s64/pkeys.c
+++ b/arch/powerpc/mm/book3s64/pkeys.c
@@ -10,58 +10,103 @@
#include <asm/mmu.h>
#include <asm/setup.h>
#include <linux/pkeys.h>
-#include <linux/of_device.h>
+#include <linux/of_fdt.h>
-DEFINE_STATIC_KEY_TRUE(pkey_disabled);
-int pkeys_total; /* Total pkeys as per device tree */
-u32 initial_allocation_mask; /* Bits set for the initially allocated keys */
-u32 reserved_allocation_mask; /* Bits set for reserved keys */
-static bool pkey_execute_disable_supported;
-static bool pkeys_devtree_defined; /* property exported by device tree */
-static u64 pkey_amr_mask; /* Bits in AMR not to be touched */
-static u64 pkey_iamr_mask; /* Bits in AMR not to be touched */
-static u64 pkey_uamor_mask; /* Bits in UMOR not to be touched */
+int num_pkey; /* Max number of pkeys supported */
+/*
+ * Keys marked in the reservation list cannot be allocated by userspace
+ */
+u32 reserved_allocation_mask __ro_after_init;
+
+/* Bits set for the initially allocated keys */
+static u32 initial_allocation_mask __ro_after_init;
+
+/*
+ * Even if we allocate keys with sys_pkey_alloc(), we need to make sure
+ * other thread still find the access denied using the same keys.
+ */
+static u64 default_amr = ~0x0UL;
+static u64 default_iamr = 0x5555555555555555UL;
+u64 default_uamor __ro_after_init;
+/*
+ * Key used to implement PROT_EXEC mmap. Denies READ/WRITE
+ * We pick key 2 because 0 is special key and 1 is reserved as per ISA.
+ */
static int execute_only_key = 2;
+static bool pkey_execute_disable_supported;
+
#define AMR_BITS_PER_PKEY 2
#define AMR_RD_BIT 0x1UL
#define AMR_WR_BIT 0x2UL
#define IAMR_EX_BIT 0x1UL
-#define PKEY_REG_BITS (sizeof(u64)*8)
+#define PKEY_REG_BITS (sizeof(u64) * 8)
#define pkeyshift(pkey) (PKEY_REG_BITS - ((pkey+1) * AMR_BITS_PER_PKEY))
-static void scan_pkey_feature(void)
+static int __init dt_scan_storage_keys(unsigned long node,
+ const char *uname, int depth,
+ void *data)
{
- u32 vals[2];
- struct device_node *cpu;
+ const char *type = of_get_flat_dt_prop(node, "device_type", NULL);
+ const __be32 *prop;
+ int *pkeys_total = (int *) data;
- cpu = of_find_node_by_type(NULL, "cpu");
- if (!cpu)
- return;
+ /* We are scanning "cpu" nodes only */
+ if (type == NULL || strcmp(type, "cpu") != 0)
+ return 0;
- if (of_property_read_u32_array(cpu,
- "ibm,processor-storage-keys", vals, 2))
- return;
+ prop = of_get_flat_dt_prop(node, "ibm,processor-storage-keys", NULL);
+ if (!prop)
+ return 0;
+ *pkeys_total = be32_to_cpu(prop[0]);
+ return 1;
+}
+
+static int scan_pkey_feature(void)
+{
+ int ret;
+ int pkeys_total = 0;
/*
- * Since any pkey can be used for data or execute, we will just treat
- * all keys as equal and track them as one entity.
+ * Pkey is not supported with Radix translation.
*/
- pkeys_total = vals[0];
- pkeys_devtree_defined = true;
+ if (early_radix_enabled())
+ return 0;
+
+ /*
+ * Only P7 and above supports SPRN_AMR update with MSR[PR] = 1
+ */
+ if (!early_cpu_has_feature(CPU_FTR_ARCH_206))
+ return 0;
+
+ ret = of_scan_flat_dt(dt_scan_storage_keys, &pkeys_total);
+ if (ret == 0) {
+ /*
+ * Let's assume 32 pkeys on P8/P9 bare metal, if its not defined by device
+ * tree. We make this exception since some version of skiboot forgot to
+ * expose this property on power8/9.
+ */
+ if (!firmware_has_feature(FW_FEATURE_LPAR)) {
+ unsigned long pvr = mfspr(SPRN_PVR);
+
+ if (PVR_VER(pvr) == PVR_POWER8 || PVR_VER(pvr) == PVR_POWER8E ||
+ PVR_VER(pvr) == PVR_POWER8NVL || PVR_VER(pvr) == PVR_POWER9)
+ pkeys_total = 32;
+ }
+ }
+
+ /*
+ * Adjust the upper limit, based on the number of bits supported by
+ * arch-neutral code.
+ */
+ pkeys_total = min_t(int, pkeys_total,
+ ((ARCH_VM_PKEY_FLAGS >> VM_PKEY_SHIFT) + 1));
+ return pkeys_total;
}
-static inline bool pkey_mmu_enabled(void)
+void __init pkey_early_init_devtree(void)
{
- if (firmware_has_feature(FW_FEATURE_LPAR))
- return pkeys_total;
- else
- return cpu_has_feature(CPU_FTR_PKEY);
-}
-
-static int pkey_initialize(void)
-{
- int os_reserved, i;
+ int pkeys_total, i;
/*
* We define PKEY_DISABLE_EXECUTE in addition to the arch-neutral
@@ -80,31 +125,14 @@
!= (sizeof(u64) * BITS_PER_BYTE));
/* scan the device tree for pkey feature */
- scan_pkey_feature();
+ pkeys_total = scan_pkey_feature();
+ if (!pkeys_total)
+ goto out;
- /*
- * Let's assume 32 pkeys on P8 bare metal, if its not defined by device
- * tree. We make this exception since skiboot forgot to expose this
- * property on power8.
- */
- if (!pkeys_devtree_defined && !firmware_has_feature(FW_FEATURE_LPAR) &&
- cpu_has_feature(CPU_FTRS_POWER8))
- pkeys_total = 32;
+ /* Allow all keys to be modified by default */
+ default_uamor = ~0x0UL;
- /*
- * Adjust the upper limit, based on the number of bits supported by
- * arch-neutral code.
- */
- pkeys_total = min_t(int, pkeys_total,
- ((ARCH_VM_PKEY_FLAGS >> VM_PKEY_SHIFT)+1));
-
- if (!pkey_mmu_enabled() || radix_enabled() || !pkeys_total)
- static_branch_enable(&pkey_disabled);
- else
- static_branch_disable(&pkey_disabled);
-
- if (static_branch_likely(&pkey_disabled))
- return 0;
+ cur_cpu_spec->mmu_features |= MMU_FTR_PKEY;
/*
* The device tree cannot be relied to indicate support for
@@ -118,53 +146,86 @@
#ifdef CONFIG_PPC_4K_PAGES
/*
* The OS can manage only 8 pkeys due to its inability to represent them
- * in the Linux 4K PTE.
+ * in the Linux 4K PTE. Mark all other keys reserved.
*/
- os_reserved = pkeys_total - 8;
+ num_pkey = min(8, pkeys_total);
#else
- os_reserved = 0;
+ num_pkey = pkeys_total;
#endif
- /* Bits are in LE format. */
- reserved_allocation_mask = (0x1 << 1) | (0x1 << execute_only_key);
- /* register mask is in BE format */
- pkey_amr_mask = ~0x0ul;
- pkey_amr_mask &= ~(0x3ul << pkeyshift(0));
-
- pkey_iamr_mask = ~0x0ul;
- pkey_iamr_mask &= ~(0x3ul << pkeyshift(0));
- pkey_iamr_mask &= ~(0x3ul << pkeyshift(execute_only_key));
-
- pkey_uamor_mask = ~0x0ul;
- pkey_uamor_mask &= ~(0x3ul << pkeyshift(0));
- pkey_uamor_mask &= ~(0x3ul << pkeyshift(execute_only_key));
-
- /* mark the rest of the keys as reserved and hence unavailable */
- for (i = (pkeys_total - os_reserved); i < pkeys_total; i++) {
- reserved_allocation_mask |= (0x1 << i);
- pkey_uamor_mask &= ~(0x3ul << pkeyshift(i));
- }
- initial_allocation_mask = reserved_allocation_mask | (0x1 << 0);
-
- if (unlikely((pkeys_total - os_reserved) <= execute_only_key)) {
+ if (unlikely(num_pkey <= execute_only_key) || !pkey_execute_disable_supported) {
/*
* Insufficient number of keys to support
* execute only key. Mark it unavailable.
- * Any AMR, UAMOR, IAMR bit set for
- * this key is irrelevant since this key
- * can never be allocated.
*/
execute_only_key = -1;
+ } else {
+ /*
+ * Mark the execute_only_pkey as not available for
+ * user allocation via pkey_alloc.
+ */
+ reserved_allocation_mask |= (0x1 << execute_only_key);
+
+ /*
+ * Deny READ/WRITE for execute_only_key.
+ * Allow execute in IAMR.
+ */
+ default_amr |= (0x3ul << pkeyshift(execute_only_key));
+ default_iamr &= ~(0x1ul << pkeyshift(execute_only_key));
+
+ /*
+ * Clear the uamor bits for this key.
+ */
+ default_uamor &= ~(0x3ul << pkeyshift(execute_only_key));
}
- return 0;
-}
+ /*
+ * Allow access for only key 0. And prevent any other modification.
+ */
+ default_amr &= ~(0x3ul << pkeyshift(0));
+ default_iamr &= ~(0x1ul << pkeyshift(0));
+ default_uamor &= ~(0x3ul << pkeyshift(0));
+ /*
+ * key 0 is special in that we want to consider it an allocated
+ * key which is preallocated. We don't allow changing AMR bits
+ * w.r.t key 0. But one can pkey_free(key0)
+ */
+ initial_allocation_mask |= (0x1 << 0);
-arch_initcall(pkey_initialize);
+ /*
+ * key 1 is recommended not to be used. PowerISA(3.0) page 1015,
+ * programming note.
+ */
+ reserved_allocation_mask |= (0x1 << 1);
+ default_uamor &= ~(0x3ul << pkeyshift(1));
+
+ /*
+ * Prevent the usage of OS reserved keys. Update UAMOR
+ * for those keys. Also mark the rest of the bits in the
+ * 32 bit mask as reserved.
+ */
+ for (i = num_pkey; i < 32 ; i++) {
+ reserved_allocation_mask |= (0x1 << i);
+ default_uamor &= ~(0x3ul << pkeyshift(i));
+ }
+ /*
+ * Prevent the allocation of reserved keys too.
+ */
+ initial_allocation_mask |= reserved_allocation_mask;
+
+ pr_info("Enabling pkeys with max key count %d\n", num_pkey);
+out:
+ /*
+ * Setup uamor on boot cpu
+ */
+ mtspr(SPRN_UAMOR, default_uamor);
+
+ return;
+}
void pkey_mm_init(struct mm_struct *mm)
{
- if (static_branch_likely(&pkey_disabled))
+ if (!mmu_has_feature(MMU_FTR_PKEY))
return;
mm_pkey_allocation_map(mm) = initial_allocation_mask;
mm->context.execute_only_pkey = execute_only_key;
@@ -196,30 +257,6 @@
mtspr(SPRN_IAMR, value);
}
-static inline u64 read_uamor(void)
-{
- return mfspr(SPRN_UAMOR);
-}
-
-static inline void write_uamor(u64 value)
-{
- mtspr(SPRN_UAMOR, value);
-}
-
-static bool is_pkey_enabled(int pkey)
-{
- u64 uamor = read_uamor();
- u64 pkey_bits = 0x3ul << pkeyshift(pkey);
- u64 uamor_pkey_bits = (uamor & pkey_bits);
-
- /*
- * Both the bits in UAMOR corresponding to the key should be set or
- * reset.
- */
- WARN_ON(uamor_pkey_bits && (uamor_pkey_bits != pkey_bits));
- return !!(uamor_pkey_bits);
-}
-
static inline void init_amr(int pkey, u8 init_bits)
{
u64 new_amr_bits = (((u64)init_bits & 0x3UL) << pkeyshift(pkey));
@@ -245,8 +282,18 @@
{
u64 new_amr_bits = 0x0ul;
u64 new_iamr_bits = 0x0ul;
+ u64 pkey_bits, uamor_pkey_bits;
- if (!is_pkey_enabled(pkey))
+ /*
+ * Check whether the key is disabled by UAMOR.
+ */
+ pkey_bits = 0x3ul << pkeyshift(pkey);
+ uamor_pkey_bits = (default_uamor & pkey_bits);
+
+ /*
+ * Both the bits in UAMOR corresponding to the key should be set
+ */
+ if (uamor_pkey_bits != pkey_bits)
return -EINVAL;
if (init_val & PKEY_DISABLE_EXECUTE) {
@@ -268,7 +315,7 @@
void thread_pkey_regs_save(struct thread_struct *thread)
{
- if (static_branch_likely(&pkey_disabled))
+ if (!mmu_has_feature(MMU_FTR_PKEY))
return;
/*
@@ -276,38 +323,33 @@
*/
thread->amr = read_amr();
thread->iamr = read_iamr();
- thread->uamor = read_uamor();
}
void thread_pkey_regs_restore(struct thread_struct *new_thread,
struct thread_struct *old_thread)
{
- if (static_branch_likely(&pkey_disabled))
+ if (!mmu_has_feature(MMU_FTR_PKEY))
return;
if (old_thread->amr != new_thread->amr)
write_amr(new_thread->amr);
if (old_thread->iamr != new_thread->iamr)
write_iamr(new_thread->iamr);
- if (old_thread->uamor != new_thread->uamor)
- write_uamor(new_thread->uamor);
}
void thread_pkey_regs_init(struct thread_struct *thread)
{
- if (static_branch_likely(&pkey_disabled))
+ if (!mmu_has_feature(MMU_FTR_PKEY))
return;
- thread->amr = pkey_amr_mask;
- thread->iamr = pkey_iamr_mask;
- thread->uamor = pkey_uamor_mask;
+ thread->amr = default_amr;
+ thread->iamr = default_iamr;
- write_uamor(pkey_uamor_mask);
- write_amr(pkey_amr_mask);
- write_iamr(pkey_iamr_mask);
+ write_amr(default_amr);
+ write_iamr(default_iamr);
}
-int __execute_only_pkey(struct mm_struct *mm)
+int execute_only_pkey(struct mm_struct *mm)
{
return mm->context.execute_only_pkey;
}
@@ -366,7 +408,7 @@
bool arch_pte_access_permitted(u64 pte, bool write, bool execute)
{
- if (static_branch_likely(&pkey_disabled))
+ if (!mmu_has_feature(MMU_FTR_PKEY))
return true;
return pkey_access_permitted(pte_to_pkey_bits(pte), write, execute);
@@ -383,7 +425,7 @@
bool arch_vma_access_permitted(struct vm_area_struct *vma, bool write,
bool execute, bool foreign)
{
- if (static_branch_likely(&pkey_disabled))
+ if (!mmu_has_feature(MMU_FTR_PKEY))
return true;
/*
* Do not enforce our key-permissions on a foreign vma.
@@ -396,7 +438,7 @@
void arch_dup_pkeys(struct mm_struct *oldmm, struct mm_struct *mm)
{
- if (static_branch_likely(&pkey_disabled))
+ if (!mmu_has_feature(MMU_FTR_PKEY))
return;
/* Duplicate the oldmm pkey state in mm: */
diff --git a/arch/powerpc/mm/book3s64/radix_pgtable.c b/arch/powerpc/mm/book3s64/radix_pgtable.c
index bb00e0c..28c7849 100644
--- a/arch/powerpc/mm/book3s64/radix_pgtable.c
+++ b/arch/powerpc/mm/book3s64/radix_pgtable.c
@@ -15,7 +15,7 @@
#include <linux/mm.h>
#include <linux/hugetlb.h>
#include <linux/string_helpers.h>
-#include <linux/stop_machine.h>
+#include <linux/memory.h>
#include <asm/pgalloc.h>
#include <asm/mmu_context.h>
@@ -34,6 +34,7 @@
unsigned int mmu_pid_bits;
unsigned int mmu_base_pid;
+unsigned int radix_mem_block_size __ro_after_init;
static __ref void *early_alloc_pgtable(unsigned long size, int nid,
unsigned long region_start, unsigned long region_end)
@@ -56,6 +57,13 @@
return ptr;
}
+/*
+ * When allocating pud or pmd pointers, we allocate a complete page
+ * of PAGE_SIZE rather than PUD_TABLE_SIZE or PMD_TABLE_SIZE. This
+ * is to ensure that the page obtained from the memblock allocator
+ * can be completely used as page table page and can be freed
+ * correctly when the page table entries are removed.
+ */
static int early_map_kernel_page(unsigned long ea, unsigned long pa,
pgprot_t flags,
unsigned int map_page_size,
@@ -72,8 +80,8 @@
pgdp = pgd_offset_k(ea);
p4dp = p4d_offset(pgdp, ea);
if (p4d_none(*p4dp)) {
- pudp = early_alloc_pgtable(PUD_TABLE_SIZE, nid,
- region_start, region_end);
+ pudp = early_alloc_pgtable(PAGE_SIZE, nid,
+ region_start, region_end);
p4d_populate(&init_mm, p4dp, pudp);
}
pudp = pud_offset(p4dp, ea);
@@ -82,8 +90,8 @@
goto set_the_pte;
}
if (pud_none(*pudp)) {
- pmdp = early_alloc_pgtable(PMD_TABLE_SIZE, nid,
- region_start, region_end);
+ pmdp = early_alloc_pgtable(PAGE_SIZE, nid, region_start,
+ region_end);
pud_populate(&init_mm, pudp, pmdp);
}
pmdp = pmd_offset(pudp, ea);
@@ -259,6 +267,7 @@
static int __meminit create_physical_mapping(unsigned long start,
unsigned long end,
+ unsigned long max_mapping_size,
int nid, pgprot_t _prot)
{
unsigned long vaddr, addr, mapping_size = 0;
@@ -272,6 +281,8 @@
int rc;
gap = next_boundary(addr, end) - addr;
+ if (gap > max_mapping_size)
+ gap = max_mapping_size;
previous_size = mapping_size;
prev_exec = exec;
@@ -322,8 +333,9 @@
/* We don't support slb for radix */
mmu_slb_size = 0;
+
/*
- * Create the linear mapping, using standard page size for now
+ * Create the linear mapping
*/
for_each_memblock(memory, reg) {
/*
@@ -339,6 +351,7 @@
WARN_ON(create_physical_mapping(reg->base,
reg->base + reg->size,
+ radix_mem_block_size,
-1, PAGE_KERNEL));
}
@@ -479,6 +492,57 @@
return 1;
}
+#ifdef CONFIG_MEMORY_HOTPLUG
+static int __init probe_memory_block_size(unsigned long node, const char *uname, int
+ depth, void *data)
+{
+ unsigned long *mem_block_size = (unsigned long *)data;
+ const __be64 *prop;
+ int len;
+
+ if (depth != 1)
+ return 0;
+
+ if (strcmp(uname, "ibm,dynamic-reconfiguration-memory"))
+ return 0;
+
+ prop = of_get_flat_dt_prop(node, "ibm,lmb-size", &len);
+ if (!prop || len < sizeof(__be64))
+ /*
+ * Nothing in the device tree
+ */
+ *mem_block_size = MIN_MEMORY_BLOCK_SIZE;
+ else
+ *mem_block_size = be64_to_cpup(prop);
+ return 1;
+}
+
+static unsigned long radix_memory_block_size(void)
+{
+ unsigned long mem_block_size = MIN_MEMORY_BLOCK_SIZE;
+
+ /*
+ * OPAL firmware feature is set by now. Hence we are ok
+ * to test OPAL feature.
+ */
+ if (firmware_has_feature(FW_FEATURE_OPAL))
+ mem_block_size = 1UL * 1024 * 1024 * 1024;
+ else
+ of_scan_flat_dt(probe_memory_block_size, &mem_block_size);
+
+ return mem_block_size;
+}
+
+#else /* CONFIG_MEMORY_HOTPLUG */
+
+static unsigned long radix_memory_block_size(void)
+{
+ return 1UL * 1024 * 1024 * 1024;
+}
+
+#endif /* CONFIG_MEMORY_HOTPLUG */
+
+
void __init radix__early_init_devtree(void)
{
int rc;
@@ -487,17 +551,27 @@
* Try to find the available page sizes in the device-tree
*/
rc = of_scan_flat_dt(radix_dt_scan_page_sizes, NULL);
- if (rc != 0) /* Found */
- goto found;
- /*
- * let's assume we have page 4k and 64k support
- */
- mmu_psize_defs[MMU_PAGE_4K].shift = 12;
- mmu_psize_defs[MMU_PAGE_4K].ap = 0x0;
+ if (!rc) {
+ /*
+ * No page size details found in device tree.
+ * Let's assume we have page 4k and 64k support
+ */
+ mmu_psize_defs[MMU_PAGE_4K].shift = 12;
+ mmu_psize_defs[MMU_PAGE_4K].ap = 0x0;
- mmu_psize_defs[MMU_PAGE_64K].shift = 16;
- mmu_psize_defs[MMU_PAGE_64K].ap = 0x5;
-found:
+ mmu_psize_defs[MMU_PAGE_64K].shift = 16;
+ mmu_psize_defs[MMU_PAGE_64K].ap = 0x5;
+ }
+
+ /*
+ * Max mapping size used when mapping pages. We don't use
+ * ppc_md.memory_block_size() here because this get called
+ * early and we don't have machine probe called yet. Also
+ * the pseries implementation only check for ibm,lmb-size.
+ * All hypervisor supporting radix do expose that device
+ * tree node.
+ */
+ radix_mem_block_size = radix_memory_block_size();
return;
}
@@ -519,8 +593,10 @@
if (disabled || !early_radix_enabled())
return;
- if (smp_processor_id() == boot_cpuid)
+ if (smp_processor_id() == boot_cpuid) {
pr_info("Activating Kernel Userspace Execution Prevention\n");
+ cur_cpu_spec->mmu_features |= MMU_FTR_KUEP;
+ }
/*
* Radix always uses key0 of the IAMR to determine if an access is
@@ -544,6 +620,10 @@
/* Make sure userspace can't change the AMR */
mtspr(SPRN_UAMOR, 0);
+
+ /*
+ * Set the default kernel AMR values on all cpus.
+ */
mtspr(SPRN_AMR, AMR_KUAP_BLOCKED);
isync();
}
@@ -700,30 +780,19 @@
pud_clear(pud);
}
-struct change_mapping_params {
- pte_t *pte;
- unsigned long start;
- unsigned long end;
- unsigned long aligned_start;
- unsigned long aligned_end;
-};
-
-static int __meminit stop_machine_change_mapping(void *data)
+static void free_pud_table(pud_t *pud_start, p4d_t *p4d)
{
- struct change_mapping_params *params =
- (struct change_mapping_params *)data;
+ pud_t *pud;
+ int i;
- if (!data)
- return -1;
+ for (i = 0; i < PTRS_PER_PUD; i++) {
+ pud = pud_start + i;
+ if (!pud_none(*pud))
+ return;
+ }
- spin_unlock(&init_mm.page_table_lock);
- pte_clear(&init_mm, params->aligned_start, params->pte);
- create_physical_mapping(__pa(params->aligned_start),
- __pa(params->start), -1, PAGE_KERNEL);
- create_physical_mapping(__pa(params->end), __pa(params->aligned_end),
- -1, PAGE_KERNEL);
- spin_lock(&init_mm.page_table_lock);
- return 0;
+ pud_free(&init_mm, pud_start);
+ p4d_clear(p4d);
}
static void remove_pte_table(pte_t *pte_start, unsigned long addr,
@@ -754,53 +823,7 @@
}
}
-/*
- * clear the pte and potentially split the mapping helper
- */
-static void __meminit split_kernel_mapping(unsigned long addr, unsigned long end,
- unsigned long size, pte_t *pte)
-{
- unsigned long mask = ~(size - 1);
- unsigned long aligned_start = addr & mask;
- unsigned long aligned_end = addr + size;
- struct change_mapping_params params;
- bool split_region = false;
-
- if ((end - addr) < size) {
- /*
- * We're going to clear the PTE, but not flushed
- * the mapping, time to remap and flush. The
- * effects if visible outside the processor or
- * if we are running in code close to the
- * mapping we cleared, we are in trouble.
- */
- if (overlaps_kernel_text(aligned_start, addr) ||
- overlaps_kernel_text(end, aligned_end)) {
- /*
- * Hack, just return, don't pte_clear
- */
- WARN_ONCE(1, "Linear mapping %lx->%lx overlaps kernel "
- "text, not splitting\n", addr, end);
- return;
- }
- split_region = true;
- }
-
- if (split_region) {
- params.pte = pte;
- params.start = addr;
- params.end = end;
- params.aligned_start = addr & ~(size - 1);
- params.aligned_end = min_t(unsigned long, aligned_end,
- (unsigned long)__va(memblock_end_of_DRAM()));
- stop_machine(stop_machine_change_mapping, ¶ms, NULL);
- return;
- }
-
- pte_clear(&init_mm, addr, pte);
-}
-
-static void remove_pmd_table(pmd_t *pmd_start, unsigned long addr,
+static void __meminit remove_pmd_table(pmd_t *pmd_start, unsigned long addr,
unsigned long end)
{
unsigned long next;
@@ -815,7 +838,12 @@
continue;
if (pmd_is_leaf(*pmd)) {
- split_kernel_mapping(addr, end, PMD_SIZE, (pte_t *)pmd);
+ if (!IS_ALIGNED(addr, PMD_SIZE) ||
+ !IS_ALIGNED(next, PMD_SIZE)) {
+ WARN_ONCE(1, "%s: unaligned range\n", __func__);
+ continue;
+ }
+ pte_clear(&init_mm, addr, (pte_t *)pmd);
continue;
}
@@ -825,7 +853,7 @@
}
}
-static void remove_pud_table(pud_t *pud_start, unsigned long addr,
+static void __meminit remove_pud_table(pud_t *pud_start, unsigned long addr,
unsigned long end)
{
unsigned long next;
@@ -840,7 +868,12 @@
continue;
if (pud_is_leaf(*pud)) {
- split_kernel_mapping(addr, end, PUD_SIZE, (pte_t *)pud);
+ if (!IS_ALIGNED(addr, PUD_SIZE) ||
+ !IS_ALIGNED(next, PUD_SIZE)) {
+ WARN_ONCE(1, "%s: unaligned range\n", __func__);
+ continue;
+ }
+ pte_clear(&init_mm, addr, (pte_t *)pud);
continue;
}
@@ -868,12 +901,19 @@
continue;
if (p4d_is_leaf(*p4d)) {
- split_kernel_mapping(addr, end, P4D_SIZE, (pte_t *)p4d);
+ if (!IS_ALIGNED(addr, P4D_SIZE) ||
+ !IS_ALIGNED(next, P4D_SIZE)) {
+ WARN_ONCE(1, "%s: unaligned range\n", __func__);
+ continue;
+ }
+
+ pte_clear(&init_mm, addr, (pte_t *)pgd);
continue;
}
pud_base = (pud_t *)p4d_page_vaddr(*p4d);
remove_pud_table(pud_base, addr, next);
+ free_pud_table(pud_base, p4d);
}
spin_unlock(&init_mm.page_table_lock);
@@ -889,7 +929,8 @@
return -1;
}
- return create_physical_mapping(__pa(start), __pa(end), nid, prot);
+ return create_physical_mapping(__pa(start), __pa(end),
+ radix_mem_block_size, nid, prot);
}
int __meminit radix__remove_section_mapping(unsigned long start, unsigned long end)
diff --git a/arch/powerpc/mm/book3s64/radix_tlb.c b/arch/powerpc/mm/book3s64/radix_tlb.c
index b5cc9b2..0d23376 100644
--- a/arch/powerpc/mm/book3s64/radix_tlb.c
+++ b/arch/powerpc/mm/book3s64/radix_tlb.c
@@ -16,6 +16,7 @@
#include <asm/tlbflush.h>
#include <asm/trace.h>
#include <asm/cputhreads.h>
+#include <asm/plpar_wrappers.h>
#define RIC_FLUSH_TLB 0
#define RIC_FLUSH_PWC 1
@@ -694,7 +695,14 @@
goto local;
}
- if (cputlb_use_tlbie()) {
+ if (!mmu_has_feature(MMU_FTR_GTSE)) {
+ unsigned long tgt = H_RPTI_TARGET_CMMU;
+
+ if (atomic_read(&mm->context.copros) > 0)
+ tgt |= H_RPTI_TARGET_NMMU;
+ pseries_rpt_invalidate(pid, tgt, H_RPTI_TYPE_TLB,
+ H_RPTI_PAGE_ALL, 0, -1UL);
+ } else if (cputlb_use_tlbie()) {
if (mm_needs_flush_escalation(mm))
_tlbie_pid(pid, RIC_FLUSH_ALL);
else
@@ -727,7 +735,16 @@
goto local;
}
}
- if (cputlb_use_tlbie())
+ if (!mmu_has_feature(MMU_FTR_GTSE)) {
+ unsigned long tgt = H_RPTI_TARGET_CMMU;
+ unsigned long type = H_RPTI_TYPE_TLB | H_RPTI_TYPE_PWC |
+ H_RPTI_TYPE_PRT;
+
+ if (atomic_read(&mm->context.copros) > 0)
+ tgt |= H_RPTI_TARGET_NMMU;
+ pseries_rpt_invalidate(pid, tgt, type,
+ H_RPTI_PAGE_ALL, 0, -1UL);
+ } else if (cputlb_use_tlbie())
_tlbie_pid(pid, RIC_FLUSH_ALL);
else
_tlbiel_pid_multicast(mm, pid, RIC_FLUSH_ALL);
@@ -760,7 +777,19 @@
exit_flush_lazy_tlbs(mm);
goto local;
}
- if (cputlb_use_tlbie())
+ if (!mmu_has_feature(MMU_FTR_GTSE)) {
+ unsigned long tgt, pg_sizes, size;
+
+ tgt = H_RPTI_TARGET_CMMU;
+ pg_sizes = psize_to_rpti_pgsize(psize);
+ size = 1UL << mmu_psize_to_shift(psize);
+
+ if (atomic_read(&mm->context.copros) > 0)
+ tgt |= H_RPTI_TARGET_NMMU;
+ pseries_rpt_invalidate(pid, tgt, H_RPTI_TYPE_TLB,
+ pg_sizes, vmaddr,
+ vmaddr + size);
+ } else if (cputlb_use_tlbie())
_tlbie_va(vmaddr, pid, psize, RIC_FLUSH_TLB);
else
_tlbiel_va_multicast(mm, vmaddr, pid, psize, RIC_FLUSH_TLB);
@@ -810,7 +839,14 @@
*/
void radix__flush_tlb_kernel_range(unsigned long start, unsigned long end)
{
- if (cputlb_use_tlbie())
+ if (!mmu_has_feature(MMU_FTR_GTSE)) {
+ unsigned long tgt = H_RPTI_TARGET_CMMU | H_RPTI_TARGET_NMMU;
+ unsigned long type = H_RPTI_TYPE_TLB | H_RPTI_TYPE_PWC |
+ H_RPTI_TYPE_PRT;
+
+ pseries_rpt_invalidate(0, tgt, type, H_RPTI_PAGE_ALL,
+ start, end);
+ } else if (cputlb_use_tlbie())
_tlbie_pid(0, RIC_FLUSH_ALL);
else
_tlbiel_kernel_broadcast();
@@ -864,7 +900,17 @@
nr_pages > tlb_local_single_page_flush_ceiling);
}
- if (full) {
+ if (!mmu_has_feature(MMU_FTR_GTSE) && !local) {
+ unsigned long tgt = H_RPTI_TARGET_CMMU;
+ unsigned long pg_sizes = psize_to_rpti_pgsize(mmu_virtual_psize);
+
+ if (IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE))
+ pg_sizes |= psize_to_rpti_pgsize(MMU_PAGE_2M);
+ if (atomic_read(&mm->context.copros) > 0)
+ tgt |= H_RPTI_TARGET_NMMU;
+ pseries_rpt_invalidate(pid, tgt, H_RPTI_TYPE_TLB, pg_sizes,
+ start, end);
+ } else if (full) {
if (local) {
_tlbiel_pid(pid, RIC_FLUSH_TLB);
} else {
@@ -1046,7 +1092,17 @@
nr_pages > tlb_local_single_page_flush_ceiling);
}
- if (full) {
+ if (!mmu_has_feature(MMU_FTR_GTSE) && !local) {
+ unsigned long tgt = H_RPTI_TARGET_CMMU;
+ unsigned long type = H_RPTI_TYPE_TLB;
+ unsigned long pg_sizes = psize_to_rpti_pgsize(psize);
+
+ if (also_pwc)
+ type |= H_RPTI_TYPE_PWC;
+ if (atomic_read(&mm->context.copros) > 0)
+ tgt |= H_RPTI_TARGET_NMMU;
+ pseries_rpt_invalidate(pid, tgt, type, pg_sizes, start, end);
+ } else if (full) {
if (local) {
_tlbiel_pid(pid, also_pwc ? RIC_FLUSH_ALL : RIC_FLUSH_TLB);
} else {
@@ -1111,7 +1167,19 @@
exit_flush_lazy_tlbs(mm);
goto local;
}
- if (cputlb_use_tlbie())
+ if (!mmu_has_feature(MMU_FTR_GTSE)) {
+ unsigned long tgt, type, pg_sizes;
+
+ tgt = H_RPTI_TARGET_CMMU;
+ type = H_RPTI_TYPE_TLB | H_RPTI_TYPE_PWC |
+ H_RPTI_TYPE_PRT;
+ pg_sizes = psize_to_rpti_pgsize(mmu_virtual_psize);
+
+ if (atomic_read(&mm->context.copros) > 0)
+ tgt |= H_RPTI_TARGET_NMMU;
+ pseries_rpt_invalidate(pid, tgt, type, pg_sizes,
+ addr, end);
+ } else if (cputlb_use_tlbie())
_tlbie_va_range(addr, end, pid, PAGE_SIZE, mmu_virtual_psize, true);
else
_tlbiel_va_range_multicast(mm,
diff --git a/arch/powerpc/mm/drmem.c b/arch/powerpc/mm/drmem.c
index 59327ce..b2eeea3 100644
--- a/arch/powerpc/mm/drmem.c
+++ b/arch/powerpc/mm/drmem.c
@@ -14,6 +14,8 @@
#include <asm/prom.h>
#include <asm/drmem.h>
+static int n_root_addr_cells, n_root_size_cells;
+
static struct drmem_lmb_info __drmem_info;
struct drmem_lmb_info *drmem_info = &__drmem_info;
@@ -189,12 +191,13 @@
return rc;
}
-static void __init read_drconf_v1_cell(struct drmem_lmb *lmb,
+static void read_drconf_v1_cell(struct drmem_lmb *lmb,
const __be32 **prop)
{
const __be32 *p = *prop;
- lmb->base_addr = dt_mem_next_cell(dt_root_addr_cells, &p);
+ lmb->base_addr = of_read_number(p, n_root_addr_cells);
+ p += n_root_addr_cells;
lmb->drc_index = of_read_number(p++, 1);
p++; /* skip reserved field */
@@ -205,29 +208,33 @@
*prop = p;
}
-static void __init __walk_drmem_v1_lmbs(const __be32 *prop, const __be32 *usm,
- void (*func)(struct drmem_lmb *, const __be32 **))
+static int
+__walk_drmem_v1_lmbs(const __be32 *prop, const __be32 *usm, void *data,
+ int (*func)(struct drmem_lmb *, const __be32 **, void *))
{
struct drmem_lmb lmb;
u32 i, n_lmbs;
+ int ret = 0;
n_lmbs = of_read_number(prop++, 1);
- if (n_lmbs == 0)
- return;
-
for (i = 0; i < n_lmbs; i++) {
read_drconf_v1_cell(&lmb, &prop);
- func(&lmb, &usm);
+ ret = func(&lmb, &usm, data);
+ if (ret)
+ break;
}
+
+ return ret;
}
-static void __init read_drconf_v2_cell(struct of_drconf_cell_v2 *dr_cell,
+static void read_drconf_v2_cell(struct of_drconf_cell_v2 *dr_cell,
const __be32 **prop)
{
const __be32 *p = *prop;
dr_cell->seq_lmbs = of_read_number(p++, 1);
- dr_cell->base_addr = dt_mem_next_cell(dt_root_addr_cells, &p);
+ dr_cell->base_addr = of_read_number(p, n_root_addr_cells);
+ p += n_root_addr_cells;
dr_cell->drc_index = of_read_number(p++, 1);
dr_cell->aa_index = of_read_number(p++, 1);
dr_cell->flags = of_read_number(p++, 1);
@@ -235,17 +242,16 @@
*prop = p;
}
-static void __init __walk_drmem_v2_lmbs(const __be32 *prop, const __be32 *usm,
- void (*func)(struct drmem_lmb *, const __be32 **))
+static int
+__walk_drmem_v2_lmbs(const __be32 *prop, const __be32 *usm, void *data,
+ int (*func)(struct drmem_lmb *, const __be32 **, void *))
{
struct of_drconf_cell_v2 dr_cell;
struct drmem_lmb lmb;
u32 i, j, lmb_sets;
+ int ret = 0;
lmb_sets = of_read_number(prop++, 1);
- if (lmb_sets == 0)
- return;
-
for (i = 0; i < lmb_sets; i++) {
read_drconf_v2_cell(&dr_cell, &prop);
@@ -259,21 +265,29 @@
lmb.aa_index = dr_cell.aa_index;
lmb.flags = dr_cell.flags;
- func(&lmb, &usm);
+ ret = func(&lmb, &usm, data);
+ if (ret)
+ break;
}
}
+
+ return ret;
}
#ifdef CONFIG_PPC_PSERIES
-void __init walk_drmem_lmbs_early(unsigned long node,
- void (*func)(struct drmem_lmb *, const __be32 **))
+int __init walk_drmem_lmbs_early(unsigned long node, void *data,
+ int (*func)(struct drmem_lmb *, const __be32 **, void *))
{
const __be32 *prop, *usm;
- int len;
+ int len, ret = -ENODEV;
prop = of_get_flat_dt_prop(node, "ibm,lmb-size", &len);
if (!prop || len < dt_root_size_cells * sizeof(__be32))
- return;
+ return ret;
+
+ /* Get the address & size cells */
+ n_root_addr_cells = dt_root_addr_cells;
+ n_root_size_cells = dt_root_size_cells;
drmem_info->lmb_size = dt_mem_next_cell(dt_root_size_cells, &prop);
@@ -281,20 +295,21 @@
prop = of_get_flat_dt_prop(node, "ibm,dynamic-memory", &len);
if (prop) {
- __walk_drmem_v1_lmbs(prop, usm, func);
+ ret = __walk_drmem_v1_lmbs(prop, usm, data, func);
} else {
prop = of_get_flat_dt_prop(node, "ibm,dynamic-memory-v2",
&len);
if (prop)
- __walk_drmem_v2_lmbs(prop, usm, func);
+ ret = __walk_drmem_v2_lmbs(prop, usm, data, func);
}
memblock_dump_all();
+ return ret;
}
#endif
-static int __init init_drmem_lmb_size(struct device_node *dn)
+static int init_drmem_lmb_size(struct device_node *dn)
{
const __be32 *prop;
int len;
@@ -303,12 +318,12 @@
return 0;
prop = of_get_property(dn, "ibm,lmb-size", &len);
- if (!prop || len < dt_root_size_cells * sizeof(__be32)) {
+ if (!prop || len < n_root_size_cells * sizeof(__be32)) {
pr_info("Could not determine LMB size\n");
return -1;
}
- drmem_info->lmb_size = dt_mem_next_cell(dt_root_size_cells, &prop);
+ drmem_info->lmb_size = of_read_number(prop, n_root_size_cells);
return 0;
}
@@ -329,24 +344,36 @@
return prop;
}
-void __init walk_drmem_lmbs(struct device_node *dn,
- void (*func)(struct drmem_lmb *, const __be32 **))
+int walk_drmem_lmbs(struct device_node *dn, void *data,
+ int (*func)(struct drmem_lmb *, const __be32 **, void *))
{
const __be32 *prop, *usm;
+ int ret = -ENODEV;
+
+ if (!of_root)
+ return ret;
+
+ /* Get the address & size cells */
+ of_node_get(of_root);
+ n_root_addr_cells = of_n_addr_cells(of_root);
+ n_root_size_cells = of_n_size_cells(of_root);
+ of_node_put(of_root);
if (init_drmem_lmb_size(dn))
- return;
+ return ret;
usm = of_get_usable_memory(dn);
prop = of_get_property(dn, "ibm,dynamic-memory", NULL);
if (prop) {
- __walk_drmem_v1_lmbs(prop, usm, func);
+ ret = __walk_drmem_v1_lmbs(prop, usm, data, func);
} else {
prop = of_get_property(dn, "ibm,dynamic-memory-v2", NULL);
if (prop)
- __walk_drmem_v2_lmbs(prop, usm, func);
+ ret = __walk_drmem_v2_lmbs(prop, usm, data, func);
}
+
+ return ret;
}
static void __init init_drmem_v1_lmbs(const __be32 *prop)
diff --git a/arch/powerpc/mm/fault.c b/arch/powerpc/mm/fault.c
index 641fc5f..925a723 100644
--- a/arch/powerpc/mm/fault.c
+++ b/arch/powerpc/mm/fault.c
@@ -42,39 +42,7 @@
#include <asm/kup.h>
#include <asm/inst.h>
-/*
- * Check whether the instruction inst is a store using
- * an update addressing form which will update r1.
- */
-static bool store_updates_sp(struct ppc_inst inst)
-{
- /* check for 1 in the rA field */
- if (((ppc_inst_val(inst) >> 16) & 0x1f) != 1)
- return false;
- /* check major opcode */
- switch (ppc_inst_primary_opcode(inst)) {
- case OP_STWU:
- case OP_STBU:
- case OP_STHU:
- case OP_STFSU:
- case OP_STFDU:
- return true;
- case OP_STD: /* std or stdu */
- return (ppc_inst_val(inst) & 3) == 1;
- case OP_31:
- /* check minor opcode */
- switch ((ppc_inst_val(inst) >> 1) & 0x3ff) {
- case OP_31_XOP_STDUX:
- case OP_31_XOP_STWUX:
- case OP_31_XOP_STBUX:
- case OP_31_XOP_STHUX:
- case OP_31_XOP_STFSUX:
- case OP_31_XOP_STFDUX:
- return true;
- }
- }
- return false;
-}
+
/*
* do_page_fault error handling helpers
*/
@@ -267,54 +235,6 @@
return false;
}
-static bool bad_stack_expansion(struct pt_regs *regs, unsigned long address,
- struct vm_area_struct *vma, unsigned int flags,
- bool *must_retry)
-{
- /*
- * N.B. The POWER/Open ABI allows programs to access up to
- * 288 bytes below the stack pointer.
- * The kernel signal delivery code writes up to about 1.5kB
- * below the stack pointer (r1) before decrementing it.
- * The exec code can write slightly over 640kB to the stack
- * before setting the user r1. Thus we allow the stack to
- * expand to 1MB without further checks.
- */
- if (address + 0x100000 < vma->vm_end) {
- struct ppc_inst __user *nip = (struct ppc_inst __user *)regs->nip;
- /* get user regs even if this fault is in kernel mode */
- struct pt_regs *uregs = current->thread.regs;
- if (uregs == NULL)
- return true;
-
- /*
- * A user-mode access to an address a long way below
- * the stack pointer is only valid if the instruction
- * is one which would update the stack pointer to the
- * address accessed if the instruction completed,
- * i.e. either stwu rs,n(r1) or stwux rs,r1,rb
- * (or the byte, halfword, float or double forms).
- *
- * If we don't check this then any write to the area
- * between the last mapped region and the stack will
- * expand the stack rather than segfaulting.
- */
- if (address + 2048 >= uregs->gpr[1])
- return false;
-
- if ((flags & FAULT_FLAG_WRITE) && (flags & FAULT_FLAG_USER) &&
- access_ok(nip, sizeof(*nip))) {
- struct ppc_inst inst;
-
- if (!probe_user_read_inst(&inst, nip))
- return !store_updates_sp(inst);
- *must_retry = true;
- }
- return true;
- }
- return false;
-}
-
#ifdef CONFIG_PPC_MEM_KEYS
static bool access_pkey_error(bool is_write, bool is_exec, bool is_pkey,
struct vm_area_struct *vma)
@@ -480,7 +400,6 @@
int is_user = user_mode(regs);
int is_write = page_fault_is_write(error_code);
vm_fault_t fault, major = 0;
- bool must_retry = false;
bool kprobe_fault = kprobe_page_fault(regs, 11);
if (unlikely(debugger_fault_handler(regs) || kprobe_fault))
@@ -569,30 +488,15 @@
vma = find_vma(mm, address);
if (unlikely(!vma))
return bad_area(regs, address);
- if (likely(vma->vm_start <= address))
- goto good_area;
- if (unlikely(!(vma->vm_flags & VM_GROWSDOWN)))
- return bad_area(regs, address);
- /* The stack is being expanded, check if it's valid */
- if (unlikely(bad_stack_expansion(regs, address, vma, flags,
- &must_retry))) {
- if (!must_retry)
+ if (unlikely(vma->vm_start > address)) {
+ if (unlikely(!(vma->vm_flags & VM_GROWSDOWN)))
return bad_area(regs, address);
- mmap_read_unlock(mm);
- if (fault_in_pages_readable((const char __user *)regs->nip,
- sizeof(unsigned int)))
- return bad_area_nosemaphore(regs, address);
- goto retry;
+ if (unlikely(expand_stack(vma, address)))
+ return bad_area(regs, address);
}
- /* Try to expand it */
- if (unlikely(expand_stack(vma, address)))
- return bad_area(regs, address);
-
-good_area:
-
#ifdef CONFIG_PPC_MEM_KEYS
if (unlikely(access_pkey_error(is_write, is_exec,
(error_code & DSISR_KEYFAULT), vma)))
diff --git a/arch/powerpc/mm/hugetlbpage.c b/arch/powerpc/mm/hugetlbpage.c
index e9bfbcc..2629254 100644
--- a/arch/powerpc/mm/hugetlbpage.c
+++ b/arch/powerpc/mm/hugetlbpage.c
@@ -684,3 +684,21 @@
}
}
}
+
+void __init gigantic_hugetlb_cma_reserve(void)
+{
+ unsigned long order = 0;
+
+ if (radix_enabled())
+ order = PUD_SHIFT - PAGE_SHIFT;
+ else if (!firmware_has_feature(FW_FEATURE_LPAR) && mmu_psize_defs[MMU_PAGE_16G].shift)
+ /*
+ * For pseries we do use ibm,expected#pages for reserving 16G pages.
+ */
+ order = mmu_psize_to_shift(MMU_PAGE_16G) - PAGE_SHIFT;
+
+ if (order) {
+ VM_WARN_ON(order < MAX_ORDER);
+ hugetlb_cma_reserve(order);
+ }
+}
diff --git a/arch/powerpc/mm/init_32.c b/arch/powerpc/mm/init_32.c
index 7ea19dc..02c7db4 100644
--- a/arch/powerpc/mm/init_32.c
+++ b/arch/powerpc/mm/init_32.c
@@ -170,6 +170,8 @@
btext_unmap();
#endif
+ kasan_mmu_init();
+
setup_kup();
/* Shortly after that, the entire linear mapping will be available */
diff --git a/arch/powerpc/mm/init_64.c b/arch/powerpc/mm/init_64.c
index 3fd504d..02e127f 100644
--- a/arch/powerpc/mm/init_64.c
+++ b/arch/powerpc/mm/init_64.c
@@ -406,13 +406,15 @@
}
if (!(vec5[OV5_INDX(OV5_RADIX_GTSE)] &
OV5_FEAT(OV5_RADIX_GTSE))) {
- pr_warn("WARNING: Hypervisor doesn't support RADIX with GTSE\n");
- }
+ cur_cpu_spec->mmu_features &= ~MMU_FTR_GTSE;
+ } else
+ cur_cpu_spec->mmu_features |= MMU_FTR_GTSE;
/* Do radix anyway - the hypervisor said we had to */
cur_cpu_spec->mmu_features |= MMU_FTR_TYPE_RADIX;
} else if (mmu_supported == OV5_FEAT(OV5_MMU_HASH)) {
/* Hypervisor only supports hash - disable radix */
cur_cpu_spec->mmu_features &= ~MMU_FTR_TYPE_RADIX;
+ cur_cpu_spec->mmu_features &= ~MMU_FTR_GTSE;
}
}
diff --git a/arch/powerpc/mm/kasan/kasan_init_32.c b/arch/powerpc/mm/kasan/kasan_init_32.c
index 0760e1e..fb29404 100644
--- a/arch/powerpc/mm/kasan/kasan_init_32.c
+++ b/arch/powerpc/mm/kasan/kasan_init_32.c
@@ -115,24 +115,17 @@
unsigned long k_end = (unsigned long)kasan_mem_to_shadow((void *)VMALLOC_END);
kasan_update_early_region(k_start, k_end, __pte(0));
+
+#ifdef MODULES_VADDR
+ k_start = (unsigned long)kasan_mem_to_shadow((void *)MODULES_VADDR);
+ k_end = (unsigned long)kasan_mem_to_shadow((void *)MODULES_END);
+ kasan_update_early_region(k_start, k_end, __pte(0));
+#endif
}
-static void __init kasan_mmu_init(void)
+void __init kasan_mmu_init(void)
{
int ret;
- struct memblock_region *reg;
-
- for_each_memblock(memory, reg) {
- phys_addr_t base = reg->base;
- phys_addr_t top = min(base + reg->size, total_lowmem);
-
- if (base >= top)
- continue;
-
- ret = kasan_init_region(__va(base), top - base);
- if (ret)
- panic("kasan: kasan_init_region() failed");
- }
if (early_mmu_has_feature(MMU_FTR_HPTE_TABLE) ||
IS_ENABLED(CONFIG_KASAN_VMALLOC)) {
@@ -141,12 +134,24 @@
if (ret)
panic("kasan: kasan_init_shadow_page_tables() failed");
}
-
}
void __init kasan_init(void)
{
- kasan_mmu_init();
+ struct memblock_region *reg;
+
+ for_each_memblock(memory, reg) {
+ phys_addr_t base = reg->base;
+ phys_addr_t top = min(base + reg->size, total_lowmem);
+ int ret;
+
+ if (base >= top)
+ continue;
+
+ ret = kasan_init_region(__va(base), top - base);
+ if (ret)
+ panic("kasan: kasan_init_region() failed");
+ }
kasan_remap_early_shadow_ro();
diff --git a/arch/powerpc/mm/mem.c b/arch/powerpc/mm/mem.c
index ec68c9e..42e2587 100644
--- a/arch/powerpc/mm/mem.c
+++ b/arch/powerpc/mm/mem.c
@@ -126,8 +126,6 @@
unsigned long nr_pages = size >> PAGE_SHIFT;
int rc;
- resize_hpt_for_hotplug(memblock_phys_mem_size());
-
start = (unsigned long)__va(start);
rc = create_section_mapping(start, start + size, nid,
params->pgprot);
@@ -160,9 +158,6 @@
* hit that section of memory
*/
vm_unmap_aliases();
-
- if (resize_hpt_for_hotplug(memblock_phys_mem_size()) == -ENOSPC)
- pr_warn("Hash collision while resizing HPT\n");
}
#endif
diff --git a/arch/powerpc/mm/nohash/tlb_low_64e.S b/arch/powerpc/mm/nohash/tlb_low_64e.S
index d5e2704..bf24451 100644
--- a/arch/powerpc/mm/nohash/tlb_low_64e.S
+++ b/arch/powerpc/mm/nohash/tlb_low_64e.S
@@ -71,7 +71,6 @@
END_BTB_FLUSH_SECTION
std r7,EX_TLB_R7(r12)
#endif
- TLB_MISS_PROLOG_STATS
.endm
.macro tlb_epilog_bolted
@@ -85,7 +84,6 @@
mtcr r14
ld r14,EX_TLB_R14(r12)
ld r15,EX_TLB_R15(r12)
- TLB_MISS_RESTORE_STATS
ld r16,EX_TLB_R16(r12)
mfspr r12,SPRN_SPRG_GEN_SCRATCH
.endm
@@ -128,7 +126,6 @@
ori r10,r10,_PAGE_PRESENT
oris r11,r10,_PAGE_ACCESSED@h
- TLB_MISS_STATS_SAVE_INFO_BOLTED
bne tlb_miss_kernel_bolted
tlb_miss_common_bolted:
@@ -209,7 +206,6 @@
tlbwe
tlb_miss_done_bolted:
- TLB_MISS_STATS_X(MMSTAT_TLB_MISS_NORM_OK)
tlb_epilog_bolted
rfi
@@ -229,11 +225,9 @@
andi. r10,r11,_PAGE_EXEC|_PAGE_BAP_SX
bne itlb_miss_fault_bolted
dtlb_miss_fault_bolted:
- TLB_MISS_STATS_D(MMSTAT_TLB_MISS_NORM_FAULT)
tlb_epilog_bolted
b exc_data_storage_book3e
itlb_miss_fault_bolted:
- TLB_MISS_STATS_I(MMSTAT_TLB_MISS_NORM_FAULT)
tlb_epilog_bolted
b exc_instruction_storage_book3e
@@ -243,7 +237,6 @@
rldicl. r10,r16,64-PGTABLE_EADDR_SIZE,PGTABLE_EADDR_SIZE+4
srdi r15,r16,60 /* get region */
- TLB_MISS_STATS_SAVE_INFO_BOLTED
bne- itlb_miss_fault_bolted
li r11,_PAGE_PRESENT|_PAGE_EXEC /* Base perm */
@@ -276,7 +269,6 @@
srdi. r15,r16,60 /* get region */
ori r16,r16,1
- TLB_MISS_STATS_SAVE_INFO_BOLTED
bne tlb_miss_kernel_e6500 /* user/kernel test */
b tlb_miss_common_e6500
@@ -288,7 +280,6 @@
srdi. r15,r16,60 /* get region */
rldicr r16,r16,0,62
- TLB_MISS_STATS_SAVE_INFO_BOLTED
bne tlb_miss_kernel_e6500 /* user vs kernel check */
/*
@@ -460,7 +451,6 @@
.endm
tlb_unlock_e6500
- TLB_MISS_STATS_X(MMSTAT_TLB_MISS_NORM_OK)
tlb_epilog_bolted
rfi
@@ -519,11 +509,9 @@
andi. r16,r16,1
bne itlb_miss_fault_e6500
dtlb_miss_fault_e6500:
- TLB_MISS_STATS_D(MMSTAT_TLB_MISS_NORM_FAULT)
tlb_epilog_bolted
b exc_data_storage_book3e
itlb_miss_fault_e6500:
- TLB_MISS_STATS_I(MMSTAT_TLB_MISS_NORM_FAULT)
tlb_epilog_bolted
b exc_instruction_storage_book3e
#endif /* CONFIG_PPC_FSL_BOOK3E */
@@ -548,7 +536,6 @@
mfspr r16,SPRN_DEAR /* get faulting address */
srdi r15,r16,60 /* get region */
cmpldi cr0,r15,0xc /* linear mapping ? */
- TLB_MISS_STATS_SAVE_INFO
beq tlb_load_linear /* yes -> go to linear map load */
/* The page tables are mapped virtually linear. At this point, though,
@@ -600,7 +587,6 @@
/* We got a crappy address, just fault with whatever DEAR and ESR
* are here
*/
- TLB_MISS_STATS_D(MMSTAT_TLB_MISS_NORM_FAULT)
TLB_MISS_EPILOG_ERROR
b exc_data_storage_book3e
@@ -624,7 +610,6 @@
*/
srdi r15,r16,60 /* get region */
cmpldi cr0,r15,0xc /* linear mapping ? */
- TLB_MISS_STATS_SAVE_INFO
beq tlb_load_linear /* yes -> go to linear map load */
/* We do the user/kernel test for the PID here along with the RW test
@@ -646,7 +631,6 @@
beq+ normal_tlb_miss
/* We got a crappy address, just fault */
- TLB_MISS_STATS_I(MMSTAT_TLB_MISS_NORM_FAULT)
TLB_MISS_EPILOG_ERROR
b exc_instruction_storage_book3e
@@ -745,7 +729,6 @@
* level 0 and just going back to userland. They are only needed
* if you are going to take an access fault
*/
- TLB_MISS_STATS_X(MMSTAT_TLB_MISS_NORM_OK)
TLB_MISS_EPILOG_SUCCESS
rfi
@@ -757,11 +740,9 @@
ld r15,EX_TLB_ESR(r12)
mtspr SPRN_DEAR,r14
mtspr SPRN_ESR,r15
- TLB_MISS_STATS_D(MMSTAT_TLB_MISS_NORM_FAULT)
TLB_MISS_EPILOG_ERROR
b exc_data_storage_book3e
-1: TLB_MISS_STATS_I(MMSTAT_TLB_MISS_NORM_FAULT)
- TLB_MISS_EPILOG_ERROR
+1: TLB_MISS_EPILOG_ERROR
b exc_instruction_storage_book3e
@@ -899,7 +880,6 @@
1:
END_MMU_FTR_SECTION_IFSET(MMU_FTR_USE_TLBRSRV)
/* Return to caller, normal case */
- TLB_MISS_STATS_X(MMSTAT_TLB_MISS_PT_OK);
TLB_MISS_EPILOG_SUCCESS
rfi
@@ -935,18 +915,15 @@
beq 1f
mtspr SPRN_DEAR,r15
mtspr SPRN_ESR,r16
- TLB_MISS_STATS_D(MMSTAT_TLB_MISS_PT_FAULT);
TLB_MISS_EPILOG_ERROR
b exc_data_storage_book3e
-1: TLB_MISS_STATS_I(MMSTAT_TLB_MISS_PT_FAULT);
- TLB_MISS_EPILOG_ERROR
+1: TLB_MISS_EPILOG_ERROR
b exc_instruction_storage_book3e
virt_page_table_tlb_miss_whacko_fault:
/* The linear fault will restart everything so ESR and DEAR will
* not have been clobbered, let's just fault with what we have
*/
- TLB_MISS_STATS_X(MMSTAT_TLB_MISS_PT_FAULT);
TLB_MISS_EPILOG_ERROR
b exc_data_storage_book3e
@@ -971,7 +948,6 @@
mfspr r16,SPRN_DEAR /* get faulting address */
srdi r11,r16,60 /* get region */
cmpldi cr0,r11,0xc /* linear mapping ? */
- TLB_MISS_STATS_SAVE_INFO
beq tlb_load_linear /* yes -> go to linear map load */
/* We do the user/kernel test for the PID here along with the RW test
@@ -991,7 +967,6 @@
/* We got a crappy address, just fault with whatever DEAR and ESR
* are here
*/
- TLB_MISS_STATS_D(MMSTAT_TLB_MISS_NORM_FAULT)
TLB_MISS_EPILOG_ERROR
b exc_data_storage_book3e
@@ -1015,7 +990,6 @@
*/
srdi r11,r16,60 /* get region */
cmpldi cr0,r11,0xc /* linear mapping ? */
- TLB_MISS_STATS_SAVE_INFO
beq tlb_load_linear /* yes -> go to linear map load */
/* We do the user/kernel test for the PID here along with the RW test
@@ -1033,7 +1007,6 @@
beq+ htw_tlb_miss
/* We got a crappy address, just fault */
- TLB_MISS_STATS_I(MMSTAT_TLB_MISS_NORM_FAULT)
TLB_MISS_EPILOG_ERROR
b exc_instruction_storage_book3e
@@ -1130,7 +1103,6 @@
* level 0 and just going back to userland. They are only needed
* if you are going to take an access fault
*/
- TLB_MISS_STATS_X(MMSTAT_TLB_MISS_PT_OK)
TLB_MISS_EPILOG_SUCCESS
rfi
@@ -1142,11 +1114,9 @@
beq 1f
mtspr SPRN_DEAR,r16
mtspr SPRN_ESR,r14
- TLB_MISS_STATS_D(MMSTAT_TLB_MISS_PT_FAULT)
TLB_MISS_EPILOG_ERROR
b exc_data_storage_book3e
-1: TLB_MISS_STATS_I(MMSTAT_TLB_MISS_PT_FAULT)
- TLB_MISS_EPILOG_ERROR
+1: TLB_MISS_EPILOG_ERROR
b exc_instruction_storage_book3e
/*
@@ -1221,7 +1191,6 @@
* We do that because we can't resume a fault within a TLB
* miss handler, due to MAS and TLB reservation being clobbered.
*/
- TLB_MISS_STATS_X(MMSTAT_TLB_MISS_LINEAR)
TLB_MISS_EPILOG_ERROR
rfi
@@ -1233,13 +1202,3 @@
b exc_data_storage_book3e
1: TLB_MISS_EPILOG_ERROR_SPECIAL
b exc_instruction_storage_book3e
-
-
-#ifdef CONFIG_BOOK3E_MMU_TLB_STATS
-.tlb_stat_inc:
-1: ldarx r8,0,r9
- addi r8,r8,1
- stdcx. r8,0,r9
- bne- 1b
- blr
-#endif
diff --git a/arch/powerpc/mm/numa.c b/arch/powerpc/mm/numa.c
index 03a81d6..1f61fa2 100644
--- a/arch/powerpc/mm/numa.c
+++ b/arch/powerpc/mm/numa.c
@@ -221,7 +221,8 @@
}
}
-/* Returns nid in the range [0..MAX_NUMNODES-1], or -1 if no useful numa
+/*
+ * Returns nid in the range [0..nr_node_ids], or -1 if no useful NUMA
* info is found.
*/
static int associativity_to_nid(const __be32 *associativity)
@@ -235,7 +236,7 @@
nid = of_read_number(&associativity[min_common_depth], 1);
/* POWER4 LPAR uses 0xffff as invalid node */
- if (nid == 0xffff || nid >= MAX_NUMNODES)
+ if (nid == 0xffff || nid >= nr_node_ids)
nid = NUMA_NO_NODE;
if (nid > 0 &&
@@ -448,7 +449,7 @@
index = lmb->aa_index * aa.array_sz + min_common_depth - 1;
nid = of_read_number(&aa.arrays[index], 1);
- if (nid == 0xffff || nid >= MAX_NUMNODES)
+ if (nid == 0xffff || nid >= nr_node_ids)
nid = default_nid;
if (nid > 0) {
@@ -644,8 +645,9 @@
* Extract NUMA information from the ibm,dynamic-reconfiguration-memory
* node. This assumes n_mem_{addr,size}_cells have been set.
*/
-static void __init numa_setup_drmem_lmb(struct drmem_lmb *lmb,
- const __be32 **usm)
+static int __init numa_setup_drmem_lmb(struct drmem_lmb *lmb,
+ const __be32 **usm,
+ void *data)
{
unsigned int ranges, is_kexec_kdump = 0;
unsigned long base, size, sz;
@@ -657,7 +659,7 @@
*/
if ((lmb->flags & DRCONF_MEM_RESERVED)
|| !(lmb->flags & DRCONF_MEM_ASSIGNED))
- return;
+ return 0;
if (*usm)
is_kexec_kdump = 1;
@@ -669,7 +671,7 @@
if (is_kexec_kdump) {
ranges = read_usm_ranges(usm);
if (!ranges) /* there are no (base, size) duple */
- return;
+ return 0;
}
do {
@@ -686,6 +688,8 @@
if (sz)
memblock_set_node(base, sz, &memblock.memory, nid);
} while (--ranges);
+
+ return 0;
}
static int __init parse_numa_properties(void)
@@ -787,7 +791,7 @@
*/
memory = of_find_node_by_path("/ibm,dynamic-reconfiguration-memory");
if (memory) {
- walk_drmem_lmbs(memory, numa_setup_drmem_lmb);
+ walk_drmem_lmbs(memory, NULL, numa_setup_drmem_lmb);
of_node_put(memory);
}
@@ -983,28 +987,6 @@
}
early_param("numa", early_numa);
-/*
- * The platform can inform us through one of several mechanisms
- * (post-migration device tree updates, PRRN or VPHN) that the NUMA
- * assignment of a resource has changed. This controls whether we act
- * on that. Disabled by default.
- */
-static bool topology_updates_enabled;
-
-static int __init early_topology_updates(char *p)
-{
- if (!p)
- return 0;
-
- if (!strcmp(p, "on")) {
- pr_warn("Caution: enabling topology updates\n");
- topology_updates_enabled = true;
- }
-
- return 0;
-}
-early_param("topology_updates", early_topology_updates);
-
#ifdef CONFIG_MEMORY_HOTPLUG
/*
* Find the node associated with a hot added memory section for
@@ -1143,98 +1125,9 @@
/* Virtual Processor Home Node (VPHN) support */
#ifdef CONFIG_PPC_SPLPAR
-struct topology_update_data {
- struct topology_update_data *next;
- unsigned int cpu;
- int old_nid;
- int new_nid;
-};
-
-#define TOPOLOGY_DEF_TIMER_SECS 60
-
-static u8 vphn_cpu_change_counts[NR_CPUS][MAX_DISTANCE_REF_POINTS];
-static cpumask_t cpu_associativity_changes_mask;
-static int vphn_enabled;
-static int prrn_enabled;
-static void reset_topology_timer(void);
-static int topology_timer_secs = 1;
static int topology_inited;
/*
- * Change polling interval for associativity changes.
- */
-int timed_topology_update(int nsecs)
-{
- if (vphn_enabled) {
- if (nsecs > 0)
- topology_timer_secs = nsecs;
- else
- topology_timer_secs = TOPOLOGY_DEF_TIMER_SECS;
-
- reset_topology_timer();
- }
-
- return 0;
-}
-
-/*
- * Store the current values of the associativity change counters in the
- * hypervisor.
- */
-static void setup_cpu_associativity_change_counters(void)
-{
- int cpu;
-
- /* The VPHN feature supports a maximum of 8 reference points */
- BUILD_BUG_ON(MAX_DISTANCE_REF_POINTS > 8);
-
- for_each_possible_cpu(cpu) {
- int i;
- u8 *counts = vphn_cpu_change_counts[cpu];
- volatile u8 *hypervisor_counts = lppaca_of(cpu).vphn_assoc_counts;
-
- for (i = 0; i < distance_ref_points_depth; i++)
- counts[i] = hypervisor_counts[i];
- }
-}
-
-/*
- * The hypervisor maintains a set of 8 associativity change counters in
- * the VPA of each cpu that correspond to the associativity levels in the
- * ibm,associativity-reference-points property. When an associativity
- * level changes, the corresponding counter is incremented.
- *
- * Set a bit in cpu_associativity_changes_mask for each cpu whose home
- * node associativity levels have changed.
- *
- * Returns the number of cpus with unhandled associativity changes.
- */
-static int update_cpu_associativity_changes_mask(void)
-{
- int cpu;
- cpumask_t *changes = &cpu_associativity_changes_mask;
-
- for_each_possible_cpu(cpu) {
- int i, changed = 0;
- u8 *counts = vphn_cpu_change_counts[cpu];
- volatile u8 *hypervisor_counts = lppaca_of(cpu).vphn_assoc_counts;
-
- for (i = 0; i < distance_ref_points_depth; i++) {
- if (hypervisor_counts[i] != counts[i]) {
- counts[i] = hypervisor_counts[i];
- changed = 1;
- }
- }
- if (changed) {
- cpumask_or(changes, changes, cpu_sibling_mask(cpu));
- cpu = cpu_last_thread_sibling(cpu);
- }
- }
-
- return cpumask_weight(changes);
-}
-
-/*
* Retrieve the new associativity information for a virtual processor's
* home node.
*/
@@ -1249,7 +1142,6 @@
switch (rc) {
case H_SUCCESS:
dbg("VPHN hcall succeeded. Reset polling...\n");
- timed_topology_update(0);
goto out;
case H_FUNCTION:
@@ -1268,8 +1160,6 @@
, rc);
break;
}
-
- stop_topology_update();
out:
return rc;
}
@@ -1313,380 +1203,8 @@
return new_nid;
}
-/*
- * Update the CPU maps and sysfs entries for a single CPU when its NUMA
- * characteristics change. This function doesn't perform any locking and is
- * only safe to call from stop_machine().
- */
-static int update_cpu_topology(void *data)
-{
- struct topology_update_data *update;
- unsigned long cpu;
-
- if (!data)
- return -EINVAL;
-
- cpu = smp_processor_id();
-
- for (update = data; update; update = update->next) {
- int new_nid = update->new_nid;
- if (cpu != update->cpu)
- continue;
-
- unmap_cpu_from_node(cpu);
- map_cpu_to_node(cpu, new_nid);
- set_cpu_numa_node(cpu, new_nid);
- set_cpu_numa_mem(cpu, local_memory_node(new_nid));
- vdso_getcpu_init();
- }
-
- return 0;
-}
-
-static int update_lookup_table(void *data)
-{
- struct topology_update_data *update;
-
- if (!data)
- return -EINVAL;
-
- /*
- * Upon topology update, the numa-cpu lookup table needs to be updated
- * for all threads in the core, including offline CPUs, to ensure that
- * future hotplug operations respect the cpu-to-node associativity
- * properly.
- */
- for (update = data; update; update = update->next) {
- int nid, base, j;
-
- nid = update->new_nid;
- base = cpu_first_thread_sibling(update->cpu);
-
- for (j = 0; j < threads_per_core; j++) {
- update_numa_cpu_lookup_table(base + j, nid);
- }
- }
-
- return 0;
-}
-
-/*
- * Update the node maps and sysfs entries for each cpu whose home node
- * has changed. Returns 1 when the topology has changed, and 0 otherwise.
- *
- * cpus_locked says whether we already hold cpu_hotplug_lock.
- */
-int numa_update_cpu_topology(bool cpus_locked)
-{
- unsigned int cpu, sibling, changed = 0;
- struct topology_update_data *updates, *ud;
- cpumask_t updated_cpus;
- struct device *dev;
- int weight, new_nid, i = 0;
-
- if (!prrn_enabled && !vphn_enabled && topology_inited)
- return 0;
-
- weight = cpumask_weight(&cpu_associativity_changes_mask);
- if (!weight)
- return 0;
-
- updates = kcalloc(weight, sizeof(*updates), GFP_KERNEL);
- if (!updates)
- return 0;
-
- cpumask_clear(&updated_cpus);
-
- for_each_cpu(cpu, &cpu_associativity_changes_mask) {
- /*
- * If siblings aren't flagged for changes, updates list
- * will be too short. Skip on this update and set for next
- * update.
- */
- if (!cpumask_subset(cpu_sibling_mask(cpu),
- &cpu_associativity_changes_mask)) {
- pr_info("Sibling bits not set for associativity "
- "change, cpu%d\n", cpu);
- cpumask_or(&cpu_associativity_changes_mask,
- &cpu_associativity_changes_mask,
- cpu_sibling_mask(cpu));
- cpu = cpu_last_thread_sibling(cpu);
- continue;
- }
-
- new_nid = find_and_online_cpu_nid(cpu);
-
- if (new_nid == numa_cpu_lookup_table[cpu]) {
- cpumask_andnot(&cpu_associativity_changes_mask,
- &cpu_associativity_changes_mask,
- cpu_sibling_mask(cpu));
- dbg("Assoc chg gives same node %d for cpu%d\n",
- new_nid, cpu);
- cpu = cpu_last_thread_sibling(cpu);
- continue;
- }
-
- for_each_cpu(sibling, cpu_sibling_mask(cpu)) {
- ud = &updates[i++];
- ud->next = &updates[i];
- ud->cpu = sibling;
- ud->new_nid = new_nid;
- ud->old_nid = numa_cpu_lookup_table[sibling];
- cpumask_set_cpu(sibling, &updated_cpus);
- }
- cpu = cpu_last_thread_sibling(cpu);
- }
-
- /*
- * Prevent processing of 'updates' from overflowing array
- * where last entry filled in a 'next' pointer.
- */
- if (i)
- updates[i-1].next = NULL;
-
- pr_debug("Topology update for the following CPUs:\n");
- if (cpumask_weight(&updated_cpus)) {
- for (ud = &updates[0]; ud; ud = ud->next) {
- pr_debug("cpu %d moving from node %d "
- "to %d\n", ud->cpu,
- ud->old_nid, ud->new_nid);
- }
- }
-
- /*
- * In cases where we have nothing to update (because the updates list
- * is too short or because the new topology is same as the old one),
- * skip invoking update_cpu_topology() via stop-machine(). This is
- * necessary (and not just a fast-path optimization) since stop-machine
- * can end up electing a random CPU to run update_cpu_topology(), and
- * thus trick us into setting up incorrect cpu-node mappings (since
- * 'updates' is kzalloc()'ed).
- *
- * And for the similar reason, we will skip all the following updating.
- */
- if (!cpumask_weight(&updated_cpus))
- goto out;
-
- if (cpus_locked)
- stop_machine_cpuslocked(update_cpu_topology, &updates[0],
- &updated_cpus);
- else
- stop_machine(update_cpu_topology, &updates[0], &updated_cpus);
-
- /*
- * Update the numa-cpu lookup table with the new mappings, even for
- * offline CPUs. It is best to perform this update from the stop-
- * machine context.
- */
- if (cpus_locked)
- stop_machine_cpuslocked(update_lookup_table, &updates[0],
- cpumask_of(raw_smp_processor_id()));
- else
- stop_machine(update_lookup_table, &updates[0],
- cpumask_of(raw_smp_processor_id()));
-
- for (ud = &updates[0]; ud; ud = ud->next) {
- unregister_cpu_under_node(ud->cpu, ud->old_nid);
- register_cpu_under_node(ud->cpu, ud->new_nid);
-
- dev = get_cpu_device(ud->cpu);
- if (dev)
- kobject_uevent(&dev->kobj, KOBJ_CHANGE);
- cpumask_clear_cpu(ud->cpu, &cpu_associativity_changes_mask);
- changed = 1;
- }
-
-out:
- kfree(updates);
- return changed;
-}
-
-int arch_update_cpu_topology(void)
-{
- return numa_update_cpu_topology(true);
-}
-
-static void topology_work_fn(struct work_struct *work)
-{
- rebuild_sched_domains();
-}
-static DECLARE_WORK(topology_work, topology_work_fn);
-
-static void topology_schedule_update(void)
-{
- schedule_work(&topology_work);
-}
-
-static void topology_timer_fn(struct timer_list *unused)
-{
- if (prrn_enabled && cpumask_weight(&cpu_associativity_changes_mask))
- topology_schedule_update();
- else if (vphn_enabled) {
- if (update_cpu_associativity_changes_mask() > 0)
- topology_schedule_update();
- reset_topology_timer();
- }
-}
-static struct timer_list topology_timer;
-
-static void reset_topology_timer(void)
-{
- if (vphn_enabled)
- mod_timer(&topology_timer, jiffies + topology_timer_secs * HZ);
-}
-
-#ifdef CONFIG_SMP
-
-static int dt_update_callback(struct notifier_block *nb,
- unsigned long action, void *data)
-{
- struct of_reconfig_data *update = data;
- int rc = NOTIFY_DONE;
-
- switch (action) {
- case OF_RECONFIG_UPDATE_PROPERTY:
- if (of_node_is_type(update->dn, "cpu") &&
- !of_prop_cmp(update->prop->name, "ibm,associativity")) {
- u32 core_id;
- of_property_read_u32(update->dn, "reg", &core_id);
- rc = dlpar_cpu_readd(core_id);
- rc = NOTIFY_OK;
- }
- break;
- }
-
- return rc;
-}
-
-static struct notifier_block dt_update_nb = {
- .notifier_call = dt_update_callback,
-};
-
-#endif
-
-/*
- * Start polling for associativity changes.
- */
-int start_topology_update(void)
-{
- int rc = 0;
-
- if (!topology_updates_enabled)
- return 0;
-
- if (firmware_has_feature(FW_FEATURE_PRRN)) {
- if (!prrn_enabled) {
- prrn_enabled = 1;
-#ifdef CONFIG_SMP
- rc = of_reconfig_notifier_register(&dt_update_nb);
-#endif
- }
- }
- if (firmware_has_feature(FW_FEATURE_VPHN) &&
- lppaca_shared_proc(get_lppaca())) {
- if (!vphn_enabled) {
- vphn_enabled = 1;
- setup_cpu_associativity_change_counters();
- timer_setup(&topology_timer, topology_timer_fn,
- TIMER_DEFERRABLE);
- reset_topology_timer();
- }
- }
-
- pr_info("Starting topology update%s%s\n",
- (prrn_enabled ? " prrn_enabled" : ""),
- (vphn_enabled ? " vphn_enabled" : ""));
-
- return rc;
-}
-
-/*
- * Disable polling for VPHN associativity changes.
- */
-int stop_topology_update(void)
-{
- int rc = 0;
-
- if (!topology_updates_enabled)
- return 0;
-
- if (prrn_enabled) {
- prrn_enabled = 0;
-#ifdef CONFIG_SMP
- rc = of_reconfig_notifier_unregister(&dt_update_nb);
-#endif
- }
- if (vphn_enabled) {
- vphn_enabled = 0;
- rc = del_timer_sync(&topology_timer);
- }
-
- pr_info("Stopping topology update\n");
-
- return rc;
-}
-
-int prrn_is_enabled(void)
-{
- return prrn_enabled;
-}
-
-static int topology_read(struct seq_file *file, void *v)
-{
- if (vphn_enabled || prrn_enabled)
- seq_puts(file, "on\n");
- else
- seq_puts(file, "off\n");
-
- return 0;
-}
-
-static int topology_open(struct inode *inode, struct file *file)
-{
- return single_open(file, topology_read, NULL);
-}
-
-static ssize_t topology_write(struct file *file, const char __user *buf,
- size_t count, loff_t *off)
-{
- char kbuf[4]; /* "on" or "off" plus null. */
- int read_len;
-
- read_len = count < 3 ? count : 3;
- if (copy_from_user(kbuf, buf, read_len))
- return -EINVAL;
-
- kbuf[read_len] = '\0';
-
- if (!strncmp(kbuf, "on", 2)) {
- topology_updates_enabled = true;
- start_topology_update();
- } else if (!strncmp(kbuf, "off", 3)) {
- stop_topology_update();
- topology_updates_enabled = false;
- } else
- return -EINVAL;
-
- return count;
-}
-
-static const struct proc_ops topology_proc_ops = {
- .proc_read = seq_read,
- .proc_write = topology_write,
- .proc_open = topology_open,
- .proc_release = single_release,
-};
-
static int topology_update_init(void)
{
- start_topology_update();
-
- if (vphn_enabled)
- topology_schedule_update();
-
- if (!proc_create("powerpc/topology_updates", 0644, NULL, &topology_proc_ops))
- return -ENOMEM;
-
topology_inited = 1;
return 0;
}
diff --git a/arch/powerpc/mm/pgtable-frag.c b/arch/powerpc/mm/pgtable-frag.c
index ee4bd6d..97ae493 100644
--- a/arch/powerpc/mm/pgtable-frag.c
+++ b/arch/powerpc/mm/pgtable-frag.c
@@ -110,6 +110,9 @@
{
struct page *page = virt_to_page(table);
+ if (PageReserved(page))
+ return free_reserved_page(page);
+
BUG_ON(atomic_read(&page->pt_frag_refcount) <= 0);
if (atomic_dec_and_test(&page->pt_frag_refcount)) {
if (!kernel)
diff --git a/arch/powerpc/mm/ptdump/hashpagetable.c b/arch/powerpc/mm/ptdump/hashpagetable.c
index ff4b05a9..ad6df9a 100644
--- a/arch/powerpc/mm/ptdump/hashpagetable.c
+++ b/arch/powerpc/mm/ptdump/hashpagetable.c
@@ -258,7 +258,7 @@
for (i = 0; i < HPTES_PER_GROUP; i += 4, hpte_group += 4) {
lpar_rc = plpar_pte_read_4(0, hpte_group, (void *)ptes);
- if (lpar_rc != H_SUCCESS)
+ if (lpar_rc)
continue;
for (j = 0; j < 4; j++) {
if (HPTE_V_COMPARE(ptes[j].v, want_v) &&
diff --git a/arch/powerpc/mm/ptdump/ptdump.c b/arch/powerpc/mm/ptdump/ptdump.c
index f7ba13c..aca354f 100644
--- a/arch/powerpc/mm/ptdump/ptdump.c
+++ b/arch/powerpc/mm/ptdump/ptdump.c
@@ -73,6 +73,10 @@
static struct addr_marker address_markers[] = {
{ 0, "Start of kernel VM" },
+#ifdef MODULES_VADDR
+ { 0, "modules start" },
+ { 0, "modules end" },
+#endif
{ 0, "vmalloc() Area" },
{ 0, "vmalloc() End" },
#ifdef CONFIG_PPC64
@@ -194,6 +198,24 @@
st->wx_pages += (addr - st->start_address) / PAGE_SIZE;
}
+static void note_page_update_state(struct pg_state *st, unsigned long addr,
+ unsigned int level, u64 val, unsigned long page_size)
+{
+ u64 flag = val & pg_level[level].mask;
+ u64 pa = val & PTE_RPN_MASK;
+
+ st->level = level;
+ st->current_flags = flag;
+ st->start_address = addr;
+ st->start_pa = pa;
+ st->page_size = page_size;
+
+ while (addr >= st->marker[1].start_address) {
+ st->marker++;
+ pt_dump_seq_printf(st->seq, "---[ %s ]---\n", st->marker->name);
+ }
+}
+
static void note_page(struct pg_state *st, unsigned long addr,
unsigned int level, u64 val, unsigned long page_size)
{
@@ -202,13 +224,8 @@
/* At first no level is set */
if (!st->level) {
- st->level = level;
- st->current_flags = flag;
- st->start_address = addr;
- st->start_pa = pa;
- st->last_pa = pa;
- st->page_size = page_size;
pt_dump_seq_printf(st->seq, "---[ %s ]---\n", st->marker->name);
+ note_page_update_state(st, addr, level, val, page_size);
/*
* Dump the section of virtual memory when:
* - the PTE flags from one entry to the next differs.
@@ -240,19 +257,9 @@
* Address indicates we have passed the end of the
* current section of virtual memory
*/
- while (addr >= st->marker[1].start_address) {
- st->marker++;
- pt_dump_seq_printf(st->seq, "---[ %s ]---\n", st->marker->name);
- }
- st->start_address = addr;
- st->start_pa = pa;
- st->last_pa = pa;
- st->page_size = page_size;
- st->current_flags = flag;
- st->level = level;
- } else {
- st->last_pa = pa;
+ note_page_update_state(st, addr, level, val, page_size);
}
+ st->last_pa = pa;
}
static void walk_pte(struct pg_state *st, pmd_t *pmd, unsigned long start)
@@ -347,7 +354,15 @@
{
int i = 0;
+#ifdef CONFIG_PPC64
address_markers[i++].start_address = PAGE_OFFSET;
+#else
+ address_markers[i++].start_address = TASK_SIZE;
+#endif
+#ifdef MODULES_VADDR
+ address_markers[i++].start_address = MODULES_VADDR;
+ address_markers[i++].start_address = MODULES_END;
+#endif
address_markers[i++].start_address = VMALLOC_START;
address_markers[i++].start_address = VMALLOC_END;
#ifdef CONFIG_PPC64
@@ -384,7 +399,7 @@
struct pg_state st = {
.seq = m,
.marker = address_markers,
- .start_address = PAGE_OFFSET,
+ .start_address = IS_ENABLED(CONFIG_PPC64) ? PAGE_OFFSET : TASK_SIZE,
};
#ifdef CONFIG_PPC64
@@ -428,7 +443,7 @@
.seq = NULL,
.marker = address_markers,
.check_wx = true,
- .start_address = PAGE_OFFSET,
+ .start_address = IS_ENABLED(CONFIG_PPC64) ? PAGE_OFFSET : TASK_SIZE,
};
#ifdef CONFIG_PPC64
diff --git a/arch/powerpc/net/bpf_jit.h b/arch/powerpc/net/bpf_jit.h
index 55d4377..d0a67a1 100644
--- a/arch/powerpc/net/bpf_jit.h
+++ b/arch/powerpc/net/bpf_jit.h
@@ -11,6 +11,7 @@
#ifndef __ASSEMBLY__
#include <asm/types.h>
+#include <asm/ppc-opcode.h>
#ifdef PPC64_ELF_ABI_v1
#define FUNCTION_DESCR_SIZE 24
@@ -18,167 +19,10 @@
#define FUNCTION_DESCR_SIZE 0
#endif
-/*
- * 16-bit immediate helper macros: HA() is for use with sign-extending instrs
- * (e.g. LD, ADDI). If the bottom 16 bits is "-ve", add another bit into the
- * top half to negate the effect (i.e. 0xffff + 1 = 0x(1)0000).
- */
-#define IMM_H(i) ((uintptr_t)(i)>>16)
-#define IMM_HA(i) (((uintptr_t)(i)>>16) + \
- (((uintptr_t)(i) & 0x8000) >> 15))
-#define IMM_L(i) ((uintptr_t)(i) & 0xffff)
-
#define PLANT_INSTR(d, idx, instr) \
do { if (d) { (d)[idx] = instr; } idx++; } while (0)
#define EMIT(instr) PLANT_INSTR(image, ctx->idx, instr)
-#define PPC_NOP() EMIT(PPC_INST_NOP)
-#define PPC_BLR() EMIT(PPC_INST_BLR)
-#define PPC_BLRL() EMIT(PPC_INST_BLRL)
-#define PPC_MTLR(r) EMIT(PPC_INST_MTLR | ___PPC_RT(r))
-#define PPC_BCTR() EMIT(PPC_INST_BCTR)
-#define PPC_MTCTR(r) EMIT(PPC_INST_MTCTR | ___PPC_RT(r))
-#define PPC_ADDI(d, a, i) EMIT(PPC_INST_ADDI | ___PPC_RT(d) | \
- ___PPC_RA(a) | IMM_L(i))
-#define PPC_MR(d, a) PPC_OR(d, a, a)
-#define PPC_LI(r, i) PPC_ADDI(r, 0, i)
-#define PPC_ADDIS(d, a, i) EMIT(PPC_INST_ADDIS | \
- ___PPC_RT(d) | ___PPC_RA(a) | IMM_L(i))
-#define PPC_LIS(r, i) PPC_ADDIS(r, 0, i)
-#define PPC_STD(r, base, i) EMIT(PPC_INST_STD | ___PPC_RS(r) | \
- ___PPC_RA(base) | ((i) & 0xfffc))
-#define PPC_STDX(r, base, b) EMIT(PPC_INST_STDX | ___PPC_RS(r) | \
- ___PPC_RA(base) | ___PPC_RB(b))
-#define PPC_STDU(r, base, i) EMIT(PPC_INST_STDU | ___PPC_RS(r) | \
- ___PPC_RA(base) | ((i) & 0xfffc))
-#define PPC_STW(r, base, i) EMIT(PPC_INST_STW | ___PPC_RS(r) | \
- ___PPC_RA(base) | IMM_L(i))
-#define PPC_STWU(r, base, i) EMIT(PPC_INST_STWU | ___PPC_RS(r) | \
- ___PPC_RA(base) | IMM_L(i))
-#define PPC_STH(r, base, i) EMIT(PPC_INST_STH | ___PPC_RS(r) | \
- ___PPC_RA(base) | IMM_L(i))
-#define PPC_STB(r, base, i) EMIT(PPC_INST_STB | ___PPC_RS(r) | \
- ___PPC_RA(base) | IMM_L(i))
-
-#define PPC_LBZ(r, base, i) EMIT(PPC_INST_LBZ | ___PPC_RT(r) | \
- ___PPC_RA(base) | IMM_L(i))
-#define PPC_LD(r, base, i) EMIT(PPC_INST_LD | ___PPC_RT(r) | \
- ___PPC_RA(base) | ((i) & 0xfffc))
-#define PPC_LDX(r, base, b) EMIT(PPC_INST_LDX | ___PPC_RT(r) | \
- ___PPC_RA(base) | ___PPC_RB(b))
-#define PPC_LWZ(r, base, i) EMIT(PPC_INST_LWZ | ___PPC_RT(r) | \
- ___PPC_RA(base) | IMM_L(i))
-#define PPC_LHZ(r, base, i) EMIT(PPC_INST_LHZ | ___PPC_RT(r) | \
- ___PPC_RA(base) | IMM_L(i))
-#define PPC_LHBRX(r, base, b) EMIT(PPC_INST_LHBRX | ___PPC_RT(r) | \
- ___PPC_RA(base) | ___PPC_RB(b))
-#define PPC_LDBRX(r, base, b) EMIT(PPC_INST_LDBRX | ___PPC_RT(r) | \
- ___PPC_RA(base) | ___PPC_RB(b))
-
-#define PPC_BPF_LDARX(t, a, b, eh) EMIT(PPC_INST_LDARX | ___PPC_RT(t) | \
- ___PPC_RA(a) | ___PPC_RB(b) | \
- __PPC_EH(eh))
-#define PPC_BPF_LWARX(t, a, b, eh) EMIT(PPC_INST_LWARX | ___PPC_RT(t) | \
- ___PPC_RA(a) | ___PPC_RB(b) | \
- __PPC_EH(eh))
-#define PPC_BPF_STWCX(s, a, b) EMIT(PPC_INST_STWCX | ___PPC_RS(s) | \
- ___PPC_RA(a) | ___PPC_RB(b))
-#define PPC_BPF_STDCX(s, a, b) EMIT(PPC_INST_STDCX | ___PPC_RS(s) | \
- ___PPC_RA(a) | ___PPC_RB(b))
-#define PPC_CMPWI(a, i) EMIT(PPC_INST_CMPWI | ___PPC_RA(a) | IMM_L(i))
-#define PPC_CMPDI(a, i) EMIT(PPC_INST_CMPDI | ___PPC_RA(a) | IMM_L(i))
-#define PPC_CMPW(a, b) EMIT(PPC_INST_CMPW | ___PPC_RA(a) | \
- ___PPC_RB(b))
-#define PPC_CMPD(a, b) EMIT(PPC_INST_CMPD | ___PPC_RA(a) | \
- ___PPC_RB(b))
-#define PPC_CMPLWI(a, i) EMIT(PPC_INST_CMPLWI | ___PPC_RA(a) | IMM_L(i))
-#define PPC_CMPLDI(a, i) EMIT(PPC_INST_CMPLDI | ___PPC_RA(a) | IMM_L(i))
-#define PPC_CMPLW(a, b) EMIT(PPC_INST_CMPLW | ___PPC_RA(a) | \
- ___PPC_RB(b))
-#define PPC_CMPLD(a, b) EMIT(PPC_INST_CMPLD | ___PPC_RA(a) | \
- ___PPC_RB(b))
-
-#define PPC_SUB(d, a, b) EMIT(PPC_INST_SUB | ___PPC_RT(d) | \
- ___PPC_RB(a) | ___PPC_RA(b))
-#define PPC_ADD(d, a, b) EMIT(PPC_INST_ADD | ___PPC_RT(d) | \
- ___PPC_RA(a) | ___PPC_RB(b))
-#define PPC_MULD(d, a, b) EMIT(PPC_INST_MULLD | ___PPC_RT(d) | \
- ___PPC_RA(a) | ___PPC_RB(b))
-#define PPC_MULW(d, a, b) EMIT(PPC_INST_MULLW | ___PPC_RT(d) | \
- ___PPC_RA(a) | ___PPC_RB(b))
-#define PPC_MULHWU(d, a, b) EMIT(PPC_INST_MULHWU | ___PPC_RT(d) | \
- ___PPC_RA(a) | ___PPC_RB(b))
-#define PPC_MULI(d, a, i) EMIT(PPC_INST_MULLI | ___PPC_RT(d) | \
- ___PPC_RA(a) | IMM_L(i))
-#define PPC_DIVWU(d, a, b) EMIT(PPC_INST_DIVWU | ___PPC_RT(d) | \
- ___PPC_RA(a) | ___PPC_RB(b))
-#define PPC_DIVDU(d, a, b) EMIT(PPC_INST_DIVDU | ___PPC_RT(d) | \
- ___PPC_RA(a) | ___PPC_RB(b))
-#define PPC_AND(d, a, b) EMIT(PPC_INST_AND | ___PPC_RA(d) | \
- ___PPC_RS(a) | ___PPC_RB(b))
-#define PPC_ANDI(d, a, i) EMIT(PPC_INST_ANDI | ___PPC_RA(d) | \
- ___PPC_RS(a) | IMM_L(i))
-#define PPC_AND_DOT(d, a, b) EMIT(PPC_INST_ANDDOT | ___PPC_RA(d) | \
- ___PPC_RS(a) | ___PPC_RB(b))
-#define PPC_OR(d, a, b) EMIT(PPC_INST_OR | ___PPC_RA(d) | \
- ___PPC_RS(a) | ___PPC_RB(b))
-#define PPC_MR(d, a) PPC_OR(d, a, a)
-#define PPC_ORI(d, a, i) EMIT(PPC_INST_ORI | ___PPC_RA(d) | \
- ___PPC_RS(a) | IMM_L(i))
-#define PPC_ORIS(d, a, i) EMIT(PPC_INST_ORIS | ___PPC_RA(d) | \
- ___PPC_RS(a) | IMM_L(i))
-#define PPC_XOR(d, a, b) EMIT(PPC_INST_XOR | ___PPC_RA(d) | \
- ___PPC_RS(a) | ___PPC_RB(b))
-#define PPC_XORI(d, a, i) EMIT(PPC_INST_XORI | ___PPC_RA(d) | \
- ___PPC_RS(a) | IMM_L(i))
-#define PPC_XORIS(d, a, i) EMIT(PPC_INST_XORIS | ___PPC_RA(d) | \
- ___PPC_RS(a) | IMM_L(i))
-#define PPC_EXTSW(d, a) EMIT(PPC_INST_EXTSW | ___PPC_RA(d) | \
- ___PPC_RS(a))
-#define PPC_SLW(d, a, s) EMIT(PPC_INST_SLW | ___PPC_RA(d) | \
- ___PPC_RS(a) | ___PPC_RB(s))
-#define PPC_SLD(d, a, s) EMIT(PPC_INST_SLD | ___PPC_RA(d) | \
- ___PPC_RS(a) | ___PPC_RB(s))
-#define PPC_SRW(d, a, s) EMIT(PPC_INST_SRW | ___PPC_RA(d) | \
- ___PPC_RS(a) | ___PPC_RB(s))
-#define PPC_SRAW(d, a, s) EMIT(PPC_INST_SRAW | ___PPC_RA(d) | \
- ___PPC_RS(a) | ___PPC_RB(s))
-#define PPC_SRAWI(d, a, i) EMIT(PPC_INST_SRAWI | ___PPC_RA(d) | \
- ___PPC_RS(a) | __PPC_SH(i))
-#define PPC_SRD(d, a, s) EMIT(PPC_INST_SRD | ___PPC_RA(d) | \
- ___PPC_RS(a) | ___PPC_RB(s))
-#define PPC_SRAD(d, a, s) EMIT(PPC_INST_SRAD | ___PPC_RA(d) | \
- ___PPC_RS(a) | ___PPC_RB(s))
-#define PPC_SRADI(d, a, i) EMIT(PPC_INST_SRADI | ___PPC_RA(d) | \
- ___PPC_RS(a) | __PPC_SH64(i))
-#define PPC_RLWINM(d, a, i, mb, me) EMIT(PPC_INST_RLWINM | ___PPC_RA(d) | \
- ___PPC_RS(a) | __PPC_SH(i) | \
- __PPC_MB(mb) | __PPC_ME(me))
-#define PPC_RLWINM_DOT(d, a, i, mb, me) EMIT(PPC_INST_RLWINM_DOT | \
- ___PPC_RA(d) | ___PPC_RS(a) | \
- __PPC_SH(i) | __PPC_MB(mb) | \
- __PPC_ME(me))
-#define PPC_RLWIMI(d, a, i, mb, me) EMIT(PPC_INST_RLWIMI | ___PPC_RA(d) | \
- ___PPC_RS(a) | __PPC_SH(i) | \
- __PPC_MB(mb) | __PPC_ME(me))
-#define PPC_RLDICL(d, a, i, mb) EMIT(PPC_INST_RLDICL | ___PPC_RA(d) | \
- ___PPC_RS(a) | __PPC_SH64(i) | \
- __PPC_MB64(mb))
-#define PPC_RLDICR(d, a, i, me) EMIT(PPC_INST_RLDICR | ___PPC_RA(d) | \
- ___PPC_RS(a) | __PPC_SH64(i) | \
- __PPC_ME64(me))
-
-/* slwi = rlwinm Rx, Ry, n, 0, 31-n */
-#define PPC_SLWI(d, a, i) PPC_RLWINM(d, a, i, 0, 31-(i))
-/* srwi = rlwinm Rx, Ry, 32-n, n, 31 */
-#define PPC_SRWI(d, a, i) PPC_RLWINM(d, a, 32-(i), i, 31)
-/* sldi = rldicr Rx, Ry, n, 63-n */
-#define PPC_SLDI(d, a, i) PPC_RLDICR(d, a, i, 63-(i))
-/* sldi = rldicl Rx, Ry, 64-n, n */
-#define PPC_SRDI(d, a, i) PPC_RLDICL(d, a, 64-(i), i)
-
-#define PPC_NEG(d, a) EMIT(PPC_INST_NEG | ___PPC_RT(d) | ___PPC_RA(a))
-
/* Long jump; (unconditional 'branch') */
#define PPC_JMP(dest) EMIT(PPC_INST_BRANCH | \
(((dest) - (ctx->idx * 4)) & 0x03fffffc))
@@ -191,11 +35,11 @@
#define PPC_LI32(d, i) do { \
if ((int)(uintptr_t)(i) >= -32768 && \
(int)(uintptr_t)(i) < 32768) \
- PPC_LI(d, i); \
+ EMIT(PPC_RAW_LI(d, i)); \
else { \
- PPC_LIS(d, IMM_H(i)); \
+ EMIT(PPC_RAW_LIS(d, IMM_H(i))); \
if (IMM_L(i)) \
- PPC_ORI(d, d, IMM_L(i)); \
+ EMIT(PPC_RAW_ORI(d, d, IMM_L(i))); \
} } while(0)
#define PPC_LI64(d, i) do { \
@@ -204,19 +48,21 @@
PPC_LI32(d, i); \
else { \
if (!((uintptr_t)(i) & 0xffff800000000000ULL)) \
- PPC_LI(d, ((uintptr_t)(i) >> 32) & 0xffff); \
+ EMIT(PPC_RAW_LI(d, ((uintptr_t)(i) >> 32) & \
+ 0xffff)); \
else { \
- PPC_LIS(d, ((uintptr_t)(i) >> 48)); \
+ EMIT(PPC_RAW_LIS(d, ((uintptr_t)(i) >> 48))); \
if ((uintptr_t)(i) & 0x0000ffff00000000ULL) \
- PPC_ORI(d, d, \
- ((uintptr_t)(i) >> 32) & 0xffff); \
+ EMIT(PPC_RAW_ORI(d, d, \
+ ((uintptr_t)(i) >> 32) & 0xffff)); \
} \
- PPC_SLDI(d, d, 32); \
+ EMIT(PPC_RAW_SLDI(d, d, 32)); \
if ((uintptr_t)(i) & 0x00000000ffff0000ULL) \
- PPC_ORIS(d, d, \
- ((uintptr_t)(i) >> 16) & 0xffff); \
+ EMIT(PPC_RAW_ORIS(d, d, \
+ ((uintptr_t)(i) >> 16) & 0xffff)); \
if ((uintptr_t)(i) & 0x000000000000ffffULL) \
- PPC_ORI(d, d, (uintptr_t)(i) & 0xffff); \
+ EMIT(PPC_RAW_ORI(d, d, (uintptr_t)(i) & \
+ 0xffff)); \
} } while (0)
#ifdef CONFIG_PPC64
@@ -240,7 +86,7 @@
#define PPC_BCC(cond, dest) do { \
if (is_nearbranch((dest) - (ctx->idx * 4))) { \
PPC_BCC_SHORT(cond, dest); \
- PPC_NOP(); \
+ EMIT(PPC_RAW_NOP()); \
} else { \
/* Flip the 'T or F' bit to invert comparison */ \
PPC_BCC_SHORT(cond ^ COND_CMP_TRUE, (ctx->idx+2)*4); \
diff --git a/arch/powerpc/net/bpf_jit32.h b/arch/powerpc/net/bpf_jit32.h
index 4ec2a9f..448dfd4 100644
--- a/arch/powerpc/net/bpf_jit32.h
+++ b/arch/powerpc/net/bpf_jit32.h
@@ -72,21 +72,21 @@
DECLARE_LOAD_FUNC(sk_load_byte);
DECLARE_LOAD_FUNC(sk_load_byte_msh);
-#define PPC_LBZ_OFFS(r, base, i) do { if ((i) < 32768) PPC_LBZ(r, base, i); \
- else { PPC_ADDIS(r, base, IMM_HA(i)); \
- PPC_LBZ(r, r, IMM_L(i)); } } while(0)
+#define PPC_LBZ_OFFS(r, base, i) do { if ((i) < 32768) EMIT(PPC_RAW_LBZ(r, base, i)); \
+ else { EMIT(PPC_RAW_ADDIS(r, base, IMM_HA(i))); \
+ EMIT(PPC_RAW_LBZ(r, r, IMM_L(i))); } } while(0)
-#define PPC_LD_OFFS(r, base, i) do { if ((i) < 32768) PPC_LD(r, base, i); \
- else { PPC_ADDIS(r, base, IMM_HA(i)); \
- PPC_LD(r, r, IMM_L(i)); } } while(0)
+#define PPC_LD_OFFS(r, base, i) do { if ((i) < 32768) EMIT(PPC_RAW_LD(r, base, i)); \
+ else { EMIT(PPC_RAW_ADDIS(r, base, IMM_HA(i))); \
+ EMIT(PPC_RAW_LD(r, r, IMM_L(i))); } } while(0)
-#define PPC_LWZ_OFFS(r, base, i) do { if ((i) < 32768) PPC_LWZ(r, base, i); \
- else { PPC_ADDIS(r, base, IMM_HA(i)); \
- PPC_LWZ(r, r, IMM_L(i)); } } while(0)
+#define PPC_LWZ_OFFS(r, base, i) do { if ((i) < 32768) EMIT(PPC_RAW_LWZ(r, base, i)); \
+ else { EMIT(PPC_RAW_ADDIS(r, base, IMM_HA(i))); \
+ EMIT(PPC_RAW_LWZ(r, r, IMM_L(i))); } } while(0)
-#define PPC_LHZ_OFFS(r, base, i) do { if ((i) < 32768) PPC_LHZ(r, base, i); \
- else { PPC_ADDIS(r, base, IMM_HA(i)); \
- PPC_LHZ(r, r, IMM_L(i)); } } while(0)
+#define PPC_LHZ_OFFS(r, base, i) do { if ((i) < 32768) EMIT(PPC_RAW_LHZ(r, base, i)); \
+ else { EMIT(PPC_RAW_ADDIS(r, base, IMM_HA(i))); \
+ EMIT(PPC_RAW_LHZ(r, r, IMM_L(i))); } } while(0)
#ifdef CONFIG_PPC64
#define PPC_LL_OFFS(r, base, i) do { PPC_LD_OFFS(r, base, i); } while(0)
@@ -107,20 +107,20 @@
} while(0)
#endif
#else
-#define PPC_BPF_LOAD_CPU(r) do { PPC_LI(r, 0); } while(0)
+#define PPC_BPF_LOAD_CPU(r) do { EMIT(PPC_RAW_LI(r, 0)); } while(0)
#endif
#define PPC_LHBRX_OFFS(r, base, i) \
- do { PPC_LI32(r, i); PPC_LHBRX(r, r, base); } while(0)
+ do { PPC_LI32(r, i); EMIT(PPC_RAW_LHBRX(r, r, base)); } while(0)
#ifdef __LITTLE_ENDIAN__
#define PPC_NTOHS_OFFS(r, base, i) PPC_LHBRX_OFFS(r, base, i)
#else
#define PPC_NTOHS_OFFS(r, base, i) PPC_LHZ_OFFS(r, base, i)
#endif
-#define PPC_BPF_LL(r, base, i) do { PPC_LWZ(r, base, i); } while(0)
-#define PPC_BPF_STL(r, base, i) do { PPC_STW(r, base, i); } while(0)
-#define PPC_BPF_STLU(r, base, i) do { PPC_STWU(r, base, i); } while(0)
+#define PPC_BPF_LL(r, base, i) do { EMIT(PPC_RAW_LWZ(r, base, i)); } while(0)
+#define PPC_BPF_STL(r, base, i) do { EMIT(PPC_RAW_STW(r, base, i)); } while(0)
+#define PPC_BPF_STLU(r, base, i) do { EMIT(PPC_RAW_STWU(r, base, i)); } while(0)
#define SEEN_DATAREF 0x10000 /* might call external helpers */
#define SEEN_XREG 0x20000 /* X reg is used */
diff --git a/arch/powerpc/net/bpf_jit64.h b/arch/powerpc/net/bpf_jit64.h
index cf3a7e3..2e33c667 100644
--- a/arch/powerpc/net/bpf_jit64.h
+++ b/arch/powerpc/net/bpf_jit64.h
@@ -70,19 +70,21 @@
*/
#define PPC_BPF_LL(r, base, i) do { \
if ((i) % 4) { \
- PPC_LI(b2p[TMP_REG_2], (i)); \
- PPC_LDX(r, base, b2p[TMP_REG_2]); \
+ EMIT(PPC_RAW_LI(b2p[TMP_REG_2], (i)));\
+ EMIT(PPC_RAW_LDX(r, base, \
+ b2p[TMP_REG_2])); \
} else \
- PPC_LD(r, base, i); \
+ EMIT(PPC_RAW_LD(r, base, i)); \
} while(0)
#define PPC_BPF_STL(r, base, i) do { \
if ((i) % 4) { \
- PPC_LI(b2p[TMP_REG_2], (i)); \
- PPC_STDX(r, base, b2p[TMP_REG_2]); \
+ EMIT(PPC_RAW_LI(b2p[TMP_REG_2], (i)));\
+ EMIT(PPC_RAW_STDX(r, base, \
+ b2p[TMP_REG_2])); \
} else \
- PPC_STD(r, base, i); \
+ EMIT(PPC_RAW_STD(r, base, i)); \
} while(0)
-#define PPC_BPF_STLU(r, base, i) do { PPC_STDU(r, base, i); } while(0)
+#define PPC_BPF_STLU(r, base, i) do { EMIT(PPC_RAW_STDU(r, base, i)); } while(0)
#define SEEN_FUNC 0x1000 /* might call external helpers */
#define SEEN_STACK 0x2000 /* uses BPF stack */
diff --git a/arch/powerpc/net/bpf_jit_comp.c b/arch/powerpc/net/bpf_jit_comp.c
index 0acc9d5..16d09b3 100644
--- a/arch/powerpc/net/bpf_jit_comp.c
+++ b/arch/powerpc/net/bpf_jit_comp.c
@@ -61,7 +61,7 @@
PPC_LWZ_OFFS(r_scratch1, r_skb, offsetof(struct sk_buff,
data_len));
PPC_LWZ_OFFS(r_HL, r_skb, offsetof(struct sk_buff, len));
- PPC_SUB(r_HL, r_HL, r_scratch1);
+ EMIT(PPC_RAW_SUB(r_HL, r_HL, r_scratch1));
PPC_LL_OFFS(r_D, r_skb, offsetof(struct sk_buff, data));
}
@@ -70,12 +70,12 @@
* TODO: Could also detect whether first instr. sets X and
* avoid this (as below, with A).
*/
- PPC_LI(r_X, 0);
+ EMIT(PPC_RAW_LI(r_X, 0));
}
/* make sure we dont leak kernel information to user */
if (bpf_needs_clear_a(&filter[0]))
- PPC_LI(r_A, 0);
+ EMIT(PPC_RAW_LI(r_A, 0));
}
static void bpf_jit_build_epilogue(u32 *image, struct codegen_context *ctx)
@@ -83,10 +83,10 @@
int i;
if (ctx->seen & (SEEN_MEM | SEEN_DATAREF)) {
- PPC_ADDI(1, 1, BPF_PPC_STACKFRAME);
+ EMIT(PPC_RAW_ADDI(1, 1, BPF_PPC_STACKFRAME));
if (ctx->seen & SEEN_DATAREF) {
PPC_BPF_LL(0, 1, PPC_LR_STKOFF);
- PPC_MTLR(0);
+ EMIT(PPC_RAW_MTLR(0));
PPC_BPF_LL(r_D, 1, -(REG_SZ*(32-r_D)));
PPC_BPF_LL(r_HL, 1, -(REG_SZ*(32-r_HL)));
}
@@ -100,7 +100,7 @@
}
/* The RETs have left a return value in R3. */
- PPC_BLR();
+ EMIT(PPC_RAW_BLR());
}
#define CHOOSE_LOAD_FUNC(K, func) \
@@ -134,124 +134,124 @@
/*** ALU ops ***/
case BPF_ALU | BPF_ADD | BPF_X: /* A += X; */
ctx->seen |= SEEN_XREG;
- PPC_ADD(r_A, r_A, r_X);
+ EMIT(PPC_RAW_ADD(r_A, r_A, r_X));
break;
case BPF_ALU | BPF_ADD | BPF_K: /* A += K; */
if (!K)
break;
- PPC_ADDI(r_A, r_A, IMM_L(K));
+ EMIT(PPC_RAW_ADDI(r_A, r_A, IMM_L(K)));
if (K >= 32768)
- PPC_ADDIS(r_A, r_A, IMM_HA(K));
+ EMIT(PPC_RAW_ADDIS(r_A, r_A, IMM_HA(K)));
break;
case BPF_ALU | BPF_SUB | BPF_X: /* A -= X; */
ctx->seen |= SEEN_XREG;
- PPC_SUB(r_A, r_A, r_X);
+ EMIT(PPC_RAW_SUB(r_A, r_A, r_X));
break;
case BPF_ALU | BPF_SUB | BPF_K: /* A -= K */
if (!K)
break;
- PPC_ADDI(r_A, r_A, IMM_L(-K));
+ EMIT(PPC_RAW_ADDI(r_A, r_A, IMM_L(-K)));
if (K >= 32768)
- PPC_ADDIS(r_A, r_A, IMM_HA(-K));
+ EMIT(PPC_RAW_ADDIS(r_A, r_A, IMM_HA(-K)));
break;
case BPF_ALU | BPF_MUL | BPF_X: /* A *= X; */
ctx->seen |= SEEN_XREG;
- PPC_MULW(r_A, r_A, r_X);
+ EMIT(PPC_RAW_MULW(r_A, r_A, r_X));
break;
case BPF_ALU | BPF_MUL | BPF_K: /* A *= K */
if (K < 32768)
- PPC_MULI(r_A, r_A, K);
+ EMIT(PPC_RAW_MULI(r_A, r_A, K));
else {
PPC_LI32(r_scratch1, K);
- PPC_MULW(r_A, r_A, r_scratch1);
+ EMIT(PPC_RAW_MULW(r_A, r_A, r_scratch1));
}
break;
case BPF_ALU | BPF_MOD | BPF_X: /* A %= X; */
case BPF_ALU | BPF_DIV | BPF_X: /* A /= X; */
ctx->seen |= SEEN_XREG;
- PPC_CMPWI(r_X, 0);
+ EMIT(PPC_RAW_CMPWI(r_X, 0));
if (ctx->pc_ret0 != -1) {
PPC_BCC(COND_EQ, addrs[ctx->pc_ret0]);
} else {
PPC_BCC_SHORT(COND_NE, (ctx->idx*4)+12);
- PPC_LI(r_ret, 0);
+ EMIT(PPC_RAW_LI(r_ret, 0));
PPC_JMP(exit_addr);
}
if (code == (BPF_ALU | BPF_MOD | BPF_X)) {
- PPC_DIVWU(r_scratch1, r_A, r_X);
- PPC_MULW(r_scratch1, r_X, r_scratch1);
- PPC_SUB(r_A, r_A, r_scratch1);
+ EMIT(PPC_RAW_DIVWU(r_scratch1, r_A, r_X));
+ EMIT(PPC_RAW_MULW(r_scratch1, r_X, r_scratch1));
+ EMIT(PPC_RAW_SUB(r_A, r_A, r_scratch1));
} else {
- PPC_DIVWU(r_A, r_A, r_X);
+ EMIT(PPC_RAW_DIVWU(r_A, r_A, r_X));
}
break;
case BPF_ALU | BPF_MOD | BPF_K: /* A %= K; */
PPC_LI32(r_scratch2, K);
- PPC_DIVWU(r_scratch1, r_A, r_scratch2);
- PPC_MULW(r_scratch1, r_scratch2, r_scratch1);
- PPC_SUB(r_A, r_A, r_scratch1);
+ EMIT(PPC_RAW_DIVWU(r_scratch1, r_A, r_scratch2));
+ EMIT(PPC_RAW_MULW(r_scratch1, r_scratch2, r_scratch1));
+ EMIT(PPC_RAW_SUB(r_A, r_A, r_scratch1));
break;
case BPF_ALU | BPF_DIV | BPF_K: /* A /= K */
if (K == 1)
break;
PPC_LI32(r_scratch1, K);
- PPC_DIVWU(r_A, r_A, r_scratch1);
+ EMIT(PPC_RAW_DIVWU(r_A, r_A, r_scratch1));
break;
case BPF_ALU | BPF_AND | BPF_X:
ctx->seen |= SEEN_XREG;
- PPC_AND(r_A, r_A, r_X);
+ EMIT(PPC_RAW_AND(r_A, r_A, r_X));
break;
case BPF_ALU | BPF_AND | BPF_K:
if (!IMM_H(K))
- PPC_ANDI(r_A, r_A, K);
+ EMIT(PPC_RAW_ANDI(r_A, r_A, K));
else {
PPC_LI32(r_scratch1, K);
- PPC_AND(r_A, r_A, r_scratch1);
+ EMIT(PPC_RAW_AND(r_A, r_A, r_scratch1));
}
break;
case BPF_ALU | BPF_OR | BPF_X:
ctx->seen |= SEEN_XREG;
- PPC_OR(r_A, r_A, r_X);
+ EMIT(PPC_RAW_OR(r_A, r_A, r_X));
break;
case BPF_ALU | BPF_OR | BPF_K:
if (IMM_L(K))
- PPC_ORI(r_A, r_A, IMM_L(K));
+ EMIT(PPC_RAW_ORI(r_A, r_A, IMM_L(K)));
if (K >= 65536)
- PPC_ORIS(r_A, r_A, IMM_H(K));
+ EMIT(PPC_RAW_ORIS(r_A, r_A, IMM_H(K)));
break;
case BPF_ANC | SKF_AD_ALU_XOR_X:
case BPF_ALU | BPF_XOR | BPF_X: /* A ^= X */
ctx->seen |= SEEN_XREG;
- PPC_XOR(r_A, r_A, r_X);
+ EMIT(PPC_RAW_XOR(r_A, r_A, r_X));
break;
case BPF_ALU | BPF_XOR | BPF_K: /* A ^= K */
if (IMM_L(K))
- PPC_XORI(r_A, r_A, IMM_L(K));
+ EMIT(PPC_RAW_XORI(r_A, r_A, IMM_L(K)));
if (K >= 65536)
- PPC_XORIS(r_A, r_A, IMM_H(K));
+ EMIT(PPC_RAW_XORIS(r_A, r_A, IMM_H(K)));
break;
case BPF_ALU | BPF_LSH | BPF_X: /* A <<= X; */
ctx->seen |= SEEN_XREG;
- PPC_SLW(r_A, r_A, r_X);
+ EMIT(PPC_RAW_SLW(r_A, r_A, r_X));
break;
case BPF_ALU | BPF_LSH | BPF_K:
if (K == 0)
break;
else
- PPC_SLWI(r_A, r_A, K);
+ EMIT(PPC_RAW_SLWI(r_A, r_A, K));
break;
case BPF_ALU | BPF_RSH | BPF_X: /* A >>= X; */
ctx->seen |= SEEN_XREG;
- PPC_SRW(r_A, r_A, r_X);
+ EMIT(PPC_RAW_SRW(r_A, r_A, r_X));
break;
case BPF_ALU | BPF_RSH | BPF_K: /* A >>= K; */
if (K == 0)
break;
else
- PPC_SRWI(r_A, r_A, K);
+ EMIT(PPC_RAW_SRWI(r_A, r_A, K));
break;
case BPF_ALU | BPF_NEG:
- PPC_NEG(r_A, r_A);
+ EMIT(PPC_RAW_NEG(r_A, r_A));
break;
case BPF_RET | BPF_K:
PPC_LI32(r_ret, K);
@@ -277,24 +277,24 @@
if (ctx->seen)
PPC_JMP(exit_addr);
else
- PPC_BLR();
+ EMIT(PPC_RAW_BLR());
}
break;
case BPF_RET | BPF_A:
- PPC_MR(r_ret, r_A);
+ EMIT(PPC_RAW_MR(r_ret, r_A));
if (i != flen - 1) {
if (ctx->seen)
PPC_JMP(exit_addr);
else
- PPC_BLR();
+ EMIT(PPC_RAW_BLR());
}
break;
case BPF_MISC | BPF_TAX: /* X = A */
- PPC_MR(r_X, r_A);
+ EMIT(PPC_RAW_MR(r_X, r_A));
break;
case BPF_MISC | BPF_TXA: /* A = X */
ctx->seen |= SEEN_XREG;
- PPC_MR(r_A, r_X);
+ EMIT(PPC_RAW_MR(r_A, r_X));
break;
/*** Constant loads/M[] access ***/
@@ -305,19 +305,19 @@
PPC_LI32(r_X, K);
break;
case BPF_LD | BPF_MEM: /* A = mem[K] */
- PPC_MR(r_A, r_M + (K & 0xf));
+ EMIT(PPC_RAW_MR(r_A, r_M + (K & 0xf)));
ctx->seen |= SEEN_MEM | (1<<(K & 0xf));
break;
case BPF_LDX | BPF_MEM: /* X = mem[K] */
- PPC_MR(r_X, r_M + (K & 0xf));
+ EMIT(PPC_RAW_MR(r_X, r_M + (K & 0xf)));
ctx->seen |= SEEN_MEM | (1<<(K & 0xf));
break;
case BPF_ST: /* mem[K] = A */
- PPC_MR(r_M + (K & 0xf), r_A);
+ EMIT(PPC_RAW_MR(r_M + (K & 0xf), r_A));
ctx->seen |= SEEN_MEM | (1<<(K & 0xf));
break;
case BPF_STX: /* mem[K] = X */
- PPC_MR(r_M + (K & 0xf), r_X);
+ EMIT(PPC_RAW_MR(r_M + (K & 0xf), r_X));
ctx->seen |= SEEN_XREG | SEEN_MEM | (1<<(K & 0xf));
break;
case BPF_LD | BPF_W | BPF_LEN: /* A = skb->len; */
@@ -346,13 +346,13 @@
type) != 2);
PPC_LL_OFFS(r_scratch1, r_skb, offsetof(struct sk_buff,
dev));
- PPC_CMPDI(r_scratch1, 0);
+ EMIT(PPC_RAW_CMPDI(r_scratch1, 0));
if (ctx->pc_ret0 != -1) {
PPC_BCC(COND_EQ, addrs[ctx->pc_ret0]);
} else {
/* Exit, returning 0; first pass hits here. */
PPC_BCC_SHORT(COND_NE, ctx->idx * 4 + 12);
- PPC_LI(r_ret, 0);
+ EMIT(PPC_RAW_LI(r_ret, 0));
PPC_JMP(exit_addr);
}
if (code == (BPF_ANC | SKF_AD_IFINDEX)) {
@@ -383,9 +383,9 @@
case BPF_ANC | SKF_AD_VLAN_TAG_PRESENT:
PPC_LBZ_OFFS(r_A, r_skb, PKT_VLAN_PRESENT_OFFSET());
if (PKT_VLAN_PRESENT_BIT)
- PPC_SRWI(r_A, r_A, PKT_VLAN_PRESENT_BIT);
+ EMIT(PPC_RAW_SRWI(r_A, r_A, PKT_VLAN_PRESENT_BIT));
if (PKT_VLAN_PRESENT_BIT < 7)
- PPC_ANDI(r_A, r_A, 1);
+ EMIT(PPC_RAW_ANDI(r_A, r_A, 1));
break;
case BPF_ANC | SKF_AD_QUEUE:
BUILD_BUG_ON(sizeof_field(struct sk_buff,
@@ -395,8 +395,8 @@
break;
case BPF_ANC | SKF_AD_PKTTYPE:
PPC_LBZ_OFFS(r_A, r_skb, PKT_TYPE_OFFSET());
- PPC_ANDI(r_A, r_A, PKT_TYPE_MAX);
- PPC_SRWI(r_A, r_A, 5);
+ EMIT(PPC_RAW_ANDI(r_A, r_A, PKT_TYPE_MAX));
+ EMIT(PPC_RAW_SRWI(r_A, r_A, 5));
break;
case BPF_ANC | SKF_AD_CPU:
PPC_BPF_LOAD_CPU(r_A);
@@ -414,9 +414,9 @@
/* Load from [K]. */
ctx->seen |= SEEN_DATAREF;
PPC_FUNC_ADDR(r_scratch1, func);
- PPC_MTLR(r_scratch1);
+ EMIT(PPC_RAW_MTLR(r_scratch1));
PPC_LI32(r_addr, K);
- PPC_BLRL();
+ EMIT(PPC_RAW_BLRL());
/*
* Helper returns 'lt' condition on error, and an
* appropriate return value in r3
@@ -440,11 +440,11 @@
*/
ctx->seen |= SEEN_DATAREF | SEEN_XREG;
PPC_FUNC_ADDR(r_scratch1, func);
- PPC_MTLR(r_scratch1);
- PPC_ADDI(r_addr, r_X, IMM_L(K));
+ EMIT(PPC_RAW_MTLR(r_scratch1));
+ EMIT(PPC_RAW_ADDI(r_addr, r_X, IMM_L(K)));
if (K >= 32768)
- PPC_ADDIS(r_addr, r_addr, IMM_HA(K));
- PPC_BLRL();
+ EMIT(PPC_RAW_ADDIS(r_addr, r_addr, IMM_HA(K)));
+ EMIT(PPC_RAW_BLRL());
/* If error, cr0.LT set */
PPC_BCC(COND_LT, exit_addr);
break;
@@ -489,30 +489,30 @@
case BPF_JMP | BPF_JGE | BPF_X:
case BPF_JMP | BPF_JEQ | BPF_X:
ctx->seen |= SEEN_XREG;
- PPC_CMPLW(r_A, r_X);
+ EMIT(PPC_RAW_CMPLW(r_A, r_X));
break;
case BPF_JMP | BPF_JSET | BPF_X:
ctx->seen |= SEEN_XREG;
- PPC_AND_DOT(r_scratch1, r_A, r_X);
+ EMIT(PPC_RAW_AND_DOT(r_scratch1, r_A, r_X));
break;
case BPF_JMP | BPF_JEQ | BPF_K:
case BPF_JMP | BPF_JGT | BPF_K:
case BPF_JMP | BPF_JGE | BPF_K:
if (K < 32768)
- PPC_CMPLWI(r_A, K);
+ EMIT(PPC_RAW_CMPLWI(r_A, K));
else {
PPC_LI32(r_scratch1, K);
- PPC_CMPLW(r_A, r_scratch1);
+ EMIT(PPC_RAW_CMPLW(r_A, r_scratch1));
}
break;
case BPF_JMP | BPF_JSET | BPF_K:
if (K < 32768)
/* PPC_ANDI is /only/ dot-form */
- PPC_ANDI(r_scratch1, r_A, K);
+ EMIT(PPC_RAW_ANDI(r_scratch1, r_A, K));
else {
PPC_LI32(r_scratch1, K);
- PPC_AND_DOT(r_scratch1, r_A,
- r_scratch1);
+ EMIT(PPC_RAW_AND_DOT(r_scratch1, r_A,
+ r_scratch1));
}
break;
}
diff --git a/arch/powerpc/net/bpf_jit_comp64.c b/arch/powerpc/net/bpf_jit_comp64.c
index be3517e..022103c 100644
--- a/arch/powerpc/net/bpf_jit_comp64.c
+++ b/arch/powerpc/net/bpf_jit_comp64.c
@@ -95,12 +95,12 @@
* invoked through a tail call.
*/
if (ctx->seen & SEEN_TAILCALL) {
- PPC_LI(b2p[TMP_REG_1], 0);
+ EMIT(PPC_RAW_LI(b2p[TMP_REG_1], 0));
/* this goes in the redzone */
PPC_BPF_STL(b2p[TMP_REG_1], 1, -(BPF_PPC_STACK_SAVE + 8));
} else {
- PPC_NOP();
- PPC_NOP();
+ EMIT(PPC_RAW_NOP());
+ EMIT(PPC_RAW_NOP());
}
#define BPF_TAILCALL_PROLOGUE_SIZE 8
@@ -129,8 +129,8 @@
/* Setup frame pointer to point to the bpf stack area */
if (bpf_is_seen_register(ctx, BPF_REG_FP))
- PPC_ADDI(b2p[BPF_REG_FP], 1,
- STACK_FRAME_MIN_SIZE + ctx->stack_size);
+ EMIT(PPC_RAW_ADDI(b2p[BPF_REG_FP], 1,
+ STACK_FRAME_MIN_SIZE + ctx->stack_size));
}
static void bpf_jit_emit_common_epilogue(u32 *image, struct codegen_context *ctx)
@@ -144,10 +144,10 @@
/* Tear down our stack frame */
if (bpf_has_stack_frame(ctx)) {
- PPC_ADDI(1, 1, BPF_PPC_STACKFRAME + ctx->stack_size);
+ EMIT(PPC_RAW_ADDI(1, 1, BPF_PPC_STACKFRAME + ctx->stack_size));
if (ctx->seen & SEEN_FUNC) {
PPC_BPF_LL(0, 1, PPC_LR_STKOFF);
- PPC_MTLR(0);
+ EMIT(PPC_RAW_MTLR(0));
}
}
}
@@ -157,9 +157,9 @@
bpf_jit_emit_common_epilogue(image, ctx);
/* Move result to r3 */
- PPC_MR(3, b2p[BPF_REG_0]);
+ EMIT(PPC_RAW_MR(3, b2p[BPF_REG_0]));
- PPC_BLR();
+ EMIT(PPC_RAW_BLR());
}
static void bpf_jit_emit_func_call_hlp(u32 *image, struct codegen_context *ctx,
@@ -171,7 +171,7 @@
/* Load actual entry point from function descriptor */
PPC_BPF_LL(b2p[TMP_REG_1], b2p[TMP_REG_2], 0);
/* ... and move it to LR */
- PPC_MTLR(b2p[TMP_REG_1]);
+ EMIT(PPC_RAW_MTLR(b2p[TMP_REG_1]));
/*
* Load TOC from function descriptor at offset 8.
* We can clobber r2 since we get called through a
@@ -182,9 +182,9 @@
#else
/* We can clobber r12 */
PPC_FUNC_ADDR(12, func);
- PPC_MTLR(12);
+ EMIT(PPC_RAW_MTLR(12));
#endif
- PPC_BLRL();
+ EMIT(PPC_RAW_BLRL());
}
static void bpf_jit_emit_func_call_rel(u32 *image, struct codegen_context *ctx,
@@ -206,7 +206,7 @@
* that PPC_LI64() can emit.
*/
for (i = ctx->idx - ctx_idx; i < 5; i++)
- PPC_NOP();
+ EMIT(PPC_RAW_NOP());
#ifdef PPC64_ELF_ABI_v1
/*
@@ -220,8 +220,8 @@
PPC_BPF_LL(12, 12, 0);
#endif
- PPC_MTLR(12);
- PPC_BLRL();
+ EMIT(PPC_RAW_MTLR(12));
+ EMIT(PPC_RAW_BLRL());
}
static void bpf_jit_emit_tail_call(u32 *image, struct codegen_context *ctx, u32 out)
@@ -239,9 +239,9 @@
* if (index >= array->map.max_entries)
* goto out;
*/
- PPC_LWZ(b2p[TMP_REG_1], b2p_bpf_array, offsetof(struct bpf_array, map.max_entries));
- PPC_RLWINM(b2p_index, b2p_index, 0, 0, 31);
- PPC_CMPLW(b2p_index, b2p[TMP_REG_1]);
+ EMIT(PPC_RAW_LWZ(b2p[TMP_REG_1], b2p_bpf_array, offsetof(struct bpf_array, map.max_entries)));
+ EMIT(PPC_RAW_RLWINM(b2p_index, b2p_index, 0, 0, 31));
+ EMIT(PPC_RAW_CMPLW(b2p_index, b2p[TMP_REG_1]));
PPC_BCC(COND_GE, out);
/*
@@ -249,42 +249,42 @@
* goto out;
*/
PPC_BPF_LL(b2p[TMP_REG_1], 1, bpf_jit_stack_tailcallcnt(ctx));
- PPC_CMPLWI(b2p[TMP_REG_1], MAX_TAIL_CALL_CNT);
+ EMIT(PPC_RAW_CMPLWI(b2p[TMP_REG_1], MAX_TAIL_CALL_CNT));
PPC_BCC(COND_GT, out);
/*
* tail_call_cnt++;
*/
- PPC_ADDI(b2p[TMP_REG_1], b2p[TMP_REG_1], 1);
+ EMIT(PPC_RAW_ADDI(b2p[TMP_REG_1], b2p[TMP_REG_1], 1));
PPC_BPF_STL(b2p[TMP_REG_1], 1, bpf_jit_stack_tailcallcnt(ctx));
/* prog = array->ptrs[index]; */
- PPC_MULI(b2p[TMP_REG_1], b2p_index, 8);
- PPC_ADD(b2p[TMP_REG_1], b2p[TMP_REG_1], b2p_bpf_array);
+ EMIT(PPC_RAW_MULI(b2p[TMP_REG_1], b2p_index, 8));
+ EMIT(PPC_RAW_ADD(b2p[TMP_REG_1], b2p[TMP_REG_1], b2p_bpf_array));
PPC_BPF_LL(b2p[TMP_REG_1], b2p[TMP_REG_1], offsetof(struct bpf_array, ptrs));
/*
* if (prog == NULL)
* goto out;
*/
- PPC_CMPLDI(b2p[TMP_REG_1], 0);
+ EMIT(PPC_RAW_CMPLDI(b2p[TMP_REG_1], 0));
PPC_BCC(COND_EQ, out);
/* goto *(prog->bpf_func + prologue_size); */
PPC_BPF_LL(b2p[TMP_REG_1], b2p[TMP_REG_1], offsetof(struct bpf_prog, bpf_func));
#ifdef PPC64_ELF_ABI_v1
/* skip past the function descriptor */
- PPC_ADDI(b2p[TMP_REG_1], b2p[TMP_REG_1],
- FUNCTION_DESCR_SIZE + BPF_TAILCALL_PROLOGUE_SIZE);
+ EMIT(PPC_RAW_ADDI(b2p[TMP_REG_1], b2p[TMP_REG_1],
+ FUNCTION_DESCR_SIZE + BPF_TAILCALL_PROLOGUE_SIZE));
#else
- PPC_ADDI(b2p[TMP_REG_1], b2p[TMP_REG_1], BPF_TAILCALL_PROLOGUE_SIZE);
+ EMIT(PPC_RAW_ADDI(b2p[TMP_REG_1], b2p[TMP_REG_1], BPF_TAILCALL_PROLOGUE_SIZE));
#endif
- PPC_MTCTR(b2p[TMP_REG_1]);
+ EMIT(PPC_RAW_MTCTR(b2p[TMP_REG_1]));
/* tear down stack, restore NVRs, ... */
bpf_jit_emit_common_epilogue(image, ctx);
- PPC_BCTR();
+ EMIT(PPC_RAW_BCTR());
/* out: */
}
@@ -340,11 +340,11 @@
*/
case BPF_ALU | BPF_ADD | BPF_X: /* (u32) dst += (u32) src */
case BPF_ALU64 | BPF_ADD | BPF_X: /* dst += src */
- PPC_ADD(dst_reg, dst_reg, src_reg);
+ EMIT(PPC_RAW_ADD(dst_reg, dst_reg, src_reg));
goto bpf_alu32_trunc;
case BPF_ALU | BPF_SUB | BPF_X: /* (u32) dst -= (u32) src */
case BPF_ALU64 | BPF_SUB | BPF_X: /* dst -= src */
- PPC_SUB(dst_reg, dst_reg, src_reg);
+ EMIT(PPC_RAW_SUB(dst_reg, dst_reg, src_reg));
goto bpf_alu32_trunc;
case BPF_ALU | BPF_ADD | BPF_K: /* (u32) dst += (u32) imm */
case BPF_ALU | BPF_SUB | BPF_K: /* (u32) dst -= (u32) imm */
@@ -354,53 +354,53 @@
imm = -imm;
if (imm) {
if (imm >= -32768 && imm < 32768)
- PPC_ADDI(dst_reg, dst_reg, IMM_L(imm));
+ EMIT(PPC_RAW_ADDI(dst_reg, dst_reg, IMM_L(imm)));
else {
PPC_LI32(b2p[TMP_REG_1], imm);
- PPC_ADD(dst_reg, dst_reg, b2p[TMP_REG_1]);
+ EMIT(PPC_RAW_ADD(dst_reg, dst_reg, b2p[TMP_REG_1]));
}
}
goto bpf_alu32_trunc;
case BPF_ALU | BPF_MUL | BPF_X: /* (u32) dst *= (u32) src */
case BPF_ALU64 | BPF_MUL | BPF_X: /* dst *= src */
if (BPF_CLASS(code) == BPF_ALU)
- PPC_MULW(dst_reg, dst_reg, src_reg);
+ EMIT(PPC_RAW_MULW(dst_reg, dst_reg, src_reg));
else
- PPC_MULD(dst_reg, dst_reg, src_reg);
+ EMIT(PPC_RAW_MULD(dst_reg, dst_reg, src_reg));
goto bpf_alu32_trunc;
case BPF_ALU | BPF_MUL | BPF_K: /* (u32) dst *= (u32) imm */
case BPF_ALU64 | BPF_MUL | BPF_K: /* dst *= imm */
if (imm >= -32768 && imm < 32768)
- PPC_MULI(dst_reg, dst_reg, IMM_L(imm));
+ EMIT(PPC_RAW_MULI(dst_reg, dst_reg, IMM_L(imm)));
else {
PPC_LI32(b2p[TMP_REG_1], imm);
if (BPF_CLASS(code) == BPF_ALU)
- PPC_MULW(dst_reg, dst_reg,
- b2p[TMP_REG_1]);
+ EMIT(PPC_RAW_MULW(dst_reg, dst_reg,
+ b2p[TMP_REG_1]));
else
- PPC_MULD(dst_reg, dst_reg,
- b2p[TMP_REG_1]);
+ EMIT(PPC_RAW_MULD(dst_reg, dst_reg,
+ b2p[TMP_REG_1]));
}
goto bpf_alu32_trunc;
case BPF_ALU | BPF_DIV | BPF_X: /* (u32) dst /= (u32) src */
case BPF_ALU | BPF_MOD | BPF_X: /* (u32) dst %= (u32) src */
if (BPF_OP(code) == BPF_MOD) {
- PPC_DIVWU(b2p[TMP_REG_1], dst_reg, src_reg);
- PPC_MULW(b2p[TMP_REG_1], src_reg,
- b2p[TMP_REG_1]);
- PPC_SUB(dst_reg, dst_reg, b2p[TMP_REG_1]);
+ EMIT(PPC_RAW_DIVWU(b2p[TMP_REG_1], dst_reg, src_reg));
+ EMIT(PPC_RAW_MULW(b2p[TMP_REG_1], src_reg,
+ b2p[TMP_REG_1]));
+ EMIT(PPC_RAW_SUB(dst_reg, dst_reg, b2p[TMP_REG_1]));
} else
- PPC_DIVWU(dst_reg, dst_reg, src_reg);
+ EMIT(PPC_RAW_DIVWU(dst_reg, dst_reg, src_reg));
goto bpf_alu32_trunc;
case BPF_ALU64 | BPF_DIV | BPF_X: /* dst /= src */
case BPF_ALU64 | BPF_MOD | BPF_X: /* dst %= src */
if (BPF_OP(code) == BPF_MOD) {
- PPC_DIVDU(b2p[TMP_REG_1], dst_reg, src_reg);
- PPC_MULD(b2p[TMP_REG_1], src_reg,
- b2p[TMP_REG_1]);
- PPC_SUB(dst_reg, dst_reg, b2p[TMP_REG_1]);
+ EMIT(PPC_RAW_DIVDU(b2p[TMP_REG_1], dst_reg, src_reg));
+ EMIT(PPC_RAW_MULD(b2p[TMP_REG_1], src_reg,
+ b2p[TMP_REG_1]));
+ EMIT(PPC_RAW_SUB(dst_reg, dst_reg, b2p[TMP_REG_1]));
} else
- PPC_DIVDU(dst_reg, dst_reg, src_reg);
+ EMIT(PPC_RAW_DIVDU(dst_reg, dst_reg, src_reg));
break;
case BPF_ALU | BPF_MOD | BPF_K: /* (u32) dst %= (u32) imm */
case BPF_ALU | BPF_DIV | BPF_K: /* (u32) dst /= (u32) imm */
@@ -415,35 +415,37 @@
switch (BPF_CLASS(code)) {
case BPF_ALU:
if (BPF_OP(code) == BPF_MOD) {
- PPC_DIVWU(b2p[TMP_REG_2], dst_reg,
- b2p[TMP_REG_1]);
- PPC_MULW(b2p[TMP_REG_1],
+ EMIT(PPC_RAW_DIVWU(b2p[TMP_REG_2],
+ dst_reg,
+ b2p[TMP_REG_1]));
+ EMIT(PPC_RAW_MULW(b2p[TMP_REG_1],
b2p[TMP_REG_1],
- b2p[TMP_REG_2]);
- PPC_SUB(dst_reg, dst_reg,
- b2p[TMP_REG_1]);
+ b2p[TMP_REG_2]));
+ EMIT(PPC_RAW_SUB(dst_reg, dst_reg,
+ b2p[TMP_REG_1]));
} else
- PPC_DIVWU(dst_reg, dst_reg,
- b2p[TMP_REG_1]);
+ EMIT(PPC_RAW_DIVWU(dst_reg, dst_reg,
+ b2p[TMP_REG_1]));
break;
case BPF_ALU64:
if (BPF_OP(code) == BPF_MOD) {
- PPC_DIVDU(b2p[TMP_REG_2], dst_reg,
- b2p[TMP_REG_1]);
- PPC_MULD(b2p[TMP_REG_1],
+ EMIT(PPC_RAW_DIVDU(b2p[TMP_REG_2],
+ dst_reg,
+ b2p[TMP_REG_1]));
+ EMIT(PPC_RAW_MULD(b2p[TMP_REG_1],
b2p[TMP_REG_1],
- b2p[TMP_REG_2]);
- PPC_SUB(dst_reg, dst_reg,
- b2p[TMP_REG_1]);
+ b2p[TMP_REG_2]));
+ EMIT(PPC_RAW_SUB(dst_reg, dst_reg,
+ b2p[TMP_REG_1]));
} else
- PPC_DIVDU(dst_reg, dst_reg,
- b2p[TMP_REG_1]);
+ EMIT(PPC_RAW_DIVDU(dst_reg, dst_reg,
+ b2p[TMP_REG_1]));
break;
}
goto bpf_alu32_trunc;
case BPF_ALU | BPF_NEG: /* (u32) dst = -dst */
case BPF_ALU64 | BPF_NEG: /* dst = -dst */
- PPC_NEG(dst_reg, dst_reg);
+ EMIT(PPC_RAW_NEG(dst_reg, dst_reg));
goto bpf_alu32_trunc;
/*
@@ -451,101 +453,101 @@
*/
case BPF_ALU | BPF_AND | BPF_X: /* (u32) dst = dst & src */
case BPF_ALU64 | BPF_AND | BPF_X: /* dst = dst & src */
- PPC_AND(dst_reg, dst_reg, src_reg);
+ EMIT(PPC_RAW_AND(dst_reg, dst_reg, src_reg));
goto bpf_alu32_trunc;
case BPF_ALU | BPF_AND | BPF_K: /* (u32) dst = dst & imm */
case BPF_ALU64 | BPF_AND | BPF_K: /* dst = dst & imm */
if (!IMM_H(imm))
- PPC_ANDI(dst_reg, dst_reg, IMM_L(imm));
+ EMIT(PPC_RAW_ANDI(dst_reg, dst_reg, IMM_L(imm)));
else {
/* Sign-extended */
PPC_LI32(b2p[TMP_REG_1], imm);
- PPC_AND(dst_reg, dst_reg, b2p[TMP_REG_1]);
+ EMIT(PPC_RAW_AND(dst_reg, dst_reg, b2p[TMP_REG_1]));
}
goto bpf_alu32_trunc;
case BPF_ALU | BPF_OR | BPF_X: /* dst = (u32) dst | (u32) src */
case BPF_ALU64 | BPF_OR | BPF_X: /* dst = dst | src */
- PPC_OR(dst_reg, dst_reg, src_reg);
+ EMIT(PPC_RAW_OR(dst_reg, dst_reg, src_reg));
goto bpf_alu32_trunc;
case BPF_ALU | BPF_OR | BPF_K:/* dst = (u32) dst | (u32) imm */
case BPF_ALU64 | BPF_OR | BPF_K:/* dst = dst | imm */
if (imm < 0 && BPF_CLASS(code) == BPF_ALU64) {
/* Sign-extended */
PPC_LI32(b2p[TMP_REG_1], imm);
- PPC_OR(dst_reg, dst_reg, b2p[TMP_REG_1]);
+ EMIT(PPC_RAW_OR(dst_reg, dst_reg, b2p[TMP_REG_1]));
} else {
if (IMM_L(imm))
- PPC_ORI(dst_reg, dst_reg, IMM_L(imm));
+ EMIT(PPC_RAW_ORI(dst_reg, dst_reg, IMM_L(imm)));
if (IMM_H(imm))
- PPC_ORIS(dst_reg, dst_reg, IMM_H(imm));
+ EMIT(PPC_RAW_ORIS(dst_reg, dst_reg, IMM_H(imm)));
}
goto bpf_alu32_trunc;
case BPF_ALU | BPF_XOR | BPF_X: /* (u32) dst ^= src */
case BPF_ALU64 | BPF_XOR | BPF_X: /* dst ^= src */
- PPC_XOR(dst_reg, dst_reg, src_reg);
+ EMIT(PPC_RAW_XOR(dst_reg, dst_reg, src_reg));
goto bpf_alu32_trunc;
case BPF_ALU | BPF_XOR | BPF_K: /* (u32) dst ^= (u32) imm */
case BPF_ALU64 | BPF_XOR | BPF_K: /* dst ^= imm */
if (imm < 0 && BPF_CLASS(code) == BPF_ALU64) {
/* Sign-extended */
PPC_LI32(b2p[TMP_REG_1], imm);
- PPC_XOR(dst_reg, dst_reg, b2p[TMP_REG_1]);
+ EMIT(PPC_RAW_XOR(dst_reg, dst_reg, b2p[TMP_REG_1]));
} else {
if (IMM_L(imm))
- PPC_XORI(dst_reg, dst_reg, IMM_L(imm));
+ EMIT(PPC_RAW_XORI(dst_reg, dst_reg, IMM_L(imm)));
if (IMM_H(imm))
- PPC_XORIS(dst_reg, dst_reg, IMM_H(imm));
+ EMIT(PPC_RAW_XORIS(dst_reg, dst_reg, IMM_H(imm)));
}
goto bpf_alu32_trunc;
case BPF_ALU | BPF_LSH | BPF_X: /* (u32) dst <<= (u32) src */
/* slw clears top 32 bits */
- PPC_SLW(dst_reg, dst_reg, src_reg);
+ EMIT(PPC_RAW_SLW(dst_reg, dst_reg, src_reg));
/* skip zero extension move, but set address map. */
if (insn_is_zext(&insn[i + 1]))
addrs[++i] = ctx->idx * 4;
break;
case BPF_ALU64 | BPF_LSH | BPF_X: /* dst <<= src; */
- PPC_SLD(dst_reg, dst_reg, src_reg);
+ EMIT(PPC_RAW_SLD(dst_reg, dst_reg, src_reg));
break;
case BPF_ALU | BPF_LSH | BPF_K: /* (u32) dst <<== (u32) imm */
/* with imm 0, we still need to clear top 32 bits */
- PPC_SLWI(dst_reg, dst_reg, imm);
+ EMIT(PPC_RAW_SLWI(dst_reg, dst_reg, imm));
if (insn_is_zext(&insn[i + 1]))
addrs[++i] = ctx->idx * 4;
break;
case BPF_ALU64 | BPF_LSH | BPF_K: /* dst <<== imm */
if (imm != 0)
- PPC_SLDI(dst_reg, dst_reg, imm);
+ EMIT(PPC_RAW_SLDI(dst_reg, dst_reg, imm));
break;
case BPF_ALU | BPF_RSH | BPF_X: /* (u32) dst >>= (u32) src */
- PPC_SRW(dst_reg, dst_reg, src_reg);
+ EMIT(PPC_RAW_SRW(dst_reg, dst_reg, src_reg));
if (insn_is_zext(&insn[i + 1]))
addrs[++i] = ctx->idx * 4;
break;
case BPF_ALU64 | BPF_RSH | BPF_X: /* dst >>= src */
- PPC_SRD(dst_reg, dst_reg, src_reg);
+ EMIT(PPC_RAW_SRD(dst_reg, dst_reg, src_reg));
break;
case BPF_ALU | BPF_RSH | BPF_K: /* (u32) dst >>= (u32) imm */
- PPC_SRWI(dst_reg, dst_reg, imm);
+ EMIT(PPC_RAW_SRWI(dst_reg, dst_reg, imm));
if (insn_is_zext(&insn[i + 1]))
addrs[++i] = ctx->idx * 4;
break;
case BPF_ALU64 | BPF_RSH | BPF_K: /* dst >>= imm */
if (imm != 0)
- PPC_SRDI(dst_reg, dst_reg, imm);
+ EMIT(PPC_RAW_SRDI(dst_reg, dst_reg, imm));
break;
case BPF_ALU | BPF_ARSH | BPF_X: /* (s32) dst >>= src */
- PPC_SRAW(dst_reg, dst_reg, src_reg);
+ EMIT(PPC_RAW_SRAW(dst_reg, dst_reg, src_reg));
goto bpf_alu32_trunc;
case BPF_ALU64 | BPF_ARSH | BPF_X: /* (s64) dst >>= src */
- PPC_SRAD(dst_reg, dst_reg, src_reg);
+ EMIT(PPC_RAW_SRAD(dst_reg, dst_reg, src_reg));
break;
case BPF_ALU | BPF_ARSH | BPF_K: /* (s32) dst >>= imm */
- PPC_SRAWI(dst_reg, dst_reg, imm);
+ EMIT(PPC_RAW_SRAWI(dst_reg, dst_reg, imm));
goto bpf_alu32_trunc;
case BPF_ALU64 | BPF_ARSH | BPF_K: /* (s64) dst >>= imm */
if (imm != 0)
- PPC_SRADI(dst_reg, dst_reg, imm);
+ EMIT(PPC_RAW_SRADI(dst_reg, dst_reg, imm));
break;
/*
@@ -555,10 +557,10 @@
case BPF_ALU64 | BPF_MOV | BPF_X: /* dst = src */
if (imm == 1) {
/* special mov32 for zext */
- PPC_RLWINM(dst_reg, dst_reg, 0, 0, 31);
+ EMIT(PPC_RAW_RLWINM(dst_reg, dst_reg, 0, 0, 31));
break;
}
- PPC_MR(dst_reg, src_reg);
+ EMIT(PPC_RAW_MR(dst_reg, src_reg));
goto bpf_alu32_trunc;
case BPF_ALU | BPF_MOV | BPF_K: /* (u32) dst = imm */
case BPF_ALU64 | BPF_MOV | BPF_K: /* dst = (s64) imm */
@@ -572,7 +574,7 @@
bpf_alu32_trunc:
/* Truncate to 32-bits */
if (BPF_CLASS(code) == BPF_ALU && !fp->aux->verifier_zext)
- PPC_RLWINM(dst_reg, dst_reg, 0, 0, 31);
+ EMIT(PPC_RAW_RLWINM(dst_reg, dst_reg, 0, 0, 31));
break;
/*
@@ -590,11 +592,11 @@
switch (imm) {
case 16:
/* Rotate 8 bits left & mask with 0x0000ff00 */
- PPC_RLWINM(b2p[TMP_REG_1], dst_reg, 8, 16, 23);
+ EMIT(PPC_RAW_RLWINM(b2p[TMP_REG_1], dst_reg, 8, 16, 23));
/* Rotate 8 bits right & insert LSB to reg */
- PPC_RLWIMI(b2p[TMP_REG_1], dst_reg, 24, 24, 31);
+ EMIT(PPC_RAW_RLWIMI(b2p[TMP_REG_1], dst_reg, 24, 24, 31));
/* Move result back to dst_reg */
- PPC_MR(dst_reg, b2p[TMP_REG_1]);
+ EMIT(PPC_RAW_MR(dst_reg, b2p[TMP_REG_1]));
break;
case 32:
/*
@@ -602,12 +604,12 @@
* 2 bytes are already in their final position
* -- byte 2 and 4 (of bytes 1, 2, 3 and 4)
*/
- PPC_RLWINM(b2p[TMP_REG_1], dst_reg, 8, 0, 31);
+ EMIT(PPC_RAW_RLWINM(b2p[TMP_REG_1], dst_reg, 8, 0, 31));
/* Rotate 24 bits and insert byte 1 */
- PPC_RLWIMI(b2p[TMP_REG_1], dst_reg, 24, 0, 7);
+ EMIT(PPC_RAW_RLWIMI(b2p[TMP_REG_1], dst_reg, 24, 0, 7));
/* Rotate 24 bits and insert byte 3 */
- PPC_RLWIMI(b2p[TMP_REG_1], dst_reg, 24, 16, 23);
- PPC_MR(dst_reg, b2p[TMP_REG_1]);
+ EMIT(PPC_RAW_RLWIMI(b2p[TMP_REG_1], dst_reg, 24, 16, 23));
+ EMIT(PPC_RAW_MR(dst_reg, b2p[TMP_REG_1]));
break;
case 64:
/*
@@ -619,8 +621,8 @@
* same across all passes
*/
PPC_BPF_STL(dst_reg, 1, bpf_jit_stack_local(ctx));
- PPC_ADDI(b2p[TMP_REG_1], 1, bpf_jit_stack_local(ctx));
- PPC_LDBRX(dst_reg, 0, b2p[TMP_REG_1]);
+ EMIT(PPC_RAW_ADDI(b2p[TMP_REG_1], 1, bpf_jit_stack_local(ctx)));
+ EMIT(PPC_RAW_LDBRX(dst_reg, 0, b2p[TMP_REG_1]));
break;
}
break;
@@ -629,14 +631,14 @@
switch (imm) {
case 16:
/* zero-extend 16 bits into 64 bits */
- PPC_RLDICL(dst_reg, dst_reg, 0, 48);
+ EMIT(PPC_RAW_RLDICL(dst_reg, dst_reg, 0, 48));
if (insn_is_zext(&insn[i + 1]))
addrs[++i] = ctx->idx * 4;
break;
case 32:
if (!fp->aux->verifier_zext)
/* zero-extend 32 bits into 64 bits */
- PPC_RLDICL(dst_reg, dst_reg, 0, 32);
+ EMIT(PPC_RAW_RLDICL(dst_reg, dst_reg, 0, 32));
break;
case 64:
/* nop */
@@ -650,18 +652,18 @@
case BPF_STX | BPF_MEM | BPF_B: /* *(u8 *)(dst + off) = src */
case BPF_ST | BPF_MEM | BPF_B: /* *(u8 *)(dst + off) = imm */
if (BPF_CLASS(code) == BPF_ST) {
- PPC_LI(b2p[TMP_REG_1], imm);
+ EMIT(PPC_RAW_LI(b2p[TMP_REG_1], imm));
src_reg = b2p[TMP_REG_1];
}
- PPC_STB(src_reg, dst_reg, off);
+ EMIT(PPC_RAW_STB(src_reg, dst_reg, off));
break;
case BPF_STX | BPF_MEM | BPF_H: /* (u16 *)(dst + off) = src */
case BPF_ST | BPF_MEM | BPF_H: /* (u16 *)(dst + off) = imm */
if (BPF_CLASS(code) == BPF_ST) {
- PPC_LI(b2p[TMP_REG_1], imm);
+ EMIT(PPC_RAW_LI(b2p[TMP_REG_1], imm));
src_reg = b2p[TMP_REG_1];
}
- PPC_STH(src_reg, dst_reg, off);
+ EMIT(PPC_RAW_STH(src_reg, dst_reg, off));
break;
case BPF_STX | BPF_MEM | BPF_W: /* *(u32 *)(dst + off) = src */
case BPF_ST | BPF_MEM | BPF_W: /* *(u32 *)(dst + off) = imm */
@@ -669,7 +671,7 @@
PPC_LI32(b2p[TMP_REG_1], imm);
src_reg = b2p[TMP_REG_1];
}
- PPC_STW(src_reg, dst_reg, off);
+ EMIT(PPC_RAW_STW(src_reg, dst_reg, off));
break;
case BPF_STX | BPF_MEM | BPF_DW: /* (u64 *)(dst + off) = src */
case BPF_ST | BPF_MEM | BPF_DW: /* *(u64 *)(dst + off) = imm */
@@ -686,24 +688,24 @@
/* *(u32 *)(dst + off) += src */
case BPF_STX | BPF_XADD | BPF_W:
/* Get EA into TMP_REG_1 */
- PPC_ADDI(b2p[TMP_REG_1], dst_reg, off);
+ EMIT(PPC_RAW_ADDI(b2p[TMP_REG_1], dst_reg, off));
tmp_idx = ctx->idx * 4;
/* load value from memory into TMP_REG_2 */
- PPC_BPF_LWARX(b2p[TMP_REG_2], 0, b2p[TMP_REG_1], 0);
+ EMIT(PPC_RAW_LWARX(b2p[TMP_REG_2], 0, b2p[TMP_REG_1], 0));
/* add value from src_reg into this */
- PPC_ADD(b2p[TMP_REG_2], b2p[TMP_REG_2], src_reg);
+ EMIT(PPC_RAW_ADD(b2p[TMP_REG_2], b2p[TMP_REG_2], src_reg));
/* store result back */
- PPC_BPF_STWCX(b2p[TMP_REG_2], 0, b2p[TMP_REG_1]);
+ EMIT(PPC_RAW_STWCX(b2p[TMP_REG_2], 0, b2p[TMP_REG_1]));
/* we're done if this succeeded */
PPC_BCC_SHORT(COND_NE, tmp_idx);
break;
/* *(u64 *)(dst + off) += src */
case BPF_STX | BPF_XADD | BPF_DW:
- PPC_ADDI(b2p[TMP_REG_1], dst_reg, off);
+ EMIT(PPC_RAW_ADDI(b2p[TMP_REG_1], dst_reg, off));
tmp_idx = ctx->idx * 4;
- PPC_BPF_LDARX(b2p[TMP_REG_2], 0, b2p[TMP_REG_1], 0);
- PPC_ADD(b2p[TMP_REG_2], b2p[TMP_REG_2], src_reg);
- PPC_BPF_STDCX(b2p[TMP_REG_2], 0, b2p[TMP_REG_1]);
+ EMIT(PPC_RAW_LDARX(b2p[TMP_REG_2], 0, b2p[TMP_REG_1], 0));
+ EMIT(PPC_RAW_ADD(b2p[TMP_REG_2], b2p[TMP_REG_2], src_reg));
+ EMIT(PPC_RAW_STDCX(b2p[TMP_REG_2], 0, b2p[TMP_REG_1]));
PPC_BCC_SHORT(COND_NE, tmp_idx);
break;
@@ -712,19 +714,19 @@
*/
/* dst = *(u8 *)(ul) (src + off) */
case BPF_LDX | BPF_MEM | BPF_B:
- PPC_LBZ(dst_reg, src_reg, off);
+ EMIT(PPC_RAW_LBZ(dst_reg, src_reg, off));
if (insn_is_zext(&insn[i + 1]))
addrs[++i] = ctx->idx * 4;
break;
/* dst = *(u16 *)(ul) (src + off) */
case BPF_LDX | BPF_MEM | BPF_H:
- PPC_LHZ(dst_reg, src_reg, off);
+ EMIT(PPC_RAW_LHZ(dst_reg, src_reg, off));
if (insn_is_zext(&insn[i + 1]))
addrs[++i] = ctx->idx * 4;
break;
/* dst = *(u32 *)(ul) (src + off) */
case BPF_LDX | BPF_MEM | BPF_W:
- PPC_LWZ(dst_reg, src_reg, off);
+ EMIT(PPC_RAW_LWZ(dst_reg, src_reg, off));
if (insn_is_zext(&insn[i + 1]))
addrs[++i] = ctx->idx * 4;
break;
@@ -775,7 +777,7 @@
else
bpf_jit_emit_func_call_rel(image, ctx, func_addr);
/* move return value from r3 to BPF_REG_0 */
- PPC_MR(b2p[BPF_REG_0], 3);
+ EMIT(PPC_RAW_MR(b2p[BPF_REG_0], 3));
break;
/*
@@ -860,9 +862,9 @@
case BPF_JMP32 | BPF_JNE | BPF_X:
/* unsigned comparison */
if (BPF_CLASS(code) == BPF_JMP32)
- PPC_CMPLW(dst_reg, src_reg);
+ EMIT(PPC_RAW_CMPLW(dst_reg, src_reg));
else
- PPC_CMPLD(dst_reg, src_reg);
+ EMIT(PPC_RAW_CMPLD(dst_reg, src_reg));
break;
case BPF_JMP | BPF_JSGT | BPF_X:
case BPF_JMP | BPF_JSLT | BPF_X:
@@ -874,21 +876,21 @@
case BPF_JMP32 | BPF_JSLE | BPF_X:
/* signed comparison */
if (BPF_CLASS(code) == BPF_JMP32)
- PPC_CMPW(dst_reg, src_reg);
+ EMIT(PPC_RAW_CMPW(dst_reg, src_reg));
else
- PPC_CMPD(dst_reg, src_reg);
+ EMIT(PPC_RAW_CMPD(dst_reg, src_reg));
break;
case BPF_JMP | BPF_JSET | BPF_X:
case BPF_JMP32 | BPF_JSET | BPF_X:
if (BPF_CLASS(code) == BPF_JMP) {
- PPC_AND_DOT(b2p[TMP_REG_1], dst_reg,
- src_reg);
+ EMIT(PPC_RAW_AND_DOT(b2p[TMP_REG_1], dst_reg,
+ src_reg));
} else {
int tmp_reg = b2p[TMP_REG_1];
- PPC_AND(tmp_reg, dst_reg, src_reg);
- PPC_RLWINM_DOT(tmp_reg, tmp_reg, 0, 0,
- 31);
+ EMIT(PPC_RAW_AND(tmp_reg, dst_reg, src_reg));
+ EMIT(PPC_RAW_RLWINM_DOT(tmp_reg, tmp_reg, 0, 0,
+ 31));
}
break;
case BPF_JMP | BPF_JNE | BPF_K:
@@ -912,19 +914,19 @@
*/
if (imm >= 0 && imm < 32768) {
if (is_jmp32)
- PPC_CMPLWI(dst_reg, imm);
+ EMIT(PPC_RAW_CMPLWI(dst_reg, imm));
else
- PPC_CMPLDI(dst_reg, imm);
+ EMIT(PPC_RAW_CMPLDI(dst_reg, imm));
} else {
/* sign-extending load */
PPC_LI32(b2p[TMP_REG_1], imm);
/* ... but unsigned comparison */
if (is_jmp32)
- PPC_CMPLW(dst_reg,
- b2p[TMP_REG_1]);
+ EMIT(PPC_RAW_CMPLW(dst_reg,
+ b2p[TMP_REG_1]));
else
- PPC_CMPLD(dst_reg,
- b2p[TMP_REG_1]);
+ EMIT(PPC_RAW_CMPLD(dst_reg,
+ b2p[TMP_REG_1]));
}
break;
}
@@ -945,17 +947,17 @@
*/
if (imm >= -32768 && imm < 32768) {
if (is_jmp32)
- PPC_CMPWI(dst_reg, imm);
+ EMIT(PPC_RAW_CMPWI(dst_reg, imm));
else
- PPC_CMPDI(dst_reg, imm);
+ EMIT(PPC_RAW_CMPDI(dst_reg, imm));
} else {
PPC_LI32(b2p[TMP_REG_1], imm);
if (is_jmp32)
- PPC_CMPW(dst_reg,
- b2p[TMP_REG_1]);
+ EMIT(PPC_RAW_CMPW(dst_reg,
+ b2p[TMP_REG_1]));
else
- PPC_CMPD(dst_reg,
- b2p[TMP_REG_1]);
+ EMIT(PPC_RAW_CMPD(dst_reg,
+ b2p[TMP_REG_1]));
}
break;
}
@@ -964,19 +966,19 @@
/* andi does not sign-extend the immediate */
if (imm >= 0 && imm < 32768)
/* PPC_ANDI is _only/always_ dot-form */
- PPC_ANDI(b2p[TMP_REG_1], dst_reg, imm);
+ EMIT(PPC_RAW_ANDI(b2p[TMP_REG_1], dst_reg, imm));
else {
int tmp_reg = b2p[TMP_REG_1];
PPC_LI32(tmp_reg, imm);
if (BPF_CLASS(code) == BPF_JMP) {
- PPC_AND_DOT(tmp_reg, dst_reg,
- tmp_reg);
+ EMIT(PPC_RAW_AND_DOT(tmp_reg, dst_reg,
+ tmp_reg));
} else {
- PPC_AND(tmp_reg, dst_reg,
- tmp_reg);
- PPC_RLWINM_DOT(tmp_reg, tmp_reg,
- 0, 0, 31);
+ EMIT(PPC_RAW_AND(tmp_reg, dst_reg,
+ tmp_reg));
+ EMIT(PPC_RAW_RLWINM_DOT(tmp_reg, tmp_reg,
+ 0, 0, 31));
}
}
break;
diff --git a/arch/powerpc/perf/Makefile b/arch/powerpc/perf/Makefile
index 53d614e..c02854d 100644
--- a/arch/powerpc/perf/Makefile
+++ b/arch/powerpc/perf/Makefile
@@ -9,7 +9,7 @@
obj64-$(CONFIG_PPC_PERF_CTRS) += ppc970-pmu.o power5-pmu.o \
power5+-pmu.o power6-pmu.o power7-pmu.o \
isa207-common.o power8-pmu.o power9-pmu.o \
- generic-compat-pmu.o
+ generic-compat-pmu.o power10-pmu.o
obj32-$(CONFIG_PPC_PERF_CTRS) += mpc7450-pmu.o
obj-$(CONFIG_PPC_POWERNV) += imc-pmu.o
diff --git a/arch/powerpc/perf/callchain.h b/arch/powerpc/perf/callchain.h
index 7a2cb9e..ae24d4a 100644
--- a/arch/powerpc/perf/callchain.h
+++ b/arch/powerpc/perf/callchain.h
@@ -2,7 +2,7 @@
#ifndef _POWERPC_PERF_CALLCHAIN_H
#define _POWERPC_PERF_CALLCHAIN_H
-int read_user_stack_slow(void __user *ptr, void *buf, int nb);
+int read_user_stack_slow(const void __user *ptr, void *buf, int nb);
void perf_callchain_user_64(struct perf_callchain_entry_ctx *entry,
struct pt_regs *regs);
void perf_callchain_user_32(struct perf_callchain_entry_ctx *entry,
@@ -16,4 +16,27 @@
return (!sp || (sp & mask) || (sp > top));
}
+/*
+ * On 32-bit we just access the address and let hash_page create a
+ * HPTE if necessary, so there is no need to fall back to reading
+ * the page tables. Since this is called at interrupt level,
+ * do_page_fault() won't treat a DSI as a page fault.
+ */
+static inline int __read_user_stack(const void __user *ptr, void *ret,
+ size_t size)
+{
+ unsigned long addr = (unsigned long)ptr;
+ int rc;
+
+ if (addr > TASK_SIZE - size || (addr & (size - 1)))
+ return -EFAULT;
+
+ rc = copy_from_user_nofault(ret, ptr, size);
+
+ if (IS_ENABLED(CONFIG_PPC64) && rc)
+ return read_user_stack_slow(ptr, ret, size);
+
+ return rc;
+}
+
#endif /* _POWERPC_PERF_CALLCHAIN_H */
diff --git a/arch/powerpc/perf/callchain_32.c b/arch/powerpc/perf/callchain_32.c
index 542e68b..64e4013 100644
--- a/arch/powerpc/perf/callchain_32.c
+++ b/arch/powerpc/perf/callchain_32.c
@@ -30,26 +30,9 @@
#endif /* CONFIG_PPC64 */
-/*
- * On 32-bit we just access the address and let hash_page create a
- * HPTE if necessary, so there is no need to fall back to reading
- * the page tables. Since this is called at interrupt level,
- * do_page_fault() won't treat a DSI as a page fault.
- */
-static int read_user_stack_32(unsigned int __user *ptr, unsigned int *ret)
+static int read_user_stack_32(const unsigned int __user *ptr, unsigned int *ret)
{
- int rc;
-
- if ((unsigned long)ptr > TASK_SIZE - sizeof(unsigned int) ||
- ((unsigned long)ptr & 3))
- return -EFAULT;
-
- rc = copy_from_user_nofault(ret, ptr, sizeof(*ret));
-
- if (IS_ENABLED(CONFIG_PPC64) && rc)
- return read_user_stack_slow(ptr, ret, 4);
-
- return rc;
+ return __read_user_stack(ptr, ret, sizeof(*ret));
}
/*
diff --git a/arch/powerpc/perf/callchain_64.c b/arch/powerpc/perf/callchain_64.c
index fa2a1b8..fed90e8 100644
--- a/arch/powerpc/perf/callchain_64.c
+++ b/arch/powerpc/perf/callchain_64.c
@@ -23,7 +23,7 @@
* interrupt context, so if the access faults, we read the page tables
* to find which page (if any) is mapped and access it directly.
*/
-int read_user_stack_slow(void __user *ptr, void *buf, int nb)
+int read_user_stack_slow(const void __user *ptr, void *buf, int nb)
{
unsigned long addr = (unsigned long) ptr;
@@ -44,16 +44,9 @@
return -EFAULT;
}
-static int read_user_stack_64(unsigned long __user *ptr, unsigned long *ret)
+static int read_user_stack_64(const unsigned long __user *ptr, unsigned long *ret)
{
- if ((unsigned long)ptr > TASK_SIZE - sizeof(unsigned long) ||
- ((unsigned long)ptr & 7))
- return -EFAULT;
-
- if (!copy_from_user_nofault(ret, ptr, sizeof(*ret)))
- return 0;
-
- return read_user_stack_slow(ptr, ret, 8);
+ return __read_user_stack(ptr, ret, sizeof(*ret));
}
/*
diff --git a/arch/powerpc/perf/core-book3s.c b/arch/powerpc/perf/core-book3s.c
index 01d7028..78fe349 100644
--- a/arch/powerpc/perf/core-book3s.c
+++ b/arch/powerpc/perf/core-book3s.c
@@ -37,12 +37,7 @@
struct perf_event *event[MAX_HWEVENTS];
u64 events[MAX_HWEVENTS];
unsigned int flags[MAX_HWEVENTS];
- /*
- * The order of the MMCR array is:
- * - 64-bit, MMCR0, MMCR1, MMCRA, MMCR2
- * - 32-bit, MMCR0, MMCR1, MMCR2
- */
- unsigned long mmcr[4];
+ struct mmcr_regs mmcr;
struct perf_event *limited_counter[MAX_LIMITED_HWCOUNTERS];
u8 limited_hwidx[MAX_LIMITED_HWCOUNTERS];
u64 alternatives[MAX_HWEVENTS][MAX_EVENT_ALTERNATIVES];
@@ -77,6 +72,11 @@
/*
* 32-bit doesn't have MMCRA but does have an MMCR2,
* and a few other names are different.
+ * Also 32-bit doesn't have MMCR3, SIER2 and SIER3.
+ * Define them as zero knowing that any code path accessing
+ * these registers (via mtspr/mfspr) are done under ppmu flag
+ * check for PPMU_ARCH_31 and we will not enter that code path
+ * for 32-bit.
*/
#ifdef CONFIG_PPC32
@@ -90,7 +90,11 @@
#define MMCR0_PMCC_U6 0
#define SPRN_MMCRA SPRN_MMCR2
+#define SPRN_MMCR3 0
+#define SPRN_SIER2 0
+#define SPRN_SIER3 0
#define MMCRA_SAMPLE_ENABLE 0
+#define MMCRA_BHRB_DISABLE 0
static inline unsigned long perf_ip_adjust(struct pt_regs *regs)
{
@@ -121,7 +125,7 @@
static void ebb_switch_out(unsigned long mmcr0) { }
static unsigned long ebb_switch_in(bool ebb, struct cpu_hw_events *cpuhw)
{
- return cpuhw->mmcr[0];
+ return cpuhw->mmcr.mmcr0;
}
static inline void power_pmu_bhrb_enable(struct perf_event *event) {}
@@ -466,8 +470,11 @@
* addresses at this point. Check the privileges before
* exporting it to userspace (avoid exposure of regions
* where we could have speculative execution)
+ * Incase of ISA v3.1, BHRB will capture only user-space
+ * addresses, hence include a check before filtering code
*/
- if (is_kernel_addr(addr) && perf_allow_kernel(&event->attr) != 0)
+ if (!(ppmu->flags & PPMU_ARCH_31) &&
+ is_kernel_addr(addr) && perf_allow_kernel(&event->attr) != 0)
continue;
/* Branches are read most recent first (ie. mfbhrb 0 is
@@ -586,11 +593,16 @@
current->thread.sdar = mfspr(SPRN_SDAR);
current->thread.mmcr0 = mmcr0 & MMCR0_USER_MASK;
current->thread.mmcr2 = mfspr(SPRN_MMCR2) & MMCR2_USER_MASK;
+ if (ppmu->flags & PPMU_ARCH_31) {
+ current->thread.mmcr3 = mfspr(SPRN_MMCR3);
+ current->thread.sier2 = mfspr(SPRN_SIER2);
+ current->thread.sier3 = mfspr(SPRN_SIER3);
+ }
}
static unsigned long ebb_switch_in(bool ebb, struct cpu_hw_events *cpuhw)
{
- unsigned long mmcr0 = cpuhw->mmcr[0];
+ unsigned long mmcr0 = cpuhw->mmcr.mmcr0;
if (!ebb)
goto out;
@@ -624,7 +636,13 @@
* unfreeze counters, it should not set exclude_xxx in its events and
* instead manage the MMCR2 entirely by itself.
*/
- mtspr(SPRN_MMCR2, cpuhw->mmcr[3] | current->thread.mmcr2);
+ mtspr(SPRN_MMCR2, cpuhw->mmcr.mmcr2 | current->thread.mmcr2);
+
+ if (ppmu->flags & PPMU_ARCH_31) {
+ mtspr(SPRN_MMCR3, current->thread.mmcr3);
+ mtspr(SPRN_SIER2, current->thread.sier2);
+ mtspr(SPRN_SIER3, current->thread.sier3);
+ }
out:
return mmcr0;
}
@@ -845,6 +863,11 @@
pr_info("EBBRR: %016lx BESCR: %016lx\n",
mfspr(SPRN_EBBRR), mfspr(SPRN_BESCR));
}
+
+ if (ppmu->flags & PPMU_ARCH_31) {
+ pr_info("MMCR3: %016lx SIER2: %016lx SIER3: %016lx\n",
+ mfspr(SPRN_MMCR3), mfspr(SPRN_SIER2), mfspr(SPRN_SIER3));
+ }
#endif
pr_info("SIAR: %016lx SDAR: %016lx SIER: %016lx\n",
mfspr(SPRN_SIAR), sdar, sier);
@@ -1196,7 +1219,7 @@
static void power_pmu_disable(struct pmu *pmu)
{
struct cpu_hw_events *cpuhw;
- unsigned long flags, mmcr0, val;
+ unsigned long flags, mmcr0, val, mmcra;
if (!ppmu)
return;
@@ -1229,12 +1252,24 @@
mb();
isync();
+ val = mmcra = cpuhw->mmcr.mmcra;
+
/*
* Disable instruction sampling if it was enabled
*/
- if (cpuhw->mmcr[2] & MMCRA_SAMPLE_ENABLE) {
- mtspr(SPRN_MMCRA,
- cpuhw->mmcr[2] & ~MMCRA_SAMPLE_ENABLE);
+ if (cpuhw->mmcr.mmcra & MMCRA_SAMPLE_ENABLE)
+ val &= ~MMCRA_SAMPLE_ENABLE;
+
+ /* Disable BHRB via mmcra (BHRBRD) for p10 */
+ if (ppmu->flags & PPMU_ARCH_31)
+ val |= MMCRA_BHRB_DISABLE;
+
+ /*
+ * Write SPRN_MMCRA if mmcra has either disabled
+ * instruction sampling or BHRB.
+ */
+ if (val != mmcra) {
+ mtspr(SPRN_MMCRA, mmcra);
mb();
isync();
}
@@ -1308,18 +1343,20 @@
* (possibly updated for removal of events).
*/
if (!cpuhw->n_added) {
- mtspr(SPRN_MMCRA, cpuhw->mmcr[2] & ~MMCRA_SAMPLE_ENABLE);
- mtspr(SPRN_MMCR1, cpuhw->mmcr[1]);
+ mtspr(SPRN_MMCRA, cpuhw->mmcr.mmcra & ~MMCRA_SAMPLE_ENABLE);
+ mtspr(SPRN_MMCR1, cpuhw->mmcr.mmcr1);
+ if (ppmu->flags & PPMU_ARCH_31)
+ mtspr(SPRN_MMCR3, cpuhw->mmcr.mmcr3);
goto out_enable;
}
/*
* Clear all MMCR settings and recompute them for the new set of events.
*/
- memset(cpuhw->mmcr, 0, sizeof(cpuhw->mmcr));
+ memset(&cpuhw->mmcr, 0, sizeof(cpuhw->mmcr));
if (ppmu->compute_mmcr(cpuhw->events, cpuhw->n_events, hwc_index,
- cpuhw->mmcr, cpuhw->event)) {
+ &cpuhw->mmcr, cpuhw->event)) {
/* shouldn't ever get here */
printk(KERN_ERR "oops compute_mmcr failed\n");
goto out;
@@ -1333,11 +1370,11 @@
*/
event = cpuhw->event[0];
if (event->attr.exclude_user)
- cpuhw->mmcr[0] |= MMCR0_FCP;
+ cpuhw->mmcr.mmcr0 |= MMCR0_FCP;
if (event->attr.exclude_kernel)
- cpuhw->mmcr[0] |= freeze_events_kernel;
+ cpuhw->mmcr.mmcr0 |= freeze_events_kernel;
if (event->attr.exclude_hv)
- cpuhw->mmcr[0] |= MMCR0_FCHV;
+ cpuhw->mmcr.mmcr0 |= MMCR0_FCHV;
}
/*
@@ -1346,12 +1383,15 @@
* Then unfreeze the events.
*/
ppc_set_pmu_inuse(1);
- mtspr(SPRN_MMCRA, cpuhw->mmcr[2] & ~MMCRA_SAMPLE_ENABLE);
- mtspr(SPRN_MMCR1, cpuhw->mmcr[1]);
- mtspr(SPRN_MMCR0, (cpuhw->mmcr[0] & ~(MMCR0_PMC1CE | MMCR0_PMCjCE))
+ mtspr(SPRN_MMCRA, cpuhw->mmcr.mmcra & ~MMCRA_SAMPLE_ENABLE);
+ mtspr(SPRN_MMCR1, cpuhw->mmcr.mmcr1);
+ mtspr(SPRN_MMCR0, (cpuhw->mmcr.mmcr0 & ~(MMCR0_PMC1CE | MMCR0_PMCjCE))
| MMCR0_FC);
if (ppmu->flags & PPMU_ARCH_207S)
- mtspr(SPRN_MMCR2, cpuhw->mmcr[3]);
+ mtspr(SPRN_MMCR2, cpuhw->mmcr.mmcr2);
+
+ if (ppmu->flags & PPMU_ARCH_31)
+ mtspr(SPRN_MMCR3, cpuhw->mmcr.mmcr3);
/*
* Read off any pre-existing events that need to move
@@ -1402,7 +1442,7 @@
perf_event_update_userpage(event);
}
cpuhw->n_limited = n_lim;
- cpuhw->mmcr[0] |= MMCR0_PMXE | MMCR0_FCECE;
+ cpuhw->mmcr.mmcr0 |= MMCR0_PMXE | MMCR0_FCECE;
out_enable:
pmao_restore_workaround(ebb);
@@ -1418,9 +1458,9 @@
/*
* Enable instruction sampling if necessary
*/
- if (cpuhw->mmcr[2] & MMCRA_SAMPLE_ENABLE) {
+ if (cpuhw->mmcr.mmcra & MMCRA_SAMPLE_ENABLE) {
mb();
- mtspr(SPRN_MMCRA, cpuhw->mmcr[2]);
+ mtspr(SPRN_MMCRA, cpuhw->mmcr.mmcra);
}
out:
@@ -1550,7 +1590,7 @@
cpuhw->flags[i-1] = cpuhw->flags[i];
}
--cpuhw->n_events;
- ppmu->disable_pmc(event->hw.idx - 1, cpuhw->mmcr);
+ ppmu->disable_pmc(event->hw.idx - 1, &cpuhw->mmcr);
if (event->hw.idx) {
write_pmc(event->hw.idx, 0);
event->hw.idx = 0;
@@ -1571,7 +1611,7 @@
}
if (cpuhw->n_events == 0) {
/* disable exceptions if no events are running */
- cpuhw->mmcr[0] &= ~(MMCR0_PMXE | MMCR0_FCECE);
+ cpuhw->mmcr.mmcr0 &= ~(MMCR0_PMXE | MMCR0_FCECE);
}
if (has_branch_stack(event))
@@ -1795,7 +1835,7 @@
static int hw_perf_cache_event(u64 config, u64 *eventp)
{
unsigned long type, op, result;
- int ev;
+ u64 ev;
if (!ppmu->cache_events)
return -EINVAL;
@@ -2246,7 +2286,7 @@
* XXX might want to use MSR.PM to keep the events frozen until
* we get back out of this interrupt.
*/
- write_mmcr0(cpuhw, cpuhw->mmcr[0]);
+ write_mmcr0(cpuhw, cpuhw->mmcr.mmcr0);
if (nmi)
nmi_exit();
@@ -2268,7 +2308,7 @@
if (ppmu) {
memset(cpuhw, 0, sizeof(*cpuhw));
- cpuhw->mmcr[0] = MMCR0_FC;
+ cpuhw->mmcr.mmcr0 = MMCR0_FC;
}
return 0;
}
@@ -2314,6 +2354,8 @@
return 0;
else if (!init_power9_pmu())
return 0;
+ else if (!init_power10_pmu())
+ return 0;
else if (!init_ppc970_pmu())
return 0;
else
diff --git a/arch/powerpc/perf/generic-compat-pmu.c b/arch/powerpc/perf/generic-compat-pmu.c
index 5e5a54d..eb8a6aaf 100644
--- a/arch/powerpc/perf/generic-compat-pmu.c
+++ b/arch/powerpc/perf/generic-compat-pmu.c
@@ -101,7 +101,7 @@
* 0 means not supported, -1 means nonsensical, other values
* are event codes.
*/
-static int generic_compat_cache_events[C(MAX)][C(OP_MAX)][C(RESULT_MAX)] = {
+static u64 generic_compat_cache_events[C(MAX)][C(OP_MAX)][C(RESULT_MAX)] = {
[ C(L1D) ] = {
[ C(OP_READ) ] = {
[ C(RESULT_ACCESS) ] = 0,
diff --git a/arch/powerpc/perf/hv-24x7.c b/arch/powerpc/perf/hv-24x7.c
index db213eb..cdb7bfb 100644
--- a/arch/powerpc/perf/hv-24x7.c
+++ b/arch/powerpc/perf/hv-24x7.c
@@ -31,6 +31,8 @@
/* Whether we have to aggregate result data for some domains. */
static bool aggregate_result_elements;
+static cpumask_t hv_24x7_cpumask;
+
static bool domain_is_valid(unsigned domain)
{
switch (domain) {
@@ -446,6 +448,12 @@
return sprintf(buf, "%s\n", (char *)d->var);
}
+static ssize_t cpumask_show(struct device *dev,
+ struct device_attribute *attr, char *buf)
+{
+ return cpumap_print_to_pagebuf(true, buf, &hv_24x7_cpumask);
+}
+
static ssize_t sockets_show(struct device *dev,
struct device_attribute *attr, char *buf)
{
@@ -1113,6 +1121,7 @@
static DEVICE_ATTR_RO(sockets);
static DEVICE_ATTR_RO(chipspersocket);
static DEVICE_ATTR_RO(coresperchip);
+static DEVICE_ATTR_RO(cpumask);
static struct bin_attribute *if_bin_attrs[] = {
&bin_attr_catalog,
@@ -1126,6 +1135,7 @@
&dev_attr_sockets.attr,
&dev_attr_chipspersocket.attr,
&dev_attr_coresperchip.attr,
+ &dev_attr_cpumask.attr,
NULL,
};
@@ -1641,6 +1651,45 @@
.capabilities = PERF_PMU_CAP_NO_EXCLUDE,
};
+static int ppc_hv_24x7_cpu_online(unsigned int cpu)
+{
+ if (cpumask_empty(&hv_24x7_cpumask))
+ cpumask_set_cpu(cpu, &hv_24x7_cpumask);
+
+ return 0;
+}
+
+static int ppc_hv_24x7_cpu_offline(unsigned int cpu)
+{
+ int target;
+
+ /* Check if exiting cpu is used for collecting 24x7 events */
+ if (!cpumask_test_and_clear_cpu(cpu, &hv_24x7_cpumask))
+ return 0;
+
+ /* Find a new cpu to collect 24x7 events */
+ target = cpumask_last(cpu_active_mask);
+
+ if (target < 0 || target >= nr_cpu_ids) {
+ pr_err("hv_24x7: CPU hotplug init failed\n");
+ return -1;
+ }
+
+ /* Migrate 24x7 events to the new target */
+ cpumask_set_cpu(target, &hv_24x7_cpumask);
+ perf_pmu_migrate_context(&h_24x7_pmu, cpu, target);
+
+ return 0;
+}
+
+static int hv_24x7_cpu_hotplug_init(void)
+{
+ return cpuhp_setup_state(CPUHP_AP_PERF_POWERPC_HV_24x7_ONLINE,
+ "perf/powerpc/hv_24x7:online",
+ ppc_hv_24x7_cpu_online,
+ ppc_hv_24x7_cpu_offline);
+}
+
static int hv_24x7_init(void)
{
int r;
@@ -1685,6 +1734,11 @@
if (r)
return r;
+ /* init cpuhotplug */
+ r = hv_24x7_cpu_hotplug_init();
+ if (r)
+ return r;
+
r = perf_pmu_register(&h_24x7_pmu, h_24x7_pmu.name, -1);
if (r)
return r;
diff --git a/arch/powerpc/perf/imc-pmu.c b/arch/powerpc/perf/imc-pmu.c
index 0edcfd0..a45d694 100644
--- a/arch/powerpc/perf/imc-pmu.c
+++ b/arch/powerpc/perf/imc-pmu.c
@@ -1288,11 +1288,30 @@
header->size = sizeof(*header) + event->header_size;
header->misc = 0;
- if (is_kernel_addr(data->ip))
- header->misc |= PERF_RECORD_MISC_KERNEL;
- else
- header->misc |= PERF_RECORD_MISC_USER;
-
+ if (cpu_has_feature(CPU_FTR_ARCH_31)) {
+ switch (IMC_TRACE_RECORD_VAL_HVPR(mem->val)) {
+ case 0:/* when MSR HV and PR not set in the trace-record */
+ header->misc |= PERF_RECORD_MISC_GUEST_KERNEL;
+ break;
+ case 1: /* MSR HV is 0 and PR is 1 */
+ header->misc |= PERF_RECORD_MISC_GUEST_USER;
+ break;
+ case 2: /* MSR HV is 1 and PR is 0 */
+ header->misc |= PERF_RECORD_MISC_HYPERVISOR;
+ break;
+ case 3: /* MSR HV is 1 and PR is 1 */
+ header->misc |= PERF_RECORD_MISC_USER;
+ break;
+ default:
+ pr_info("IMC: Unable to set the flag based on MSR bits\n");
+ break;
+ }
+ } else {
+ if (is_kernel_addr(data->ip))
+ header->misc |= PERF_RECORD_MISC_KERNEL;
+ else
+ header->misc |= PERF_RECORD_MISC_USER;
+ }
perf_event_header__init_id(header, data, event);
return 0;
diff --git a/arch/powerpc/perf/internal.h b/arch/powerpc/perf/internal.h
index f755c64..80bbf72 100644
--- a/arch/powerpc/perf/internal.h
+++ b/arch/powerpc/perf/internal.h
@@ -9,4 +9,5 @@
extern int init_power7_pmu(void);
extern int init_power8_pmu(void);
extern int init_power9_pmu(void);
+extern int init_power10_pmu(void);
extern int init_generic_compat_pmu(void);
diff --git a/arch/powerpc/perf/isa207-common.c b/arch/powerpc/perf/isa207-common.c
index 4c86da5..964437a 100644
--- a/arch/powerpc/perf/isa207-common.c
+++ b/arch/powerpc/perf/isa207-common.c
@@ -55,7 +55,9 @@
{
u64 valid_mask = EVENT_VALID_MASK;
- if (cpu_has_feature(CPU_FTR_ARCH_300))
+ if (cpu_has_feature(CPU_FTR_ARCH_31))
+ valid_mask = p10_EVENT_VALID_MASK;
+ else if (cpu_has_feature(CPU_FTR_ARCH_300))
valid_mask = p9_EVENT_VALID_MASK;
return !(event & ~valid_mask);
@@ -69,6 +71,14 @@
return false;
}
+static unsigned long sdar_mod_val(u64 event)
+{
+ if (cpu_has_feature(CPU_FTR_ARCH_31))
+ return p10_SDAR_MODE(event);
+
+ return p9_SDAR_MODE(event);
+}
+
static void mmcra_sdar_mode(u64 event, unsigned long *mmcra)
{
/*
@@ -79,7 +89,7 @@
* MMCRA[SDAR_MODE] will be programmed as "0b01" for continous sampling
* mode and will be un-changed when setting MMCRA[63] (Marked events).
*
- * Incase of Power9:
+ * Incase of Power9/power10:
* Marked event: MMCRA[SDAR_MODE] will be set to 0b00 ('No Updates'),
* or if group already have any marked events.
* For rest
@@ -90,8 +100,8 @@
if (cpu_has_feature(CPU_FTR_ARCH_300)) {
if (is_event_marked(event) || (*mmcra & MMCRA_SAMPLE_ENABLE))
*mmcra &= MMCRA_SDAR_MODE_NO_UPDATES;
- else if (p9_SDAR_MODE(event))
- *mmcra |= p9_SDAR_MODE(event) << MMCRA_SDAR_MODE_SHIFT;
+ else if (sdar_mod_val(event))
+ *mmcra |= sdar_mod_val(event) << MMCRA_SDAR_MODE_SHIFT;
else
*mmcra |= MMCRA_SDAR_MODE_DCACHE;
} else
@@ -134,7 +144,11 @@
/*
* Check the mantissa upper two bits are not zero, unless the
* exponent is also zero. See the THRESH_CMP_MANTISSA doc.
+ * Power10: thresh_cmp is replaced by l2_l3 event select.
*/
+ if (cpu_has_feature(CPU_FTR_ARCH_31))
+ return false;
+
cmp = (event >> EVENT_THR_CMP_SHIFT) & EVENT_THR_CMP_MASK;
exp = cmp >> 7;
@@ -251,7 +265,12 @@
pmc = (event >> EVENT_PMC_SHIFT) & EVENT_PMC_MASK;
unit = (event >> EVENT_UNIT_SHIFT) & EVENT_UNIT_MASK;
- cache = (event >> EVENT_CACHE_SEL_SHIFT) & EVENT_CACHE_SEL_MASK;
+ if (cpu_has_feature(CPU_FTR_ARCH_31))
+ cache = (event >> EVENT_CACHE_SEL_SHIFT) &
+ p10_EVENT_CACHE_SEL_MASK;
+ else
+ cache = (event >> EVENT_CACHE_SEL_SHIFT) &
+ EVENT_CACHE_SEL_MASK;
ebb = (event >> EVENT_EBB_SHIFT) & EVENT_EBB_MASK;
if (pmc) {
@@ -283,7 +302,10 @@
}
if (unit >= 6 && unit <= 9) {
- if (cpu_has_feature(CPU_FTR_ARCH_300)) {
+ if (cpu_has_feature(CPU_FTR_ARCH_31) && (unit == 6)) {
+ mask |= CNST_L2L3_GROUP_MASK;
+ value |= CNST_L2L3_GROUP_VAL(event >> p10_L2L3_EVENT_SHIFT);
+ } else if (cpu_has_feature(CPU_FTR_ARCH_300)) {
mask |= CNST_CACHE_GROUP_MASK;
value |= CNST_CACHE_GROUP_VAL(event & 0xff);
@@ -363,10 +385,11 @@
}
int isa207_compute_mmcr(u64 event[], int n_ev,
- unsigned int hwc[], unsigned long mmcr[],
+ unsigned int hwc[], struct mmcr_regs *mmcr,
struct perf_event *pevents[])
{
unsigned long mmcra, mmcr1, mmcr2, unit, combine, psel, cache, val;
+ unsigned long mmcr3;
unsigned int pmc, pmc_inuse;
int i;
@@ -379,7 +402,14 @@
pmc_inuse |= 1 << pmc;
}
- mmcra = mmcr1 = mmcr2 = 0;
+ mmcra = mmcr1 = mmcr2 = mmcr3 = 0;
+
+ /*
+ * Disable bhrb unless explicitly requested
+ * by setting MMCRA (BHRBRD) bit.
+ */
+ if (cpu_has_feature(CPU_FTR_ARCH_31))
+ mmcra |= MMCRA_BHRB_DISABLE;
/* Second pass: assign PMCs, set all MMCR1 fields */
for (i = 0; i < n_ev; ++i) {
@@ -438,8 +468,17 @@
mmcra |= val << MMCRA_THR_CTL_SHIFT;
val = (event[i] >> EVENT_THR_SEL_SHIFT) & EVENT_THR_SEL_MASK;
mmcra |= val << MMCRA_THR_SEL_SHIFT;
- val = (event[i] >> EVENT_THR_CMP_SHIFT) & EVENT_THR_CMP_MASK;
- mmcra |= thresh_cmp_val(val);
+ if (!cpu_has_feature(CPU_FTR_ARCH_31)) {
+ val = (event[i] >> EVENT_THR_CMP_SHIFT) &
+ EVENT_THR_CMP_MASK;
+ mmcra |= thresh_cmp_val(val);
+ }
+ }
+
+ if (cpu_has_feature(CPU_FTR_ARCH_31) && (unit == 6)) {
+ val = (event[i] >> p10_L2L3_EVENT_SHIFT) &
+ p10_EVENT_L2L3_SEL_MASK;
+ mmcr2 |= val << p10_L2L3_SEL_SHIFT;
}
if (event[i] & EVENT_WANTS_BHRB) {
@@ -447,6 +486,11 @@
mmcra |= val << MMCRA_IFM_SHIFT;
}
+ /* set MMCRA (BHRBRD) to 0 if there is user request for BHRB */
+ if (cpu_has_feature(CPU_FTR_ARCH_31) &&
+ (has_branch_stack(pevents[i]) || (event[i] & EVENT_WANTS_BHRB)))
+ mmcra &= ~MMCRA_BHRB_DISABLE;
+
if (pevents[i]->attr.exclude_user)
mmcr2 |= MMCR2_FCP(pmc);
@@ -460,34 +504,43 @@
mmcr2 |= MMCR2_FCS(pmc);
}
+ if (cpu_has_feature(CPU_FTR_ARCH_31)) {
+ if (pmc <= 4) {
+ val = (event[i] >> p10_EVENT_MMCR3_SHIFT) &
+ p10_EVENT_MMCR3_MASK;
+ mmcr3 |= val << MMCR3_SHIFT(pmc);
+ }
+ }
+
hwc[i] = pmc - 1;
}
/* Return MMCRx values */
- mmcr[0] = 0;
+ mmcr->mmcr0 = 0;
/* pmc_inuse is 1-based */
if (pmc_inuse & 2)
- mmcr[0] = MMCR0_PMC1CE;
+ mmcr->mmcr0 = MMCR0_PMC1CE;
if (pmc_inuse & 0x7c)
- mmcr[0] |= MMCR0_PMCjCE;
+ mmcr->mmcr0 |= MMCR0_PMCjCE;
/* If we're not using PMC 5 or 6, freeze them */
if (!(pmc_inuse & 0x60))
- mmcr[0] |= MMCR0_FC56;
+ mmcr->mmcr0 |= MMCR0_FC56;
- mmcr[1] = mmcr1;
- mmcr[2] = mmcra;
- mmcr[3] = mmcr2;
+ mmcr->mmcr1 = mmcr1;
+ mmcr->mmcra = mmcra;
+ mmcr->mmcr2 = mmcr2;
+ mmcr->mmcr3 = mmcr3;
return 0;
}
-void isa207_disable_pmc(unsigned int pmc, unsigned long mmcr[])
+void isa207_disable_pmc(unsigned int pmc, struct mmcr_regs *mmcr)
{
if (pmc <= 3)
- mmcr[1] &= ~(0xffUL << MMCR1_PMCSEL_SHIFT(pmc + 1));
+ mmcr->mmcr1 &= ~(0xffUL << MMCR1_PMCSEL_SHIFT(pmc + 1));
}
static int find_alternative(u64 event, const unsigned int ev_alt[][MAX_ALT], int size)
diff --git a/arch/powerpc/perf/isa207-common.h b/arch/powerpc/perf/isa207-common.h
index 63fd4f3..044de65 100644
--- a/arch/powerpc/perf/isa207-common.h
+++ b/arch/powerpc/perf/isa207-common.h
@@ -87,6 +87,31 @@
EVENT_LINUX_MASK | \
EVENT_PSEL_MASK))
+/* Contants to support power10 raw encoding format */
+#define p10_SDAR_MODE_SHIFT 22
+#define p10_SDAR_MODE_MASK 0x3ull
+#define p10_SDAR_MODE(v) (((v) >> p10_SDAR_MODE_SHIFT) & \
+ p10_SDAR_MODE_MASK)
+#define p10_EVENT_L2L3_SEL_MASK 0x1f
+#define p10_L2L3_SEL_SHIFT 3
+#define p10_L2L3_EVENT_SHIFT 40
+#define p10_EVENT_THRESH_MASK 0xffffull
+#define p10_EVENT_CACHE_SEL_MASK 0x3ull
+#define p10_EVENT_MMCR3_MASK 0x7fffull
+#define p10_EVENT_MMCR3_SHIFT 45
+
+#define p10_EVENT_VALID_MASK \
+ ((p10_SDAR_MODE_MASK << p10_SDAR_MODE_SHIFT | \
+ (p10_EVENT_THRESH_MASK << EVENT_THRESH_SHIFT) | \
+ (EVENT_SAMPLE_MASK << EVENT_SAMPLE_SHIFT) | \
+ (p10_EVENT_CACHE_SEL_MASK << EVENT_CACHE_SEL_SHIFT) | \
+ (EVENT_PMC_MASK << EVENT_PMC_SHIFT) | \
+ (EVENT_UNIT_MASK << EVENT_UNIT_SHIFT) | \
+ (p9_EVENT_COMBINE_MASK << p9_EVENT_COMBINE_SHIFT) | \
+ (p10_EVENT_MMCR3_MASK << p10_EVENT_MMCR3_SHIFT) | \
+ (EVENT_MARKED_MASK << EVENT_MARKED_SHIFT) | \
+ EVENT_LINUX_MASK | \
+ EVENT_PSEL_MASK))
/*
* Layout of constraint bits:
*
@@ -135,6 +160,9 @@
#define CNST_CACHE_PMC4_VAL (1ull << 54)
#define CNST_CACHE_PMC4_MASK CNST_CACHE_PMC4_VAL
+#define CNST_L2L3_GROUP_VAL(v) (((v) & 0x1full) << 55)
+#define CNST_L2L3_GROUP_MASK CNST_L2L3_GROUP_VAL(0x1f)
+
/*
* For NC we are counting up to 4 events. This requires three bits, and we need
* the fifth event to overflow and set the 4th bit. To achieve that we bias the
@@ -191,7 +219,7 @@
#define MMCRA_THR_CTR_EXP(v) (((v) >> MMCRA_THR_CTR_EXP_SHIFT) &\
MMCRA_THR_CTR_EXP_MASK)
-/* MMCR1 Threshold Compare bit constant for power9 */
+/* MMCRA Threshold Compare bit constant for power9 */
#define p9_MMCRA_THR_CMP_SHIFT 45
/* Bits in MMCR2 for PowerISA v2.07 */
@@ -202,6 +230,9 @@
#define MAX_ALT 2
#define MAX_PMU_COUNTERS 6
+/* Bits in MMCR3 for PowerISA v3.10 */
+#define MMCR3_SHIFT(pmc) (49 - (15 * ((pmc) - 1)))
+
#define ISA207_SIER_TYPE_SHIFT 15
#define ISA207_SIER_TYPE_MASK (0x7ull << ISA207_SIER_TYPE_SHIFT)
@@ -217,9 +248,9 @@
int isa207_get_constraint(u64 event, unsigned long *maskp, unsigned long *valp);
int isa207_compute_mmcr(u64 event[], int n_ev,
- unsigned int hwc[], unsigned long mmcr[],
+ unsigned int hwc[], struct mmcr_regs *mmcr,
struct perf_event *pevents[]);
-void isa207_disable_pmc(unsigned int pmc, unsigned long mmcr[]);
+void isa207_disable_pmc(unsigned int pmc, struct mmcr_regs *mmcr);
int isa207_get_alternatives(u64 event, u64 alt[], int size, unsigned int flags,
const unsigned int ev_alt[][MAX_ALT]);
void isa207_get_mem_data_src(union perf_mem_data_src *dsrc, u32 flags,
diff --git a/arch/powerpc/perf/mpc7450-pmu.c b/arch/powerpc/perf/mpc7450-pmu.c
index 4d5ef92..1919e9d 100644
--- a/arch/powerpc/perf/mpc7450-pmu.c
+++ b/arch/powerpc/perf/mpc7450-pmu.c
@@ -257,7 +257,7 @@
* Compute MMCR0/1/2 values for a set of events.
*/
static int mpc7450_compute_mmcr(u64 event[], int n_ev, unsigned int hwc[],
- unsigned long mmcr[],
+ struct mmcr_regs *mmcr,
struct perf_event *pevents[])
{
u8 event_index[N_CLASSES][N_COUNTER];
@@ -321,9 +321,16 @@
mmcr0 |= MMCR0_PMCnCE;
/* Return MMCRx values */
- mmcr[0] = mmcr0;
- mmcr[1] = mmcr1;
- mmcr[2] = mmcr2;
+ mmcr->mmcr0 = mmcr0;
+ mmcr->mmcr1 = mmcr1;
+ mmcr->mmcr2 = mmcr2;
+ /*
+ * 32-bit doesn't have an MMCRA and uses SPRN_MMCR2 to define
+ * SPRN_MMCRA. So assign mmcra of cpu_hw_events with `mmcr2`
+ * value to ensure that any write to this SPRN_MMCRA will
+ * use mmcr2 value.
+ */
+ mmcr->mmcra = mmcr2;
return 0;
}
@@ -331,12 +338,12 @@
* Disable counting by a PMC.
* Note that the pmc argument is 0-based here, not 1-based.
*/
-static void mpc7450_disable_pmc(unsigned int pmc, unsigned long mmcr[])
+static void mpc7450_disable_pmc(unsigned int pmc, struct mmcr_regs *mmcr)
{
if (pmc <= 1)
- mmcr[0] &= ~(pmcsel_mask[pmc] << pmcsel_shift[pmc]);
+ mmcr->mmcr0 &= ~(pmcsel_mask[pmc] << pmcsel_shift[pmc]);
else
- mmcr[1] &= ~(pmcsel_mask[pmc] << pmcsel_shift[pmc]);
+ mmcr->mmcr1 &= ~(pmcsel_mask[pmc] << pmcsel_shift[pmc]);
}
static int mpc7450_generic_events[] = {
@@ -354,7 +361,7 @@
* 0 means not supported, -1 means nonsensical, other values
* are event codes.
*/
-static int mpc7450_cache_events[C(MAX)][C(OP_MAX)][C(RESULT_MAX)] = {
+static u64 mpc7450_cache_events[C(MAX)][C(OP_MAX)][C(RESULT_MAX)] = {
[C(L1D)] = { /* RESULT_ACCESS RESULT_MISS */
[C(OP_READ)] = { 0, 0x225 },
[C(OP_WRITE)] = { 0, 0x227 },
diff --git a/arch/powerpc/perf/power10-events-list.h b/arch/powerpc/perf/power10-events-list.h
new file mode 100644
index 0000000..60c1b81
--- /dev/null
+++ b/arch/powerpc/perf/power10-events-list.h
@@ -0,0 +1,70 @@
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+/*
+ * Performance counter support for POWER10 processors.
+ *
+ * Copyright 2020 Madhavan Srinivasan, IBM Corporation.
+ * Copyright 2020 Athira Rajeev, IBM Corporation.
+ */
+
+/*
+ * Power10 event codes.
+ */
+EVENT(PM_RUN_CYC, 0x600f4);
+EVENT(PM_DISP_STALL_CYC, 0x100f8);
+EVENT(PM_EXEC_STALL, 0x30008);
+EVENT(PM_RUN_INST_CMPL, 0x500fa);
+EVENT(PM_BR_CMPL, 0x4d05e);
+EVENT(PM_BR_MPRED_CMPL, 0x400f6);
+
+/* All L1 D cache load references counted at finish, gated by reject */
+EVENT(PM_LD_REF_L1, 0x100fc);
+/* Load Missed L1 */
+EVENT(PM_LD_MISS_L1, 0x3e054);
+/* Store Missed L1 */
+EVENT(PM_ST_MISS_L1, 0x300f0);
+/* L1 cache data prefetches */
+EVENT(PM_LD_PREFETCH_CACHE_LINE_MISS, 0x1002c);
+/* Demand iCache Miss */
+EVENT(PM_L1_ICACHE_MISS, 0x200fc);
+/* Instruction fetches from L1 */
+EVENT(PM_INST_FROM_L1, 0x04080);
+/* Instruction Demand sectors wriittent into IL1 */
+EVENT(PM_INST_FROM_L1MISS, 0x03f00000001c040);
+/* Instruction prefetch written into IL1 */
+EVENT(PM_IC_PREF_REQ, 0x040a0);
+/* The data cache was reloaded from local core's L3 due to a demand load */
+EVENT(PM_DATA_FROM_L3, 0x01340000001c040);
+/* Demand LD - L3 Miss (not L2 hit and not L3 hit) */
+EVENT(PM_DATA_FROM_L3MISS, 0x300fe);
+/* Data PTEG reload */
+EVENT(PM_DTLB_MISS, 0x300fc);
+/* ITLB Reloaded */
+EVENT(PM_ITLB_MISS, 0x400fc);
+
+EVENT(PM_RUN_CYC_ALT, 0x0001e);
+EVENT(PM_RUN_INST_CMPL_ALT, 0x00002);
+
+/*
+ * Memory Access Events
+ *
+ * Primary PMU event used here is PM_MRK_INST_CMPL (0x401e0)
+ * To enable capturing of memory profiling, these MMCRA bits
+ * needs to be programmed and corresponding raw event format
+ * encoding.
+ *
+ * MMCRA bits encoding needed are
+ * SM (Sampling Mode)
+ * EM (Eligibility for Random Sampling)
+ * TECE (Threshold Event Counter Event)
+ * TS (Threshold Start Event)
+ * TE (Threshold End Event)
+ *
+ * Corresponding Raw Encoding bits:
+ * sample [EM,SM]
+ * thresh_sel (TECE)
+ * thresh start (TS)
+ * thresh end (TE)
+ */
+
+EVENT(MEM_LOADS, 0x34340401e0);
+EVENT(MEM_STORES, 0x343c0401e0);
diff --git a/arch/powerpc/perf/power10-pmu.c b/arch/powerpc/perf/power10-pmu.c
new file mode 100644
index 0000000..f7cff7f
--- /dev/null
+++ b/arch/powerpc/perf/power10-pmu.c
@@ -0,0 +1,419 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ * Performance counter support for POWER10 processors.
+ *
+ * Copyright 2020 Madhavan Srinivasan, IBM Corporation.
+ * Copyright 2020 Athira Rajeev, IBM Corporation.
+ */
+
+#define pr_fmt(fmt) "power10-pmu: " fmt
+
+#include "isa207-common.h"
+#include "internal.h"
+
+/*
+ * Raw event encoding for Power10:
+ *
+ * 60 56 52 48 44 40 36 32
+ * | - - - - | - - - - | - - - - | - - - - | - - - - | - - - - | - - - - | - - - - |
+ * | | [ ] [ src_match ] [ src_mask ] | [ ] [ l2l3_sel ] [ thresh_ctl ]
+ * | | | | | |
+ * | | *- IFM (Linux) | | thresh start/stop -*
+ * | *- BHRB (Linux) | src_sel
+ * *- EBB (Linux) *invert_bit
+ *
+ * 28 24 20 16 12 8 4 0
+ * | - - - - | - - - - | - - - - | - - - - | - - - - | - - - - | - - - - | - - - - |
+ * [ ] [ sample ] [ ] [ ] [ pmc ] [unit ] [ ] m [ pmcxsel ]
+ * | | | | | |
+ * | | | | | *- mark
+ * | | | *- L1/L2/L3 cache_sel |
+ * | | sdar_mode |
+ * | *- sampling mode for marked events *- combine
+ * |
+ * *- thresh_sel
+ *
+ * Below uses IBM bit numbering.
+ *
+ * MMCR1[x:y] = unit (PMCxUNIT)
+ * MMCR1[24] = pmc1combine[0]
+ * MMCR1[25] = pmc1combine[1]
+ * MMCR1[26] = pmc2combine[0]
+ * MMCR1[27] = pmc2combine[1]
+ * MMCR1[28] = pmc3combine[0]
+ * MMCR1[29] = pmc3combine[1]
+ * MMCR1[30] = pmc4combine[0]
+ * MMCR1[31] = pmc4combine[1]
+ *
+ * if pmc == 3 and unit == 0 and pmcxsel[0:6] == 0b0101011
+ * MMCR1[20:27] = thresh_ctl
+ * else if pmc == 4 and unit == 0xf and pmcxsel[0:6] == 0b0101001
+ * MMCR1[20:27] = thresh_ctl
+ * else
+ * MMCRA[48:55] = thresh_ctl (THRESH START/END)
+ *
+ * if thresh_sel:
+ * MMCRA[45:47] = thresh_sel
+ *
+ * if l2l3_sel:
+ * MMCR2[56:60] = l2l3_sel[0:4]
+ *
+ * MMCR1[16] = cache_sel[0]
+ * MMCR1[17] = cache_sel[1]
+ *
+ * if mark:
+ * MMCRA[63] = 1 (SAMPLE_ENABLE)
+ * MMCRA[57:59] = sample[0:2] (RAND_SAMP_ELIG)
+ * MMCRA[61:62] = sample[3:4] (RAND_SAMP_MODE)
+ *
+ * if EBB and BHRB:
+ * MMCRA[32:33] = IFM
+ *
+ * MMCRA[SDAR_MODE] = sdar_mode[0:1]
+ */
+
+/*
+ * Some power10 event codes.
+ */
+#define EVENT(_name, _code) enum{_name = _code}
+
+#include "power10-events-list.h"
+
+#undef EVENT
+
+/* MMCRA IFM bits - POWER10 */
+#define POWER10_MMCRA_IFM1 0x0000000040000000UL
+#define POWER10_MMCRA_IFM2 0x0000000080000000UL
+#define POWER10_MMCRA_IFM3 0x00000000C0000000UL
+#define POWER10_MMCRA_BHRB_MASK 0x00000000C0000000UL
+
+/* Table of alternatives, sorted by column 0 */
+static const unsigned int power10_event_alternatives[][MAX_ALT] = {
+ { PM_RUN_CYC_ALT, PM_RUN_CYC },
+ { PM_RUN_INST_CMPL_ALT, PM_RUN_INST_CMPL },
+};
+
+static int power10_get_alternatives(u64 event, unsigned int flags, u64 alt[])
+{
+ int num_alt = 0;
+
+ num_alt = isa207_get_alternatives(event, alt,
+ ARRAY_SIZE(power10_event_alternatives), flags,
+ power10_event_alternatives);
+
+ return num_alt;
+}
+
+GENERIC_EVENT_ATTR(cpu-cycles, PM_RUN_CYC);
+GENERIC_EVENT_ATTR(instructions, PM_RUN_INST_CMPL);
+GENERIC_EVENT_ATTR(branch-instructions, PM_BR_CMPL);
+GENERIC_EVENT_ATTR(branch-misses, PM_BR_MPRED_CMPL);
+GENERIC_EVENT_ATTR(cache-references, PM_LD_REF_L1);
+GENERIC_EVENT_ATTR(cache-misses, PM_LD_MISS_L1);
+GENERIC_EVENT_ATTR(mem-loads, MEM_LOADS);
+GENERIC_EVENT_ATTR(mem-stores, MEM_STORES);
+
+CACHE_EVENT_ATTR(L1-dcache-load-misses, PM_LD_MISS_L1);
+CACHE_EVENT_ATTR(L1-dcache-loads, PM_LD_REF_L1);
+CACHE_EVENT_ATTR(L1-dcache-prefetches, PM_LD_PREFETCH_CACHE_LINE_MISS);
+CACHE_EVENT_ATTR(L1-dcache-store-misses, PM_ST_MISS_L1);
+CACHE_EVENT_ATTR(L1-icache-load-misses, PM_L1_ICACHE_MISS);
+CACHE_EVENT_ATTR(L1-icache-loads, PM_INST_FROM_L1);
+CACHE_EVENT_ATTR(L1-icache-prefetches, PM_IC_PREF_REQ);
+CACHE_EVENT_ATTR(LLC-load-misses, PM_DATA_FROM_L3MISS);
+CACHE_EVENT_ATTR(LLC-loads, PM_DATA_FROM_L3);
+CACHE_EVENT_ATTR(branch-load-misses, PM_BR_MPRED_CMPL);
+CACHE_EVENT_ATTR(branch-loads, PM_BR_CMPL);
+CACHE_EVENT_ATTR(dTLB-load-misses, PM_DTLB_MISS);
+CACHE_EVENT_ATTR(iTLB-load-misses, PM_ITLB_MISS);
+
+static struct attribute *power10_events_attr[] = {
+ GENERIC_EVENT_PTR(PM_RUN_CYC),
+ GENERIC_EVENT_PTR(PM_RUN_INST_CMPL),
+ GENERIC_EVENT_PTR(PM_BR_CMPL),
+ GENERIC_EVENT_PTR(PM_BR_MPRED_CMPL),
+ GENERIC_EVENT_PTR(PM_LD_REF_L1),
+ GENERIC_EVENT_PTR(PM_LD_MISS_L1),
+ GENERIC_EVENT_PTR(MEM_LOADS),
+ GENERIC_EVENT_PTR(MEM_STORES),
+ CACHE_EVENT_PTR(PM_LD_MISS_L1),
+ CACHE_EVENT_PTR(PM_LD_REF_L1),
+ CACHE_EVENT_PTR(PM_LD_PREFETCH_CACHE_LINE_MISS),
+ CACHE_EVENT_PTR(PM_ST_MISS_L1),
+ CACHE_EVENT_PTR(PM_L1_ICACHE_MISS),
+ CACHE_EVENT_PTR(PM_INST_FROM_L1),
+ CACHE_EVENT_PTR(PM_IC_PREF_REQ),
+ CACHE_EVENT_PTR(PM_DATA_FROM_L3MISS),
+ CACHE_EVENT_PTR(PM_DATA_FROM_L3),
+ CACHE_EVENT_PTR(PM_BR_MPRED_CMPL),
+ CACHE_EVENT_PTR(PM_BR_CMPL),
+ CACHE_EVENT_PTR(PM_DTLB_MISS),
+ CACHE_EVENT_PTR(PM_ITLB_MISS),
+ NULL
+};
+
+static struct attribute_group power10_pmu_events_group = {
+ .name = "events",
+ .attrs = power10_events_attr,
+};
+
+PMU_FORMAT_ATTR(event, "config:0-59");
+PMU_FORMAT_ATTR(pmcxsel, "config:0-7");
+PMU_FORMAT_ATTR(mark, "config:8");
+PMU_FORMAT_ATTR(combine, "config:10-11");
+PMU_FORMAT_ATTR(unit, "config:12-15");
+PMU_FORMAT_ATTR(pmc, "config:16-19");
+PMU_FORMAT_ATTR(cache_sel, "config:20-21");
+PMU_FORMAT_ATTR(sdar_mode, "config:22-23");
+PMU_FORMAT_ATTR(sample_mode, "config:24-28");
+PMU_FORMAT_ATTR(thresh_sel, "config:29-31");
+PMU_FORMAT_ATTR(thresh_stop, "config:32-35");
+PMU_FORMAT_ATTR(thresh_start, "config:36-39");
+PMU_FORMAT_ATTR(l2l3_sel, "config:40-44");
+PMU_FORMAT_ATTR(src_sel, "config:45-46");
+PMU_FORMAT_ATTR(invert_bit, "config:47");
+PMU_FORMAT_ATTR(src_mask, "config:48-53");
+PMU_FORMAT_ATTR(src_match, "config:54-59");
+
+static struct attribute *power10_pmu_format_attr[] = {
+ &format_attr_event.attr,
+ &format_attr_pmcxsel.attr,
+ &format_attr_mark.attr,
+ &format_attr_combine.attr,
+ &format_attr_unit.attr,
+ &format_attr_pmc.attr,
+ &format_attr_cache_sel.attr,
+ &format_attr_sdar_mode.attr,
+ &format_attr_sample_mode.attr,
+ &format_attr_thresh_sel.attr,
+ &format_attr_thresh_stop.attr,
+ &format_attr_thresh_start.attr,
+ &format_attr_l2l3_sel.attr,
+ &format_attr_src_sel.attr,
+ &format_attr_invert_bit.attr,
+ &format_attr_src_mask.attr,
+ &format_attr_src_match.attr,
+ NULL,
+};
+
+static struct attribute_group power10_pmu_format_group = {
+ .name = "format",
+ .attrs = power10_pmu_format_attr,
+};
+
+static const struct attribute_group *power10_pmu_attr_groups[] = {
+ &power10_pmu_format_group,
+ &power10_pmu_events_group,
+ NULL,
+};
+
+static int power10_generic_events[] = {
+ [PERF_COUNT_HW_CPU_CYCLES] = PM_RUN_CYC,
+ [PERF_COUNT_HW_INSTRUCTIONS] = PM_RUN_INST_CMPL,
+ [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = PM_BR_CMPL,
+ [PERF_COUNT_HW_BRANCH_MISSES] = PM_BR_MPRED_CMPL,
+ [PERF_COUNT_HW_CACHE_REFERENCES] = PM_LD_REF_L1,
+ [PERF_COUNT_HW_CACHE_MISSES] = PM_LD_MISS_L1,
+};
+
+static u64 power10_bhrb_filter_map(u64 branch_sample_type)
+{
+ u64 pmu_bhrb_filter = 0;
+
+ /* BHRB and regular PMU events share the same privilege state
+ * filter configuration. BHRB is always recorded along with a
+ * regular PMU event. As the privilege state filter is handled
+ * in the basic PMC configuration of the accompanying regular
+ * PMU event, we ignore any separate BHRB specific request.
+ */
+
+ /* No branch filter requested */
+ if (branch_sample_type & PERF_SAMPLE_BRANCH_ANY)
+ return pmu_bhrb_filter;
+
+ /* Invalid branch filter options - HW does not support */
+ if (branch_sample_type & PERF_SAMPLE_BRANCH_ANY_RETURN)
+ return -1;
+
+ if (branch_sample_type & PERF_SAMPLE_BRANCH_IND_CALL) {
+ pmu_bhrb_filter |= POWER10_MMCRA_IFM2;
+ return pmu_bhrb_filter;
+ }
+
+ if (branch_sample_type & PERF_SAMPLE_BRANCH_COND) {
+ pmu_bhrb_filter |= POWER10_MMCRA_IFM3;
+ return pmu_bhrb_filter;
+ }
+
+ if (branch_sample_type & PERF_SAMPLE_BRANCH_CALL)
+ return -1;
+
+ if (branch_sample_type & PERF_SAMPLE_BRANCH_ANY_CALL) {
+ pmu_bhrb_filter |= POWER10_MMCRA_IFM1;
+ return pmu_bhrb_filter;
+ }
+
+ /* Every thing else is unsupported */
+ return -1;
+}
+
+static void power10_config_bhrb(u64 pmu_bhrb_filter)
+{
+ pmu_bhrb_filter &= POWER10_MMCRA_BHRB_MASK;
+
+ /* Enable BHRB filter in PMU */
+ mtspr(SPRN_MMCRA, (mfspr(SPRN_MMCRA) | pmu_bhrb_filter));
+}
+
+#define C(x) PERF_COUNT_HW_CACHE_##x
+
+/*
+ * Table of generalized cache-related events.
+ * 0 means not supported, -1 means nonsensical, other values
+ * are event codes.
+ */
+static u64 power10_cache_events[C(MAX)][C(OP_MAX)][C(RESULT_MAX)] = {
+ [C(L1D)] = {
+ [C(OP_READ)] = {
+ [C(RESULT_ACCESS)] = PM_LD_REF_L1,
+ [C(RESULT_MISS)] = PM_LD_MISS_L1,
+ },
+ [C(OP_WRITE)] = {
+ [C(RESULT_ACCESS)] = 0,
+ [C(RESULT_MISS)] = PM_ST_MISS_L1,
+ },
+ [C(OP_PREFETCH)] = {
+ [C(RESULT_ACCESS)] = PM_LD_PREFETCH_CACHE_LINE_MISS,
+ [C(RESULT_MISS)] = 0,
+ },
+ },
+ [C(L1I)] = {
+ [C(OP_READ)] = {
+ [C(RESULT_ACCESS)] = PM_INST_FROM_L1,
+ [C(RESULT_MISS)] = PM_L1_ICACHE_MISS,
+ },
+ [C(OP_WRITE)] = {
+ [C(RESULT_ACCESS)] = PM_INST_FROM_L1MISS,
+ [C(RESULT_MISS)] = -1,
+ },
+ [C(OP_PREFETCH)] = {
+ [C(RESULT_ACCESS)] = PM_IC_PREF_REQ,
+ [C(RESULT_MISS)] = 0,
+ },
+ },
+ [C(LL)] = {
+ [C(OP_READ)] = {
+ [C(RESULT_ACCESS)] = PM_DATA_FROM_L3,
+ [C(RESULT_MISS)] = PM_DATA_FROM_L3MISS,
+ },
+ [C(OP_WRITE)] = {
+ [C(RESULT_ACCESS)] = -1,
+ [C(RESULT_MISS)] = -1,
+ },
+ [C(OP_PREFETCH)] = {
+ [C(RESULT_ACCESS)] = -1,
+ [C(RESULT_MISS)] = 0,
+ },
+ },
+ [C(DTLB)] = {
+ [C(OP_READ)] = {
+ [C(RESULT_ACCESS)] = 0,
+ [C(RESULT_MISS)] = PM_DTLB_MISS,
+ },
+ [C(OP_WRITE)] = {
+ [C(RESULT_ACCESS)] = -1,
+ [C(RESULT_MISS)] = -1,
+ },
+ [C(OP_PREFETCH)] = {
+ [C(RESULT_ACCESS)] = -1,
+ [C(RESULT_MISS)] = -1,
+ },
+ },
+ [C(ITLB)] = {
+ [C(OP_READ)] = {
+ [C(RESULT_ACCESS)] = 0,
+ [C(RESULT_MISS)] = PM_ITLB_MISS,
+ },
+ [C(OP_WRITE)] = {
+ [C(RESULT_ACCESS)] = -1,
+ [C(RESULT_MISS)] = -1,
+ },
+ [C(OP_PREFETCH)] = {
+ [C(RESULT_ACCESS)] = -1,
+ [C(RESULT_MISS)] = -1,
+ },
+ },
+ [C(BPU)] = {
+ [C(OP_READ)] = {
+ [C(RESULT_ACCESS)] = PM_BR_CMPL,
+ [C(RESULT_MISS)] = PM_BR_MPRED_CMPL,
+ },
+ [C(OP_WRITE)] = {
+ [C(RESULT_ACCESS)] = -1,
+ [C(RESULT_MISS)] = -1,
+ },
+ [C(OP_PREFETCH)] = {
+ [C(RESULT_ACCESS)] = -1,
+ [C(RESULT_MISS)] = -1,
+ },
+ },
+ [C(NODE)] = {
+ [C(OP_READ)] = {
+ [C(RESULT_ACCESS)] = -1,
+ [C(RESULT_MISS)] = -1,
+ },
+ [C(OP_WRITE)] = {
+ [C(RESULT_ACCESS)] = -1,
+ [C(RESULT_MISS)] = -1,
+ },
+ [C(OP_PREFETCH)] = {
+ [C(RESULT_ACCESS)] = -1,
+ [C(RESULT_MISS)] = -1,
+ },
+ },
+};
+
+#undef C
+
+static struct power_pmu power10_pmu = {
+ .name = "POWER10",
+ .n_counter = MAX_PMU_COUNTERS,
+ .add_fields = ISA207_ADD_FIELDS,
+ .test_adder = ISA207_TEST_ADDER,
+ .group_constraint_mask = CNST_CACHE_PMC4_MASK,
+ .group_constraint_val = CNST_CACHE_PMC4_VAL,
+ .compute_mmcr = isa207_compute_mmcr,
+ .config_bhrb = power10_config_bhrb,
+ .bhrb_filter_map = power10_bhrb_filter_map,
+ .get_constraint = isa207_get_constraint,
+ .get_alternatives = power10_get_alternatives,
+ .get_mem_data_src = isa207_get_mem_data_src,
+ .get_mem_weight = isa207_get_mem_weight,
+ .disable_pmc = isa207_disable_pmc,
+ .flags = PPMU_HAS_SIER | PPMU_ARCH_207S |
+ PPMU_ARCH_31,
+ .n_generic = ARRAY_SIZE(power10_generic_events),
+ .generic_events = power10_generic_events,
+ .cache_events = &power10_cache_events,
+ .attr_groups = power10_pmu_attr_groups,
+ .bhrb_nr = 32,
+};
+
+int init_power10_pmu(void)
+{
+ int rc;
+
+ /* Comes from cpu_specs[] */
+ if (!cur_cpu_spec->oprofile_cpu_type ||
+ strcmp(cur_cpu_spec->oprofile_cpu_type, "ppc64/power10"))
+ return -ENODEV;
+
+ rc = register_power_pmu(&power10_pmu);
+ if (rc)
+ return rc;
+
+ /* Tell userspace that EBB is supported */
+ cur_cpu_spec->cpu_user_features2 |= PPC_FEATURE2_EBB;
+
+ return 0;
+}
diff --git a/arch/powerpc/perf/power5+-pmu.c b/arch/powerpc/perf/power5+-pmu.c
index f857454..a62b2cd 100644
--- a/arch/powerpc/perf/power5+-pmu.c
+++ b/arch/powerpc/perf/power5+-pmu.c
@@ -448,7 +448,8 @@
}
static int power5p_compute_mmcr(u64 event[], int n_ev,
- unsigned int hwc[], unsigned long mmcr[], struct perf_event *pevents[])
+ unsigned int hwc[], struct mmcr_regs *mmcr,
+ struct perf_event *pevents[])
{
unsigned long mmcr1 = 0;
unsigned long mmcra = 0;
@@ -586,20 +587,20 @@
}
/* Return MMCRx values */
- mmcr[0] = 0;
+ mmcr->mmcr0 = 0;
if (pmc_inuse & 1)
- mmcr[0] = MMCR0_PMC1CE;
+ mmcr->mmcr0 = MMCR0_PMC1CE;
if (pmc_inuse & 0x3e)
- mmcr[0] |= MMCR0_PMCjCE;
- mmcr[1] = mmcr1;
- mmcr[2] = mmcra;
+ mmcr->mmcr0 |= MMCR0_PMCjCE;
+ mmcr->mmcr1 = mmcr1;
+ mmcr->mmcra = mmcra;
return 0;
}
-static void power5p_disable_pmc(unsigned int pmc, unsigned long mmcr[])
+static void power5p_disable_pmc(unsigned int pmc, struct mmcr_regs *mmcr)
{
if (pmc <= 3)
- mmcr[1] &= ~(0x7fUL << MMCR1_PMCSEL_SH(pmc));
+ mmcr->mmcr1 &= ~(0x7fUL << MMCR1_PMCSEL_SH(pmc));
}
static int power5p_generic_events[] = {
@@ -618,7 +619,7 @@
* 0 means not supported, -1 means nonsensical, other values
* are event codes.
*/
-static int power5p_cache_events[C(MAX)][C(OP_MAX)][C(RESULT_MAX)] = {
+static u64 power5p_cache_events[C(MAX)][C(OP_MAX)][C(RESULT_MAX)] = {
[C(L1D)] = { /* RESULT_ACCESS RESULT_MISS */
[C(OP_READ)] = { 0x1c10a8, 0x3c1088 },
[C(OP_WRITE)] = { 0x2c10a8, 0xc10c3 },
diff --git a/arch/powerpc/perf/power5-pmu.c b/arch/powerpc/perf/power5-pmu.c
index da52eca..8732b58 100644
--- a/arch/powerpc/perf/power5-pmu.c
+++ b/arch/powerpc/perf/power5-pmu.c
@@ -379,7 +379,8 @@
}
static int power5_compute_mmcr(u64 event[], int n_ev,
- unsigned int hwc[], unsigned long mmcr[], struct perf_event *pevents[])
+ unsigned int hwc[], struct mmcr_regs *mmcr,
+ struct perf_event *pevents[])
{
unsigned long mmcr1 = 0;
unsigned long mmcra = MMCRA_SDAR_DCACHE_MISS | MMCRA_SDAR_ERAT_MISS;
@@ -528,20 +529,20 @@
}
/* Return MMCRx values */
- mmcr[0] = 0;
+ mmcr->mmcr0 = 0;
if (pmc_inuse & 1)
- mmcr[0] = MMCR0_PMC1CE;
+ mmcr->mmcr0 = MMCR0_PMC1CE;
if (pmc_inuse & 0x3e)
- mmcr[0] |= MMCR0_PMCjCE;
- mmcr[1] = mmcr1;
- mmcr[2] = mmcra;
+ mmcr->mmcr0 |= MMCR0_PMCjCE;
+ mmcr->mmcr1 = mmcr1;
+ mmcr->mmcra = mmcra;
return 0;
}
-static void power5_disable_pmc(unsigned int pmc, unsigned long mmcr[])
+static void power5_disable_pmc(unsigned int pmc, struct mmcr_regs *mmcr)
{
if (pmc <= 3)
- mmcr[1] &= ~(0x7fUL << MMCR1_PMCSEL_SH(pmc));
+ mmcr->mmcr1 &= ~(0x7fUL << MMCR1_PMCSEL_SH(pmc));
}
static int power5_generic_events[] = {
@@ -560,7 +561,7 @@
* 0 means not supported, -1 means nonsensical, other values
* are event codes.
*/
-static int power5_cache_events[C(MAX)][C(OP_MAX)][C(RESULT_MAX)] = {
+static u64 power5_cache_events[C(MAX)][C(OP_MAX)][C(RESULT_MAX)] = {
[C(L1D)] = { /* RESULT_ACCESS RESULT_MISS */
[C(OP_READ)] = { 0x4c1090, 0x3c1088 },
[C(OP_WRITE)] = { 0x3c1090, 0xc10c3 },
diff --git a/arch/powerpc/perf/power6-pmu.c b/arch/powerpc/perf/power6-pmu.c
index 3929cac..0e318cf 100644
--- a/arch/powerpc/perf/power6-pmu.c
+++ b/arch/powerpc/perf/power6-pmu.c
@@ -171,7 +171,7 @@
* Assign PMC numbers and compute MMCR1 value for a set of events
*/
static int p6_compute_mmcr(u64 event[], int n_ev,
- unsigned int hwc[], unsigned long mmcr[], struct perf_event *pevents[])
+ unsigned int hwc[], struct mmcr_regs *mmcr, struct perf_event *pevents[])
{
unsigned long mmcr1 = 0;
unsigned long mmcra = MMCRA_SDAR_DCACHE_MISS | MMCRA_SDAR_ERAT_MISS;
@@ -243,13 +243,13 @@
if (pmc < 4)
mmcr1 |= (unsigned long)psel << MMCR1_PMCSEL_SH(pmc);
}
- mmcr[0] = 0;
+ mmcr->mmcr0 = 0;
if (pmc_inuse & 1)
- mmcr[0] = MMCR0_PMC1CE;
+ mmcr->mmcr0 = MMCR0_PMC1CE;
if (pmc_inuse & 0xe)
- mmcr[0] |= MMCR0_PMCjCE;
- mmcr[1] = mmcr1;
- mmcr[2] = mmcra;
+ mmcr->mmcr0 |= MMCR0_PMCjCE;
+ mmcr->mmcr1 = mmcr1;
+ mmcr->mmcra = mmcra;
return 0;
}
@@ -457,11 +457,11 @@
return nalt;
}
-static void p6_disable_pmc(unsigned int pmc, unsigned long mmcr[])
+static void p6_disable_pmc(unsigned int pmc, struct mmcr_regs *mmcr)
{
/* Set PMCxSEL to 0 to disable PMCx */
if (pmc <= 3)
- mmcr[1] &= ~(0xffUL << MMCR1_PMCSEL_SH(pmc));
+ mmcr->mmcr1 &= ~(0xffUL << MMCR1_PMCSEL_SH(pmc));
}
static int power6_generic_events[] = {
@@ -481,7 +481,7 @@
* are event codes.
* The "DTLB" and "ITLB" events relate to the DERAT and IERAT.
*/
-static int power6_cache_events[C(MAX)][C(OP_MAX)][C(RESULT_MAX)] = {
+static u64 power6_cache_events[C(MAX)][C(OP_MAX)][C(RESULT_MAX)] = {
[C(L1D)] = { /* RESULT_ACCESS RESULT_MISS */
[C(OP_READ)] = { 0x280030, 0x80080 },
[C(OP_WRITE)] = { 0x180032, 0x80088 },
diff --git a/arch/powerpc/perf/power7-pmu.c b/arch/powerpc/perf/power7-pmu.c
index a137813..5e0bf09 100644
--- a/arch/powerpc/perf/power7-pmu.c
+++ b/arch/powerpc/perf/power7-pmu.c
@@ -242,7 +242,8 @@
}
static int power7_compute_mmcr(u64 event[], int n_ev,
- unsigned int hwc[], unsigned long mmcr[], struct perf_event *pevents[])
+ unsigned int hwc[], struct mmcr_regs *mmcr,
+ struct perf_event *pevents[])
{
unsigned long mmcr1 = 0;
unsigned long mmcra = MMCRA_SDAR_DCACHE_MISS | MMCRA_SDAR_ERAT_MISS;
@@ -298,20 +299,20 @@
}
/* Return MMCRx values */
- mmcr[0] = 0;
+ mmcr->mmcr0 = 0;
if (pmc_inuse & 1)
- mmcr[0] = MMCR0_PMC1CE;
+ mmcr->mmcr0 = MMCR0_PMC1CE;
if (pmc_inuse & 0x3e)
- mmcr[0] |= MMCR0_PMCjCE;
- mmcr[1] = mmcr1;
- mmcr[2] = mmcra;
+ mmcr->mmcr0 |= MMCR0_PMCjCE;
+ mmcr->mmcr1 = mmcr1;
+ mmcr->mmcra = mmcra;
return 0;
}
-static void power7_disable_pmc(unsigned int pmc, unsigned long mmcr[])
+static void power7_disable_pmc(unsigned int pmc, struct mmcr_regs *mmcr)
{
if (pmc <= 3)
- mmcr[1] &= ~(0xffUL << MMCR1_PMCSEL_SH(pmc));
+ mmcr->mmcr1 &= ~(0xffUL << MMCR1_PMCSEL_SH(pmc));
}
static int power7_generic_events[] = {
@@ -332,7 +333,7 @@
* 0 means not supported, -1 means nonsensical, other values
* are event codes.
*/
-static int power7_cache_events[C(MAX)][C(OP_MAX)][C(RESULT_MAX)] = {
+static u64 power7_cache_events[C(MAX)][C(OP_MAX)][C(RESULT_MAX)] = {
[C(L1D)] = { /* RESULT_ACCESS RESULT_MISS */
[C(OP_READ)] = { 0xc880, 0x400f0 },
[C(OP_WRITE)] = { 0, 0x300f0 },
diff --git a/arch/powerpc/perf/power8-pmu.c b/arch/powerpc/perf/power8-pmu.c
index 3a5fcc2..5282e84 100644
--- a/arch/powerpc/perf/power8-pmu.c
+++ b/arch/powerpc/perf/power8-pmu.c
@@ -253,7 +253,7 @@
* 0 means not supported, -1 means nonsensical, other values
* are event codes.
*/
-static int power8_cache_events[C(MAX)][C(OP_MAX)][C(RESULT_MAX)] = {
+static u64 power8_cache_events[C(MAX)][C(OP_MAX)][C(RESULT_MAX)] = {
[ C(L1D) ] = {
[ C(OP_READ) ] = {
[ C(RESULT_ACCESS) ] = PM_LD_REF_L1,
diff --git a/arch/powerpc/perf/power9-pmu.c b/arch/powerpc/perf/power9-pmu.c
index 08c3ef7..05dae38 100644
--- a/arch/powerpc/perf/power9-pmu.c
+++ b/arch/powerpc/perf/power9-pmu.c
@@ -310,7 +310,7 @@
* 0 means not supported, -1 means nonsensical, other values
* are event codes.
*/
-static int power9_cache_events[C(MAX)][C(OP_MAX)][C(RESULT_MAX)] = {
+static u64 power9_cache_events[C(MAX)][C(OP_MAX)][C(RESULT_MAX)] = {
[ C(L1D) ] = {
[ C(OP_READ) ] = {
[ C(RESULT_ACCESS) ] = PM_LD_REF_L1,
diff --git a/arch/powerpc/perf/ppc970-pmu.c b/arch/powerpc/perf/ppc970-pmu.c
index 4035d93..d35223f 100644
--- a/arch/powerpc/perf/ppc970-pmu.c
+++ b/arch/powerpc/perf/ppc970-pmu.c
@@ -253,7 +253,8 @@
}
static int p970_compute_mmcr(u64 event[], int n_ev,
- unsigned int hwc[], unsigned long mmcr[], struct perf_event *pevents[])
+ unsigned int hwc[], struct mmcr_regs *mmcr,
+ struct perf_event *pevents[])
{
unsigned long mmcr0 = 0, mmcr1 = 0, mmcra = 0;
unsigned int pmc, unit, byte, psel;
@@ -393,27 +394,26 @@
mmcra |= 0x2000; /* mark only one IOP per PPC instruction */
/* Return MMCRx values */
- mmcr[0] = mmcr0;
- mmcr[1] = mmcr1;
- mmcr[2] = mmcra;
+ mmcr->mmcr0 = mmcr0;
+ mmcr->mmcr1 = mmcr1;
+ mmcr->mmcra = mmcra;
return 0;
}
-static void p970_disable_pmc(unsigned int pmc, unsigned long mmcr[])
+static void p970_disable_pmc(unsigned int pmc, struct mmcr_regs *mmcr)
{
- int shift, i;
+ int shift;
- if (pmc <= 1) {
- shift = MMCR0_PMC1SEL_SH - 7 * pmc;
- i = 0;
- } else {
- shift = MMCR1_PMC3SEL_SH - 5 * (pmc - 2);
- i = 1;
- }
/*
* Setting the PMCxSEL field to 0x08 disables PMC x.
*/
- mmcr[i] = (mmcr[i] & ~(0x1fUL << shift)) | (0x08UL << shift);
+ if (pmc <= 1) {
+ shift = MMCR0_PMC1SEL_SH - 7 * pmc;
+ mmcr->mmcr0 = (mmcr->mmcr0 & ~(0x1fUL << shift)) | (0x08UL << shift);
+ } else {
+ shift = MMCR1_PMC3SEL_SH - 5 * (pmc - 2);
+ mmcr->mmcr1 = (mmcr->mmcr1 & ~(0x1fUL << shift)) | (0x08UL << shift);
+ }
}
static int ppc970_generic_events[] = {
@@ -432,7 +432,7 @@
* 0 means not supported, -1 means nonsensical, other values
* are event codes.
*/
-static int ppc970_cache_events[C(MAX)][C(OP_MAX)][C(RESULT_MAX)] = {
+static u64 ppc970_cache_events[C(MAX)][C(OP_MAX)][C(RESULT_MAX)] = {
[C(L1D)] = { /* RESULT_ACCESS RESULT_MISS */
[C(OP_READ)] = { 0x8810, 0x3810 },
[C(OP_WRITE)] = { 0x7810, 0x813 },
diff --git a/arch/powerpc/platforms/52xx/lite5200_sleep.S b/arch/powerpc/platforms/52xx/lite5200_sleep.S
index 70083649..11475c5 100644
--- a/arch/powerpc/platforms/52xx/lite5200_sleep.S
+++ b/arch/powerpc/platforms/52xx/lite5200_sleep.S
@@ -56,7 +56,7 @@
/*
* save stuff BDI overwrites
* 0xf0 (0xe0->0x100 gets overwritten when BDI connected;
- * even when CONFIG_BDI* is disabled and MMU XLAT commented; heisenbug?))
+ * even when CONFIG_BDI_SWITCH is disabled and MMU XLAT commented; heisenbug?))
* WARNING: self-refresh doesn't seem to work when BDI2000 is connected,
* possibly because BDI sets SDRAM registers before wakeup code does
*/
diff --git a/arch/powerpc/platforms/85xx/Kconfig b/arch/powerpc/platforms/85xx/Kconfig
index fa3d29d..b77cbb0 100644
--- a/arch/powerpc/platforms/85xx/Kconfig
+++ b/arch/powerpc/platforms/85xx/Kconfig
@@ -160,7 +160,7 @@
computers from Extreme Engineering Solutions (X-ES) based on
Freescale MPC85xx processors.
Manufacturer: Extreme Engineering Solutions, Inc.
- URL: <http://www.xes-inc.com/>
+ URL: <https://www.xes-inc.com/>
config STX_GP3
bool "Silicon Turnkey Express GP3"
diff --git a/arch/powerpc/platforms/cell/Kconfig b/arch/powerpc/platforms/cell/Kconfig
index 0f7c824..f2ff359 100644
--- a/arch/powerpc/platforms/cell/Kconfig
+++ b/arch/powerpc/platforms/cell/Kconfig
@@ -44,6 +44,7 @@
tristate "SPU file system"
default m
depends on PPC_CELL
+ depends on COREDUMP
select SPU_BASE
help
The SPU file system is used to access Synergistic Processing
diff --git a/arch/powerpc/platforms/cell/spufs/coredump.c b/arch/powerpc/platforms/cell/spufs/coredump.c
index 3b75e8f..026c181 100644
--- a/arch/powerpc/platforms/cell/spufs/coredump.c
+++ b/arch/powerpc/platforms/cell/spufs/coredump.c
@@ -66,13 +66,20 @@
*/
static struct spu_context *coredump_next_context(int *fd)
{
+ struct spu_context *ctx;
struct file *file;
int n = iterate_fd(current->files, *fd, match_context, NULL);
if (!n)
return NULL;
*fd = n - 1;
+
+ rcu_read_lock();
file = fcheck(*fd);
- return SPUFS_I(file_inode(file))->i_ctx;
+ ctx = SPUFS_I(file_inode(file))->i_ctx;
+ get_spu_context(ctx);
+ rcu_read_unlock();
+
+ return ctx;
}
int spufs_coredump_extra_notes_size(void)
@@ -83,17 +90,23 @@
fd = 0;
while ((ctx = coredump_next_context(&fd)) != NULL) {
rc = spu_acquire_saved(ctx);
- if (rc)
+ if (rc) {
+ put_spu_context(ctx);
break;
+ }
+
rc = spufs_ctx_note_size(ctx, fd);
spu_release_saved(ctx);
- if (rc < 0)
+ if (rc < 0) {
+ put_spu_context(ctx);
break;
+ }
size += rc;
/* start searching the next fd next time */
fd++;
+ put_spu_context(ctx);
}
return size;
@@ -105,7 +118,7 @@
size_t sz = spufs_coredump_read[i].size;
char fullname[80];
struct elf_note en;
- size_t ret;
+ int ret;
sprintf(fullname, "SPU/%d/%s", dfd, spufs_coredump_read[i].name);
en.n_namesz = strlen(fullname) + 1;
diff --git a/arch/powerpc/platforms/pasemi/misc.c b/arch/powerpc/platforms/pasemi/misc.c
index 1cd4ca1..1bf65d0 100644
--- a/arch/powerpc/platforms/pasemi/misc.c
+++ b/arch/powerpc/platforms/pasemi/misc.c
@@ -56,8 +56,7 @@
if (!adap_node)
continue;
- node = NULL;
- while ((node = of_get_next_child(adap_node, node))) {
+ for_each_child_of_node(adap_node, node) {
struct i2c_board_info info = {};
const u32 *addr;
int len;
diff --git a/arch/powerpc/platforms/powermac/feature.c b/arch/powerpc/platforms/powermac/feature.c
index 181caa3..5c77b9a 100644
--- a/arch/powerpc/platforms/powermac/feature.c
+++ b/arch/powerpc/platforms/powermac/feature.c
@@ -1465,7 +1465,7 @@
case 2:
if (macio->type == macio_shasta)
break;
- /* fall through */
+ fallthrough;
default:
return -ENODEV;
}
diff --git a/arch/powerpc/platforms/powermac/low_i2c.c b/arch/powerpc/platforms/powermac/low_i2c.c
index bf4be4b..f77a59b 100644
--- a/arch/powerpc/platforms/powermac/low_i2c.c
+++ b/arch/powerpc/platforms/powermac/low_i2c.c
@@ -629,8 +629,7 @@
for (i = 0; i < chans; i++)
kw_i2c_add(host, np, np, i);
} else {
- for (child = NULL;
- (child = of_get_next_child(np, child)) != NULL;) {
+ for_each_child_of_node(np, child) {
const u32 *reg = of_get_property(child,
"reg", NULL);
if (reg == NULL)
@@ -1193,8 +1192,7 @@
* platform function instance
*/
list_for_each_entry(bus, &pmac_i2c_busses, link) {
- for (np = NULL;
- (np = of_get_next_child(bus->busnode, np)) != NULL;) {
+ for_each_child_of_node(bus->busnode, np) {
struct whitelist_ent *p;
/* If multibus, check if device is on that bus */
if (bus->flags & pmac_i2c_multibus)
diff --git a/arch/powerpc/platforms/powermac/pfunc_base.c b/arch/powerpc/platforms/powermac/pfunc_base.c
index 62311e8..f542250 100644
--- a/arch/powerpc/platforms/powermac/pfunc_base.c
+++ b/arch/powerpc/platforms/powermac/pfunc_base.c
@@ -114,7 +114,7 @@
* Ok, got one, we dont need anything special to track them down, so
* we just create them all
*/
- for (gp = NULL; (gp = of_get_next_child(gparent, gp)) != NULL;) {
+ for_each_child_of_node(gparent, gp) {
const u32 *reg = of_get_property(gp, "reg", NULL);
unsigned long offset;
if (reg == NULL)
@@ -133,7 +133,7 @@
macio->of_node);
/* And now we run all the init ones */
- for (gp = NULL; (gp = of_get_next_child(gparent, gp)) != NULL;)
+ for_each_child_of_node(gparent, gp)
pmf_do_functions(gp, NULL, 0, PMF_FLAGS_ON_INIT, NULL);
/* Note: We do not at this point implement the "at sleep" or "at wake"
diff --git a/arch/powerpc/platforms/powermac/udbg_scc.c b/arch/powerpc/platforms/powermac/udbg_scc.c
index 6b61a18..f286bdf 100644
--- a/arch/powerpc/platforms/powermac/udbg_scc.c
+++ b/arch/powerpc/platforms/powermac/udbg_scc.c
@@ -80,7 +80,7 @@
path = of_get_property(of_chosen, "linux,stdout-path", NULL);
if (path != NULL)
stdout = of_find_node_by_path(path);
- for (ch = NULL; (ch = of_get_next_child(escc, ch)) != NULL;) {
+ for_each_child_of_node(escc, ch) {
if (ch == stdout)
ch_def = of_node_get(ch);
if (of_node_name_eq(ch, "ch-a"))
diff --git a/arch/powerpc/platforms/powernv/Makefile b/arch/powerpc/platforms/powernv/Makefile
index fe3f0fb..2eb6ae1 100644
--- a/arch/powerpc/platforms/powernv/Makefile
+++ b/arch/powerpc/platforms/powernv/Makefile
@@ -11,6 +11,7 @@
obj-$(CONFIG_PRESERVE_FA_DUMP) += opal-fadump.o
obj-$(CONFIG_OPAL_CORE) += opal-core.o
obj-$(CONFIG_PCI) += pci.o pci-ioda.o npu-dma.o pci-ioda-tce.o
+obj-$(CONFIG_PCI_IOV) += pci-sriov.o
obj-$(CONFIG_CXL_BASE) += pci-cxl.o
obj-$(CONFIG_EEH) += eeh-powernv.o
obj-$(CONFIG_MEMORY_FAILURE) += opal-memory-errors.o
diff --git a/arch/powerpc/platforms/powernv/eeh-powernv.c b/arch/powerpc/platforms/powernv/eeh-powernv.c
index 79409e0..9af8c3b 100644
--- a/arch/powerpc/platforms/powernv/eeh-powernv.c
+++ b/arch/powerpc/platforms/powernv/eeh-powernv.c
@@ -338,6 +338,28 @@
return 0;
}
+static struct eeh_pe *pnv_eeh_get_upstream_pe(struct pci_dev *pdev)
+{
+ struct pci_controller *hose = pdev->bus->sysdata;
+ struct pnv_phb *phb = hose->private_data;
+ struct pci_dev *parent = pdev->bus->self;
+
+#ifdef CONFIG_PCI_IOV
+ /* for VFs we use the PF's PE as the upstream PE */
+ if (pdev->is_virtfn)
+ parent = pdev->physfn;
+#endif
+
+ /* otherwise use the PE of our parent bridge */
+ if (parent) {
+ struct pnv_ioda_pe *ioda_pe = pnv_ioda_get_pe(parent);
+
+ return eeh_pe_get(phb->hose, ioda_pe->pe_number, 0);
+ }
+
+ return NULL;
+}
+
/**
* pnv_eeh_probe - Do probe on PCI device
* @pdev: pci_dev to probe
@@ -350,6 +372,7 @@
struct pci_controller *hose = pdn->phb;
struct pnv_phb *phb = hose->private_data;
struct eeh_dev *edev = pdn_to_eeh_dev(pdn);
+ struct eeh_pe *upstream_pe;
uint32_t pcie_flags;
int ret;
int config_addr = (pdn->busno << 8) | (pdn->devfn);
@@ -372,19 +395,18 @@
}
/* Skip for PCI-ISA bridge */
- if ((pdn->class_code >> 8) == PCI_CLASS_BRIDGE_ISA)
+ if ((pdev->class >> 8) == PCI_CLASS_BRIDGE_ISA)
return NULL;
eeh_edev_dbg(edev, "Probing device\n");
/* Initialize eeh device */
- edev->class_code = pdn->class_code;
edev->mode &= 0xFFFFFF00;
edev->pcix_cap = pnv_eeh_find_cap(pdn, PCI_CAP_ID_PCIX);
edev->pcie_cap = pnv_eeh_find_cap(pdn, PCI_CAP_ID_EXP);
edev->af_cap = pnv_eeh_find_cap(pdn, PCI_CAP_ID_AF);
edev->aer_cap = pnv_eeh_find_ecap(pdn, PCI_EXT_CAP_ID_ERR);
- if ((edev->class_code >> 8) == PCI_CLASS_BRIDGE_PCI) {
+ if ((pdev->class >> 8) == PCI_CLASS_BRIDGE_PCI) {
edev->mode |= EEH_DEV_BRIDGE;
if (edev->pcie_cap) {
pnv_pci_cfg_read(pdn, edev->pcie_cap + PCI_EXP_FLAGS,
@@ -399,8 +421,10 @@
edev->pe_config_addr = phb->ioda.pe_rmap[config_addr];
+ upstream_pe = pnv_eeh_get_upstream_pe(pdev);
+
/* Create PE */
- ret = eeh_add_to_parent_pe(edev);
+ ret = eeh_pe_tree_insert(edev, upstream_pe);
if (ret) {
eeh_edev_warn(edev, "Failed to add device to PE (code %d)\n", ret);
return NULL;
@@ -535,18 +559,6 @@
return 0;
}
-/**
- * pnv_eeh_get_pe_addr - Retrieve PE address
- * @pe: EEH PE
- *
- * Retrieve the PE address according to the given tranditional
- * PCI BDF (Bus/Device/Function) address.
- */
-static int pnv_eeh_get_pe_addr(struct eeh_pe *pe)
-{
- return pe->addr;
-}
-
static void pnv_eeh_get_phb_diag(struct eeh_pe *pe)
{
struct pnv_phb *phb = pe->phb->private_data;
@@ -850,32 +862,32 @@
case EEH_RESET_HOT:
/* Don't report linkDown event */
if (aer) {
- eeh_ops->read_config(pdn, aer + PCI_ERR_UNCOR_MASK,
+ eeh_ops->read_config(edev, aer + PCI_ERR_UNCOR_MASK,
4, &ctrl);
ctrl |= PCI_ERR_UNC_SURPDN;
- eeh_ops->write_config(pdn, aer + PCI_ERR_UNCOR_MASK,
+ eeh_ops->write_config(edev, aer + PCI_ERR_UNCOR_MASK,
4, ctrl);
}
- eeh_ops->read_config(pdn, PCI_BRIDGE_CONTROL, 2, &ctrl);
+ eeh_ops->read_config(edev, PCI_BRIDGE_CONTROL, 2, &ctrl);
ctrl |= PCI_BRIDGE_CTL_BUS_RESET;
- eeh_ops->write_config(pdn, PCI_BRIDGE_CONTROL, 2, ctrl);
+ eeh_ops->write_config(edev, PCI_BRIDGE_CONTROL, 2, ctrl);
msleep(EEH_PE_RST_HOLD_TIME);
break;
case EEH_RESET_DEACTIVATE:
- eeh_ops->read_config(pdn, PCI_BRIDGE_CONTROL, 2, &ctrl);
+ eeh_ops->read_config(edev, PCI_BRIDGE_CONTROL, 2, &ctrl);
ctrl &= ~PCI_BRIDGE_CTL_BUS_RESET;
- eeh_ops->write_config(pdn, PCI_BRIDGE_CONTROL, 2, ctrl);
+ eeh_ops->write_config(edev, PCI_BRIDGE_CONTROL, 2, ctrl);
msleep(EEH_PE_RST_SETTLE_TIME);
/* Continue reporting linkDown event */
if (aer) {
- eeh_ops->read_config(pdn, aer + PCI_ERR_UNCOR_MASK,
+ eeh_ops->read_config(edev, aer + PCI_ERR_UNCOR_MASK,
4, &ctrl);
ctrl &= ~PCI_ERR_UNC_SURPDN;
- eeh_ops->write_config(pdn, aer + PCI_ERR_UNCOR_MASK,
+ eeh_ops->write_config(edev, aer + PCI_ERR_UNCOR_MASK,
4, ctrl);
}
@@ -944,11 +956,12 @@
static void pnv_eeh_wait_for_pending(struct pci_dn *pdn, const char *type,
int pos, u16 mask)
{
+ struct eeh_dev *edev = pdn->edev;
int i, status = 0;
/* Wait for Transaction Pending bit to be cleared */
for (i = 0; i < 4; i++) {
- eeh_ops->read_config(pdn, pos, 2, &status);
+ eeh_ops->read_config(edev, pos, 2, &status);
if (!(status & mask))
return;
@@ -969,7 +982,7 @@
if (WARN_ON(!edev->pcie_cap))
return -ENOTTY;
- eeh_ops->read_config(pdn, edev->pcie_cap + PCI_EXP_DEVCAP, 4, ®);
+ eeh_ops->read_config(edev, edev->pcie_cap + PCI_EXP_DEVCAP, 4, ®);
if (!(reg & PCI_EXP_DEVCAP_FLR))
return -ENOTTY;
@@ -979,18 +992,18 @@
pnv_eeh_wait_for_pending(pdn, "",
edev->pcie_cap + PCI_EXP_DEVSTA,
PCI_EXP_DEVSTA_TRPND);
- eeh_ops->read_config(pdn, edev->pcie_cap + PCI_EXP_DEVCTL,
+ eeh_ops->read_config(edev, edev->pcie_cap + PCI_EXP_DEVCTL,
4, ®);
reg |= PCI_EXP_DEVCTL_BCR_FLR;
- eeh_ops->write_config(pdn, edev->pcie_cap + PCI_EXP_DEVCTL,
+ eeh_ops->write_config(edev, edev->pcie_cap + PCI_EXP_DEVCTL,
4, reg);
msleep(EEH_PE_RST_HOLD_TIME);
break;
case EEH_RESET_DEACTIVATE:
- eeh_ops->read_config(pdn, edev->pcie_cap + PCI_EXP_DEVCTL,
+ eeh_ops->read_config(edev, edev->pcie_cap + PCI_EXP_DEVCTL,
4, ®);
reg &= ~PCI_EXP_DEVCTL_BCR_FLR;
- eeh_ops->write_config(pdn, edev->pcie_cap + PCI_EXP_DEVCTL,
+ eeh_ops->write_config(edev, edev->pcie_cap + PCI_EXP_DEVCTL,
4, reg);
msleep(EEH_PE_RST_SETTLE_TIME);
break;
@@ -1007,7 +1020,7 @@
if (WARN_ON(!edev->af_cap))
return -ENOTTY;
- eeh_ops->read_config(pdn, edev->af_cap + PCI_AF_CAP, 1, &cap);
+ eeh_ops->read_config(edev, edev->af_cap + PCI_AF_CAP, 1, &cap);
if (!(cap & PCI_AF_CAP_TP) || !(cap & PCI_AF_CAP_FLR))
return -ENOTTY;
@@ -1022,12 +1035,12 @@
pnv_eeh_wait_for_pending(pdn, "AF",
edev->af_cap + PCI_AF_CTRL,
PCI_AF_STATUS_TP << 8);
- eeh_ops->write_config(pdn, edev->af_cap + PCI_AF_CTRL,
+ eeh_ops->write_config(edev, edev->af_cap + PCI_AF_CTRL,
1, PCI_AF_CTRL_FLR);
msleep(EEH_PE_RST_HOLD_TIME);
break;
case EEH_RESET_DEACTIVATE:
- eeh_ops->write_config(pdn, edev->af_cap + PCI_AF_CTRL, 1, 0);
+ eeh_ops->write_config(edev, edev->af_cap + PCI_AF_CTRL, 1, 0);
msleep(EEH_PE_RST_SETTLE_TIME);
break;
}
@@ -1261,9 +1274,11 @@
return false;
}
-static int pnv_eeh_read_config(struct pci_dn *pdn,
+static int pnv_eeh_read_config(struct eeh_dev *edev,
int where, int size, u32 *val)
{
+ struct pci_dn *pdn = eeh_dev_to_pdn(edev);
+
if (!pdn)
return PCIBIOS_DEVICE_NOT_FOUND;
@@ -1275,9 +1290,11 @@
return pnv_pci_cfg_read(pdn, where, size, val);
}
-static int pnv_eeh_write_config(struct pci_dn *pdn,
+static int pnv_eeh_write_config(struct eeh_dev *edev,
int where, int size, u32 val)
{
+ struct pci_dn *pdn = eeh_dev_to_pdn(edev);
+
if (!pdn)
return PCIBIOS_DEVICE_NOT_FOUND;
@@ -1631,34 +1648,24 @@
return ret;
}
-static int pnv_eeh_restore_config(struct pci_dn *pdn)
+static int pnv_eeh_restore_config(struct eeh_dev *edev)
{
- struct eeh_dev *edev = pdn_to_eeh_dev(pdn);
struct pnv_phb *phb;
s64 ret = 0;
- int config_addr = (pdn->busno << 8) | (pdn->devfn);
if (!edev)
return -EEXIST;
- /*
- * We have to restore the PCI config space after reset since the
- * firmware can't see SRIOV VFs.
- *
- * FIXME: The MPS, error routing rules, timeout setting are worthy
- * to be exported by firmware in extendible way.
- */
- if (edev->physfn) {
- ret = eeh_restore_vf_config(pdn);
- } else {
- phb = pdn->phb->private_data;
- ret = opal_pci_reinit(phb->opal_id,
- OPAL_REINIT_PCI_DEV, config_addr);
- }
+ if (edev->physfn)
+ return 0;
+
+ phb = edev->controller->private_data;
+ ret = opal_pci_reinit(phb->opal_id,
+ OPAL_REINIT_PCI_DEV, edev->bdfn);
if (ret) {
pr_warn("%s: Can't reinit PCI dev 0x%x (%lld)\n",
- __func__, config_addr, ret);
+ __func__, edev->bdfn, ret);
return -EIO;
}
@@ -1670,7 +1677,6 @@
.init = pnv_eeh_init,
.probe = pnv_eeh_probe,
.set_option = pnv_eeh_set_option,
- .get_pe_addr = pnv_eeh_get_pe_addr,
.get_state = pnv_eeh_get_state,
.reset = pnv_eeh_reset,
.get_log = pnv_eeh_get_log,
diff --git a/arch/powerpc/platforms/powernv/idle.c b/arch/powerpc/platforms/powernv/idle.c
index 2dd4673..77513a80 100644
--- a/arch/powerpc/platforms/powernv/idle.c
+++ b/arch/powerpc/platforms/powernv/idle.c
@@ -48,7 +48,7 @@
* First stop state levels when SPR and TB loss can occur.
*/
static u64 pnv_first_tb_loss_level = MAX_STOP_STATE + 1;
-static u64 pnv_first_spr_loss_level = MAX_STOP_STATE + 1;
+static u64 deep_spr_loss_state = MAX_STOP_STATE + 1;
/*
* psscr value and mask of the deepest stop idle state.
@@ -73,9 +73,6 @@
*/
uint64_t lpcr_val = mfspr(SPRN_LPCR);
uint64_t hid0_val = mfspr(SPRN_HID0);
- uint64_t hid1_val = mfspr(SPRN_HID1);
- uint64_t hid4_val = mfspr(SPRN_HID4);
- uint64_t hid5_val = mfspr(SPRN_HID5);
uint64_t hmeer_val = mfspr(SPRN_HMEER);
uint64_t msr_val = MSR_IDLE;
uint64_t psscr_val = pnv_deepest_stop_psscr_val;
@@ -117,6 +114,9 @@
/* Only p8 needs to set extra HID regiters */
if (!cpu_has_feature(CPU_FTR_ARCH_300)) {
+ uint64_t hid1_val = mfspr(SPRN_HID1);
+ uint64_t hid4_val = mfspr(SPRN_HID4);
+ uint64_t hid5_val = mfspr(SPRN_HID5);
rc = opal_slw_set_reg(pir, SPRN_HID1, hid1_val);
if (rc != 0)
@@ -611,6 +611,7 @@
unsigned long srr1;
unsigned long pls;
unsigned long mmcr0 = 0;
+ unsigned long mmcra = 0;
struct p9_sprs sprs = {}; /* avoid false used-uninitialised */
bool sprs_saved = false;
@@ -657,7 +658,22 @@
*/
mmcr0 = mfspr(SPRN_MMCR0);
}
- if ((psscr & PSSCR_RL_MASK) >= pnv_first_spr_loss_level) {
+
+ if (cpu_has_feature(CPU_FTR_ARCH_31)) {
+ /*
+ * POWER10 uses MMCRA (BHRBRD) as BHRB disable bit.
+ * If the user hasn't asked for the BHRB to be
+ * written, the value of MMCRA[BHRBRD] is 1.
+ * On wakeup from stop, MMCRA[BHRBD] will be 0,
+ * since it is previleged resource and will be lost.
+ * Thus, if we do not save and restore the MMCRA[BHRBD],
+ * hardware will be needlessly writing to the BHRB
+ * in problem mode.
+ */
+ mmcra = mfspr(SPRN_MMCRA);
+ }
+
+ if ((psscr & PSSCR_RL_MASK) >= deep_spr_loss_state) {
sprs.lpcr = mfspr(SPRN_LPCR);
sprs.hfscr = mfspr(SPRN_HFSCR);
sprs.fscr = mfspr(SPRN_FSCR);
@@ -700,8 +716,6 @@
WARN_ON_ONCE(mfmsr() & (MSR_IR|MSR_DR));
if ((srr1 & SRR1_WAKESTATE) != SRR1_WS_NOLOSS) {
- unsigned long mmcra;
-
/*
* We don't need an isync after the mtsprs here because the
* upcoming mtmsrd is execution synchronizing.
@@ -721,6 +735,10 @@
mtspr(SPRN_MMCR0, mmcr0);
}
+ /* Reload MMCRA to restore BHRB disable bit for POWER10 */
+ if (cpu_has_feature(CPU_FTR_ARCH_31))
+ mtspr(SPRN_MMCRA, mmcra);
+
/*
* DD2.2 and earlier need to set then clear bit 60 in MMCRA
* to ensure the PMU starts running.
@@ -741,7 +759,7 @@
* just always test PSSCR for SPR/TB state loss.
*/
pls = (psscr & PSSCR_PLS) >> PSSCR_PLS_SHIFT;
- if (likely(pls < pnv_first_spr_loss_level)) {
+ if (likely(pls < deep_spr_loss_state)) {
if (sprs_saved)
atomic_stop_thread_idle();
goto out;
@@ -1088,7 +1106,7 @@
* the deepest loss-less (OPAL_PM_STOP_INST_FAST) stop state.
*/
pnv_first_tb_loss_level = MAX_STOP_STATE + 1;
- pnv_first_spr_loss_level = MAX_STOP_STATE + 1;
+ deep_spr_loss_state = MAX_STOP_STATE + 1;
for (i = 0; i < nr_pnv_idle_states; i++) {
int err;
struct pnv_idle_states_t *state = &pnv_idle_states[i];
@@ -1099,8 +1117,8 @@
pnv_first_tb_loss_level = psscr_rl;
if ((state->flags & OPAL_PM_LOSE_FULL_CONTEXT) &&
- (pnv_first_spr_loss_level > psscr_rl))
- pnv_first_spr_loss_level = psscr_rl;
+ (deep_spr_loss_state > psscr_rl))
+ deep_spr_loss_state = psscr_rl;
/*
* The idle code does not deal with TB loss occurring
@@ -1111,8 +1129,8 @@
* compatibility.
*/
if ((state->flags & OPAL_PM_TIMEBASE_STOP) &&
- (pnv_first_spr_loss_level > psscr_rl))
- pnv_first_spr_loss_level = psscr_rl;
+ (deep_spr_loss_state > psscr_rl))
+ deep_spr_loss_state = psscr_rl;
err = validate_psscr_val_mask(&state->psscr_val,
&state->psscr_mask,
@@ -1158,7 +1176,7 @@
}
pr_info("cpuidle-powernv: First stop level that may lose SPRs = 0x%llx\n",
- pnv_first_spr_loss_level);
+ deep_spr_loss_state);
pr_info("cpuidle-powernv: First stop level that may lose timebase = 0x%llx\n",
pnv_first_tb_loss_level);
@@ -1205,7 +1223,7 @@
return;
}
- if (cpu_has_feature(CPU_FTR_ARCH_300))
+ if (pvr_version_is(PVR_POWER9))
pnv_power9_idle_init();
for (i = 0; i < nr_pnv_idle_states; i++)
diff --git a/arch/powerpc/platforms/powernv/opal-async.c b/arch/powerpc/platforms/powernv/opal-async.c
index 1656e89..c094fdf 100644
--- a/arch/powerpc/platforms/powernv/opal-async.c
+++ b/arch/powerpc/platforms/powernv/opal-async.c
@@ -104,7 +104,7 @@
*/
case ASYNC_TOKEN_DISPATCHED:
opal_async_tokens[token].state = ASYNC_TOKEN_ABANDONED;
- /* Fall through */
+ fallthrough;
default:
rc = 1;
}
diff --git a/arch/powerpc/platforms/powernv/pci-ioda-tce.c b/arch/powerpc/platforms/powernv/pci-ioda-tce.c
index f923359..5218f5d 100644
--- a/arch/powerpc/platforms/powernv/pci-ioda-tce.c
+++ b/arch/powerpc/platforms/powernv/pci-ioda-tce.c
@@ -166,7 +166,7 @@
if (!ptce) {
ptce = pnv_tce(tbl, false, idx, alloc);
if (!ptce)
- return alloc ? H_HARDWARE : H_TOO_HARD;
+ return -ENOMEM;
}
if (newtce & TCE_PCI_WRITE)
diff --git a/arch/powerpc/platforms/powernv/pci-ioda.c b/arch/powerpc/platforms/powernv/pci-ioda.c
index 73a63efc..c9c25fb 100644
--- a/arch/powerpc/platforms/powernv/pci-ioda.c
+++ b/arch/powerpc/platforms/powernv/pci-ioda.c
@@ -115,32 +115,13 @@
early_param("ppc_pci_reset_phbs", pci_reset_phbs_setup);
-static inline bool pnv_pci_is_m64(struct pnv_phb *phb, struct resource *r)
-{
- /*
- * WARNING: We cannot rely on the resource flags. The Linux PCI
- * allocation code sometimes decides to put a 64-bit prefetchable
- * BAR in the 32-bit window, so we have to compare the addresses.
- *
- * For simplicity we only test resource start.
- */
- return (r->start >= phb->ioda.m64_base &&
- r->start < (phb->ioda.m64_base + phb->ioda.m64_size));
-}
-
-static inline bool pnv_pci_is_m64_flags(unsigned long resource_flags)
-{
- unsigned long flags = (IORESOURCE_MEM_64 | IORESOURCE_PREFETCH);
-
- return (resource_flags & flags) == flags;
-}
-
static struct pnv_ioda_pe *pnv_ioda_init_pe(struct pnv_phb *phb, int pe_no)
{
s64 rc;
phb->ioda.pe_array[pe_no].phb = phb;
phb->ioda.pe_array[pe_no].pe_number = pe_no;
+ phb->ioda.pe_array[pe_no].dma_setup_done = false;
/*
* Clear the PE frozen state as it might be put into frozen state
@@ -164,26 +145,48 @@
return;
}
+ mutex_lock(&phb->ioda.pe_alloc_mutex);
if (test_and_set_bit(pe_no, phb->ioda.pe_alloc))
pr_debug("%s: PE %x was reserved on PHB#%x\n",
__func__, pe_no, phb->hose->global_number);
+ mutex_unlock(&phb->ioda.pe_alloc_mutex);
pnv_ioda_init_pe(phb, pe_no);
}
-static struct pnv_ioda_pe *pnv_ioda_alloc_pe(struct pnv_phb *phb)
+struct pnv_ioda_pe *pnv_ioda_alloc_pe(struct pnv_phb *phb, int count)
{
- long pe;
+ struct pnv_ioda_pe *ret = NULL;
+ int run = 0, pe, i;
+ mutex_lock(&phb->ioda.pe_alloc_mutex);
+
+ /* scan backwards for a run of @count cleared bits */
for (pe = phb->ioda.total_pe_num - 1; pe >= 0; pe--) {
- if (!test_and_set_bit(pe, phb->ioda.pe_alloc))
- return pnv_ioda_init_pe(phb, pe);
- }
+ if (test_bit(pe, phb->ioda.pe_alloc)) {
+ run = 0;
+ continue;
+ }
- return NULL;
+ run++;
+ if (run == count)
+ break;
+ }
+ if (run != count)
+ goto out;
+
+ for (i = pe; i < pe + count; i++) {
+ set_bit(i, phb->ioda.pe_alloc);
+ pnv_ioda_init_pe(phb, i);
+ }
+ ret = &phb->ioda.pe_array[pe];
+
+out:
+ mutex_unlock(&phb->ioda.pe_alloc_mutex);
+ return ret;
}
-static void pnv_ioda_free_pe(struct pnv_ioda_pe *pe)
+void pnv_ioda_free_pe(struct pnv_ioda_pe *pe)
{
struct pnv_phb *phb = pe->phb;
unsigned int pe_num = pe->pe_number;
@@ -192,7 +195,10 @@
WARN_ON(pe->npucomp); /* NPUs for nvlink are not supposed to be freed */
kfree(pe->npucomp);
memset(pe, 0, sizeof(struct pnv_ioda_pe));
+
+ mutex_lock(&phb->ioda.pe_alloc_mutex);
clear_bit(pe_num, phb->ioda.pe_alloc);
+ mutex_unlock(&phb->ioda.pe_alloc_mutex);
}
/* The default M64 BAR is shared by all PEs */
@@ -252,8 +258,7 @@
static void pnv_ioda_reserve_dev_m64_pe(struct pci_dev *pdev,
unsigned long *pe_bitmap)
{
- struct pci_controller *hose = pci_bus_to_host(pdev->bus);
- struct pnv_phb *phb = hose->private_data;
+ struct pnv_phb *phb = pci_bus_to_pnvhb(pdev->bus);
struct resource *r;
resource_size_t base, sgsz, start, end;
int segno, i;
@@ -311,6 +316,28 @@
}
}
+ for (index = 0; index < phb->ioda.total_pe_num; index++) {
+ int64_t rc;
+
+ /*
+ * P7IOC supports M64DT, which helps mapping M64 segment
+ * to one particular PE#. However, PHB3 has fixed mapping
+ * between M64 segment and PE#. In order to have same logic
+ * for P7IOC and PHB3, we enforce fixed mapping between M64
+ * segment and PE# on P7IOC.
+ */
+ rc = opal_pci_map_pe_mmio_window(phb->opal_id,
+ index, OPAL_M64_WINDOW_TYPE,
+ index / PNV_IODA1_M64_SEGS,
+ index % PNV_IODA1_M64_SEGS);
+ if (rc != OPAL_SUCCESS) {
+ pr_warn("%s: Error %lld mapping M64 for PHB#%x-PE#%x\n",
+ __func__, rc, phb->hose->global_number,
+ index);
+ goto fail;
+ }
+ }
+
/*
* Exclude the segments for reserved and root bus PE, which
* are first or last two PEs.
@@ -351,8 +378,7 @@
static struct pnv_ioda_pe *pnv_ioda_pick_m64_pe(struct pci_bus *bus, bool all)
{
- struct pci_controller *hose = pci_bus_to_host(bus);
- struct pnv_phb *phb = hose->private_data;
+ struct pnv_phb *phb = pci_bus_to_pnvhb(bus);
struct pnv_ioda_pe *master_pe, *pe;
unsigned long size, *pe_alloc;
int i;
@@ -403,26 +429,6 @@
pe->master = master_pe;
list_add_tail(&pe->list, &master_pe->slaves);
}
-
- /*
- * P7IOC supports M64DT, which helps mapping M64 segment
- * to one particular PE#. However, PHB3 has fixed mapping
- * between M64 segment and PE#. In order to have same logic
- * for P7IOC and PHB3, we enforce fixed mapping between M64
- * segment and PE# on P7IOC.
- */
- if (phb->type == PNV_PHB_IODA1) {
- int64_t rc;
-
- rc = opal_pci_map_pe_mmio_window(phb->opal_id,
- pe->pe_number, OPAL_M64_WINDOW_TYPE,
- pe->pe_number / PNV_IODA1_M64_SEGS,
- pe->pe_number % PNV_IODA1_M64_SEGS);
- if (rc != OPAL_SUCCESS)
- pr_warn("%s: Error %lld mapping M64 for PHB#%x-PE#%x\n",
- __func__, rc, phb->hose->global_number,
- pe->pe_number);
- }
}
kfree(pe_alloc);
@@ -673,8 +679,7 @@
struct pnv_ioda_pe *pnv_ioda_get_pe(struct pci_dev *dev)
{
- struct pci_controller *hose = pci_bus_to_host(dev->bus);
- struct pnv_phb *phb = hose->private_data;
+ struct pnv_phb *phb = pci_bus_to_pnvhb(dev->bus);
struct pci_dn *pdn = pci_get_pdn(dev);
if (!pdn)
@@ -816,7 +821,7 @@
pe_warn(pe, "OPAL error %lld remove self from PELTV\n", rc);
}
-static int pnv_ioda_deconfigure_pe(struct pnv_phb *phb, struct pnv_ioda_pe *pe)
+int pnv_ioda_deconfigure_pe(struct pnv_phb *phb, struct pnv_ioda_pe *pe)
{
struct pci_dev *parent;
uint8_t bcomp, dcomp, fcomp;
@@ -887,7 +892,7 @@
return 0;
}
-static int pnv_ioda_configure_pe(struct pnv_phb *phb, struct pnv_ioda_pe *pe)
+int pnv_ioda_configure_pe(struct pnv_phb *phb, struct pnv_ioda_pe *pe)
{
struct pci_dev *parent;
uint8_t bcomp, dcomp, fcomp;
@@ -982,95 +987,9 @@
return 0;
}
-#ifdef CONFIG_PCI_IOV
-static int pnv_pci_vf_resource_shift(struct pci_dev *dev, int offset)
-{
- struct pci_dn *pdn = pci_get_pdn(dev);
- int i;
- struct resource *res, res2;
- resource_size_t size;
- u16 num_vfs;
-
- if (!dev->is_physfn)
- return -EINVAL;
-
- /*
- * "offset" is in VFs. The M64 windows are sized so that when they
- * are segmented, each segment is the same size as the IOV BAR.
- * Each segment is in a separate PE, and the high order bits of the
- * address are the PE number. Therefore, each VF's BAR is in a
- * separate PE, and changing the IOV BAR start address changes the
- * range of PEs the VFs are in.
- */
- num_vfs = pdn->num_vfs;
- for (i = 0; i < PCI_SRIOV_NUM_BARS; i++) {
- res = &dev->resource[i + PCI_IOV_RESOURCES];
- if (!res->flags || !res->parent)
- continue;
-
- /*
- * The actual IOV BAR range is determined by the start address
- * and the actual size for num_vfs VFs BAR. This check is to
- * make sure that after shifting, the range will not overlap
- * with another device.
- */
- size = pci_iov_resource_size(dev, i + PCI_IOV_RESOURCES);
- res2.flags = res->flags;
- res2.start = res->start + (size * offset);
- res2.end = res2.start + (size * num_vfs) - 1;
-
- if (res2.end > res->end) {
- dev_err(&dev->dev, "VF BAR%d: %pR would extend past %pR (trying to enable %d VFs shifted by %d)\n",
- i, &res2, res, num_vfs, offset);
- return -EBUSY;
- }
- }
-
- /*
- * Since M64 BAR shares segments among all possible 256 PEs,
- * we have to shift the beginning of PF IOV BAR to make it start from
- * the segment which belongs to the PE number assigned to the first VF.
- * This creates a "hole" in the /proc/iomem which could be used for
- * allocating other resources so we reserve this area below and
- * release when IOV is released.
- */
- for (i = 0; i < PCI_SRIOV_NUM_BARS; i++) {
- res = &dev->resource[i + PCI_IOV_RESOURCES];
- if (!res->flags || !res->parent)
- continue;
-
- size = pci_iov_resource_size(dev, i + PCI_IOV_RESOURCES);
- res2 = *res;
- res->start += size * offset;
-
- dev_info(&dev->dev, "VF BAR%d: %pR shifted to %pR (%sabling %d VFs shifted by %d)\n",
- i, &res2, res, (offset > 0) ? "En" : "Dis",
- num_vfs, offset);
-
- if (offset < 0) {
- devm_release_resource(&dev->dev, &pdn->holes[i]);
- memset(&pdn->holes[i], 0, sizeof(pdn->holes[i]));
- }
-
- pci_update_resource(dev, i + PCI_IOV_RESOURCES);
-
- if (offset > 0) {
- pdn->holes[i].start = res2.start;
- pdn->holes[i].end = res2.start + size * offset - 1;
- pdn->holes[i].flags = IORESOURCE_BUS;
- pdn->holes[i].name = "pnv_iov_reserved";
- devm_request_resource(&dev->dev, res->parent,
- &pdn->holes[i]);
- }
- }
- return 0;
-}
-#endif /* CONFIG_PCI_IOV */
-
static struct pnv_ioda_pe *pnv_ioda_setup_dev_PE(struct pci_dev *dev)
{
- struct pci_controller *hose = pci_bus_to_host(dev->bus);
- struct pnv_phb *phb = hose->private_data;
+ struct pnv_phb *phb = pci_bus_to_pnvhb(dev->bus);
struct pci_dn *pdn = pci_get_pdn(dev);
struct pnv_ioda_pe *pe;
@@ -1082,7 +1001,7 @@
if (pdn->pe_number != IODA_INVALID_PE)
return NULL;
- pe = pnv_ioda_alloc_pe(phb);
+ pe = pnv_ioda_alloc_pe(phb, 1);
if (!pe) {
pr_warn("%s: Not enough PE# available, disabling device\n",
pci_name(dev));
@@ -1129,8 +1048,7 @@
*/
static struct pnv_ioda_pe *pnv_ioda_setup_bus_PE(struct pci_bus *bus, bool all)
{
- struct pci_controller *hose = pci_bus_to_host(bus);
- struct pnv_phb *phb = hose->private_data;
+ struct pnv_phb *phb = pci_bus_to_pnvhb(bus);
struct pnv_ioda_pe *pe = NULL;
unsigned int pe_num;
@@ -1154,7 +1072,7 @@
/* The PE number isn't pinned by M64 */
if (!pe)
- pe = pnv_ioda_alloc_pe(phb);
+ pe = pnv_ioda_alloc_pe(phb, 1);
if (!pe) {
pr_warn("%s: Not enough PE# available for PCI bus %04x:%02x\n",
@@ -1196,8 +1114,7 @@
struct pnv_ioda_pe *pe;
struct pci_dev *gpu_pdev;
struct pci_dn *npu_pdn;
- struct pci_controller *hose = pci_bus_to_host(npu_pdev->bus);
- struct pnv_phb *phb = hose->private_data;
+ struct pnv_phb *phb = pci_bus_to_pnvhb(npu_pdev->bus);
/*
* Intentionally leak a reference on the npu device (for
@@ -1297,446 +1214,12 @@
#endif
}
-#ifdef CONFIG_PCI_IOV
-static int pnv_pci_vf_release_m64(struct pci_dev *pdev, u16 num_vfs)
-{
- struct pci_bus *bus;
- struct pci_controller *hose;
- struct pnv_phb *phb;
- struct pci_dn *pdn;
- int i, j;
- int m64_bars;
-
- bus = pdev->bus;
- hose = pci_bus_to_host(bus);
- phb = hose->private_data;
- pdn = pci_get_pdn(pdev);
-
- if (pdn->m64_single_mode)
- m64_bars = num_vfs;
- else
- m64_bars = 1;
-
- for (i = 0; i < PCI_SRIOV_NUM_BARS; i++)
- for (j = 0; j < m64_bars; j++) {
- if (pdn->m64_map[j][i] == IODA_INVALID_M64)
- continue;
- opal_pci_phb_mmio_enable(phb->opal_id,
- OPAL_M64_WINDOW_TYPE, pdn->m64_map[j][i], 0);
- clear_bit(pdn->m64_map[j][i], &phb->ioda.m64_bar_alloc);
- pdn->m64_map[j][i] = IODA_INVALID_M64;
- }
-
- kfree(pdn->m64_map);
- return 0;
-}
-
-static int pnv_pci_vf_assign_m64(struct pci_dev *pdev, u16 num_vfs)
-{
- struct pci_bus *bus;
- struct pci_controller *hose;
- struct pnv_phb *phb;
- struct pci_dn *pdn;
- unsigned int win;
- struct resource *res;
- int i, j;
- int64_t rc;
- int total_vfs;
- resource_size_t size, start;
- int pe_num;
- int m64_bars;
-
- bus = pdev->bus;
- hose = pci_bus_to_host(bus);
- phb = hose->private_data;
- pdn = pci_get_pdn(pdev);
- total_vfs = pci_sriov_get_totalvfs(pdev);
-
- if (pdn->m64_single_mode)
- m64_bars = num_vfs;
- else
- m64_bars = 1;
-
- pdn->m64_map = kmalloc_array(m64_bars,
- sizeof(*pdn->m64_map),
- GFP_KERNEL);
- if (!pdn->m64_map)
- return -ENOMEM;
- /* Initialize the m64_map to IODA_INVALID_M64 */
- for (i = 0; i < m64_bars ; i++)
- for (j = 0; j < PCI_SRIOV_NUM_BARS; j++)
- pdn->m64_map[i][j] = IODA_INVALID_M64;
-
-
- for (i = 0; i < PCI_SRIOV_NUM_BARS; i++) {
- res = &pdev->resource[i + PCI_IOV_RESOURCES];
- if (!res->flags || !res->parent)
- continue;
-
- for (j = 0; j < m64_bars; j++) {
- do {
- win = find_next_zero_bit(&phb->ioda.m64_bar_alloc,
- phb->ioda.m64_bar_idx + 1, 0);
-
- if (win >= phb->ioda.m64_bar_idx + 1)
- goto m64_failed;
- } while (test_and_set_bit(win, &phb->ioda.m64_bar_alloc));
-
- pdn->m64_map[j][i] = win;
-
- if (pdn->m64_single_mode) {
- size = pci_iov_resource_size(pdev,
- PCI_IOV_RESOURCES + i);
- start = res->start + size * j;
- } else {
- size = resource_size(res);
- start = res->start;
- }
-
- /* Map the M64 here */
- if (pdn->m64_single_mode) {
- pe_num = pdn->pe_num_map[j];
- rc = opal_pci_map_pe_mmio_window(phb->opal_id,
- pe_num, OPAL_M64_WINDOW_TYPE,
- pdn->m64_map[j][i], 0);
- }
-
- rc = opal_pci_set_phb_mem_window(phb->opal_id,
- OPAL_M64_WINDOW_TYPE,
- pdn->m64_map[j][i],
- start,
- 0, /* unused */
- size);
-
-
- if (rc != OPAL_SUCCESS) {
- dev_err(&pdev->dev, "Failed to map M64 window #%d: %lld\n",
- win, rc);
- goto m64_failed;
- }
-
- if (pdn->m64_single_mode)
- rc = opal_pci_phb_mmio_enable(phb->opal_id,
- OPAL_M64_WINDOW_TYPE, pdn->m64_map[j][i], 2);
- else
- rc = opal_pci_phb_mmio_enable(phb->opal_id,
- OPAL_M64_WINDOW_TYPE, pdn->m64_map[j][i], 1);
-
- if (rc != OPAL_SUCCESS) {
- dev_err(&pdev->dev, "Failed to enable M64 window #%d: %llx\n",
- win, rc);
- goto m64_failed;
- }
- }
- }
- return 0;
-
-m64_failed:
- pnv_pci_vf_release_m64(pdev, num_vfs);
- return -EBUSY;
-}
-
-static long pnv_pci_ioda2_unset_window(struct iommu_table_group *table_group,
- int num);
-
-static void pnv_pci_ioda2_release_dma_pe(struct pci_dev *dev, struct pnv_ioda_pe *pe)
-{
- struct iommu_table *tbl;
- int64_t rc;
-
- tbl = pe->table_group.tables[0];
- rc = pnv_pci_ioda2_unset_window(&pe->table_group, 0);
- if (rc)
- pe_warn(pe, "OPAL error %lld release DMA window\n", rc);
-
- pnv_pci_ioda2_set_bypass(pe, false);
- if (pe->table_group.group) {
- iommu_group_put(pe->table_group.group);
- BUG_ON(pe->table_group.group);
- }
- iommu_tce_table_put(tbl);
-}
-
-static void pnv_ioda_release_vf_PE(struct pci_dev *pdev)
-{
- struct pci_bus *bus;
- struct pci_controller *hose;
- struct pnv_phb *phb;
- struct pnv_ioda_pe *pe, *pe_n;
- struct pci_dn *pdn;
-
- bus = pdev->bus;
- hose = pci_bus_to_host(bus);
- phb = hose->private_data;
- pdn = pci_get_pdn(pdev);
-
- if (!pdev->is_physfn)
- return;
-
- list_for_each_entry_safe(pe, pe_n, &phb->ioda.pe_list, list) {
- if (pe->parent_dev != pdev)
- continue;
-
- pnv_pci_ioda2_release_dma_pe(pdev, pe);
-
- /* Remove from list */
- mutex_lock(&phb->ioda.pe_list_mutex);
- list_del(&pe->list);
- mutex_unlock(&phb->ioda.pe_list_mutex);
-
- pnv_ioda_deconfigure_pe(phb, pe);
-
- pnv_ioda_free_pe(pe);
- }
-}
-
-void pnv_pci_sriov_disable(struct pci_dev *pdev)
-{
- struct pci_bus *bus;
- struct pci_controller *hose;
- struct pnv_phb *phb;
- struct pnv_ioda_pe *pe;
- struct pci_dn *pdn;
- u16 num_vfs, i;
-
- bus = pdev->bus;
- hose = pci_bus_to_host(bus);
- phb = hose->private_data;
- pdn = pci_get_pdn(pdev);
- num_vfs = pdn->num_vfs;
-
- /* Release VF PEs */
- pnv_ioda_release_vf_PE(pdev);
-
- if (phb->type == PNV_PHB_IODA2) {
- if (!pdn->m64_single_mode)
- pnv_pci_vf_resource_shift(pdev, -*pdn->pe_num_map);
-
- /* Release M64 windows */
- pnv_pci_vf_release_m64(pdev, num_vfs);
-
- /* Release PE numbers */
- if (pdn->m64_single_mode) {
- for (i = 0; i < num_vfs; i++) {
- if (pdn->pe_num_map[i] == IODA_INVALID_PE)
- continue;
-
- pe = &phb->ioda.pe_array[pdn->pe_num_map[i]];
- pnv_ioda_free_pe(pe);
- }
- } else
- bitmap_clear(phb->ioda.pe_alloc, *pdn->pe_num_map, num_vfs);
- /* Releasing pe_num_map */
- kfree(pdn->pe_num_map);
- }
-}
-
-static void pnv_pci_ioda2_setup_dma_pe(struct pnv_phb *phb,
+static void pnv_pci_ioda1_setup_dma_pe(struct pnv_phb *phb,
struct pnv_ioda_pe *pe);
-static void pnv_ioda_setup_vf_PE(struct pci_dev *pdev, u16 num_vfs)
-{
- struct pci_bus *bus;
- struct pci_controller *hose;
- struct pnv_phb *phb;
- struct pnv_ioda_pe *pe;
- int pe_num;
- u16 vf_index;
- struct pci_dn *pdn;
-
- bus = pdev->bus;
- hose = pci_bus_to_host(bus);
- phb = hose->private_data;
- pdn = pci_get_pdn(pdev);
-
- if (!pdev->is_physfn)
- return;
-
- /* Reserve PE for each VF */
- for (vf_index = 0; vf_index < num_vfs; vf_index++) {
- int vf_devfn = pci_iov_virtfn_devfn(pdev, vf_index);
- int vf_bus = pci_iov_virtfn_bus(pdev, vf_index);
- struct pci_dn *vf_pdn;
-
- if (pdn->m64_single_mode)
- pe_num = pdn->pe_num_map[vf_index];
- else
- pe_num = *pdn->pe_num_map + vf_index;
-
- pe = &phb->ioda.pe_array[pe_num];
- pe->pe_number = pe_num;
- pe->phb = phb;
- pe->flags = PNV_IODA_PE_VF;
- pe->pbus = NULL;
- pe->parent_dev = pdev;
- pe->mve_number = -1;
- pe->rid = (vf_bus << 8) | vf_devfn;
-
- pe_info(pe, "VF %04d:%02d:%02d.%d associated with PE#%x\n",
- hose->global_number, pdev->bus->number,
- PCI_SLOT(vf_devfn), PCI_FUNC(vf_devfn), pe_num);
-
- if (pnv_ioda_configure_pe(phb, pe)) {
- /* XXX What do we do here ? */
- pnv_ioda_free_pe(pe);
- pe->pdev = NULL;
- continue;
- }
-
- /* Put PE to the list */
- mutex_lock(&phb->ioda.pe_list_mutex);
- list_add_tail(&pe->list, &phb->ioda.pe_list);
- mutex_unlock(&phb->ioda.pe_list_mutex);
-
- /* associate this pe to it's pdn */
- list_for_each_entry(vf_pdn, &pdn->parent->child_list, list) {
- if (vf_pdn->busno == vf_bus &&
- vf_pdn->devfn == vf_devfn) {
- vf_pdn->pe_number = pe_num;
- break;
- }
- }
-
- pnv_pci_ioda2_setup_dma_pe(phb, pe);
- }
-}
-
-int pnv_pci_sriov_enable(struct pci_dev *pdev, u16 num_vfs)
-{
- struct pci_bus *bus;
- struct pci_controller *hose;
- struct pnv_phb *phb;
- struct pnv_ioda_pe *pe;
- struct pci_dn *pdn;
- int ret;
- u16 i;
-
- bus = pdev->bus;
- hose = pci_bus_to_host(bus);
- phb = hose->private_data;
- pdn = pci_get_pdn(pdev);
-
- if (phb->type == PNV_PHB_IODA2) {
- if (!pdn->vfs_expanded) {
- dev_info(&pdev->dev, "don't support this SRIOV device"
- " with non 64bit-prefetchable IOV BAR\n");
- return -ENOSPC;
- }
-
- /*
- * When M64 BARs functions in Single PE mode, the number of VFs
- * could be enabled must be less than the number of M64 BARs.
- */
- if (pdn->m64_single_mode && num_vfs > phb->ioda.m64_bar_idx) {
- dev_info(&pdev->dev, "Not enough M64 BAR for VFs\n");
- return -EBUSY;
- }
-
- /* Allocating pe_num_map */
- if (pdn->m64_single_mode)
- pdn->pe_num_map = kmalloc_array(num_vfs,
- sizeof(*pdn->pe_num_map),
- GFP_KERNEL);
- else
- pdn->pe_num_map = kmalloc(sizeof(*pdn->pe_num_map), GFP_KERNEL);
-
- if (!pdn->pe_num_map)
- return -ENOMEM;
-
- if (pdn->m64_single_mode)
- for (i = 0; i < num_vfs; i++)
- pdn->pe_num_map[i] = IODA_INVALID_PE;
-
- /* Calculate available PE for required VFs */
- if (pdn->m64_single_mode) {
- for (i = 0; i < num_vfs; i++) {
- pe = pnv_ioda_alloc_pe(phb);
- if (!pe) {
- ret = -EBUSY;
- goto m64_failed;
- }
-
- pdn->pe_num_map[i] = pe->pe_number;
- }
- } else {
- mutex_lock(&phb->ioda.pe_alloc_mutex);
- *pdn->pe_num_map = bitmap_find_next_zero_area(
- phb->ioda.pe_alloc, phb->ioda.total_pe_num,
- 0, num_vfs, 0);
- if (*pdn->pe_num_map >= phb->ioda.total_pe_num) {
- mutex_unlock(&phb->ioda.pe_alloc_mutex);
- dev_info(&pdev->dev, "Failed to enable VF%d\n", num_vfs);
- kfree(pdn->pe_num_map);
- return -EBUSY;
- }
- bitmap_set(phb->ioda.pe_alloc, *pdn->pe_num_map, num_vfs);
- mutex_unlock(&phb->ioda.pe_alloc_mutex);
- }
- pdn->num_vfs = num_vfs;
-
- /* Assign M64 window accordingly */
- ret = pnv_pci_vf_assign_m64(pdev, num_vfs);
- if (ret) {
- dev_info(&pdev->dev, "Not enough M64 window resources\n");
- goto m64_failed;
- }
-
- /*
- * When using one M64 BAR to map one IOV BAR, we need to shift
- * the IOV BAR according to the PE# allocated to the VFs.
- * Otherwise, the PE# for the VF will conflict with others.
- */
- if (!pdn->m64_single_mode) {
- ret = pnv_pci_vf_resource_shift(pdev, *pdn->pe_num_map);
- if (ret)
- goto m64_failed;
- }
- }
-
- /* Setup VF PEs */
- pnv_ioda_setup_vf_PE(pdev, num_vfs);
-
- return 0;
-
-m64_failed:
- if (pdn->m64_single_mode) {
- for (i = 0; i < num_vfs; i++) {
- if (pdn->pe_num_map[i] == IODA_INVALID_PE)
- continue;
-
- pe = &phb->ioda.pe_array[pdn->pe_num_map[i]];
- pnv_ioda_free_pe(pe);
- }
- } else
- bitmap_clear(phb->ioda.pe_alloc, *pdn->pe_num_map, num_vfs);
-
- /* Releasing pe_num_map */
- kfree(pdn->pe_num_map);
-
- return ret;
-}
-
-int pnv_pcibios_sriov_disable(struct pci_dev *pdev)
-{
- pnv_pci_sriov_disable(pdev);
-
- /* Release PCI data */
- remove_sriov_vf_pdns(pdev);
- return 0;
-}
-
-int pnv_pcibios_sriov_enable(struct pci_dev *pdev, u16 num_vfs)
-{
- /* Allocate PCI data */
- add_sriov_vf_pdns(pdev);
-
- return pnv_pci_sriov_enable(pdev, num_vfs);
-}
-#endif /* CONFIG_PCI_IOV */
static void pnv_pci_ioda_dma_dev_setup(struct pci_dev *pdev)
{
- struct pci_controller *hose = pci_bus_to_host(pdev->bus);
- struct pnv_phb *phb = hose->private_data;
+ struct pnv_phb *phb = pci_bus_to_pnvhb(pdev->bus);
struct pci_dn *pdn = pci_get_pdn(pdev);
struct pnv_ioda_pe *pe;
@@ -1762,6 +1245,24 @@
pci_info(pdev, "Added to existing PE#%x\n", pe->pe_number);
}
+ /*
+ * We assume that bridges *probably* don't need to do any DMA so we can
+ * skip allocating a TCE table, etc unless we get a non-bridge device.
+ */
+ if (!pe->dma_setup_done && !pci_is_bridge(pdev)) {
+ switch (phb->type) {
+ case PNV_PHB_IODA1:
+ pnv_pci_ioda1_setup_dma_pe(phb, pe);
+ break;
+ case PNV_PHB_IODA2:
+ pnv_pci_ioda2_setup_dma_pe(phb, pe);
+ break;
+ default:
+ pr_warn("%s: No DMA for PHB#%x (type %d)\n",
+ __func__, phb->hose->global_number, phb->type);
+ }
+ }
+
if (pdn)
pdn->pe_number = pe->pe_number;
pe->device_count++;
@@ -1847,8 +1348,7 @@
static bool pnv_pci_ioda_iommu_bypass_supported(struct pci_dev *pdev,
u64 dma_mask)
{
- struct pci_controller *hose = pci_bus_to_host(pdev->bus);
- struct pnv_phb *phb = hose->private_data;
+ struct pnv_phb *phb = pci_bus_to_pnvhb(pdev->bus);
struct pci_dn *pdn = pci_get_pdn(pdev);
struct pnv_ioda_pe *pe;
@@ -1885,19 +1385,6 @@
return false;
}
-static void pnv_ioda_setup_bus_dma(struct pnv_ioda_pe *pe, struct pci_bus *bus)
-{
- struct pci_dev *dev;
-
- list_for_each_entry(dev, &bus->devices, bus_list) {
- set_iommu_table_base(&dev->dev, pe->table_group.tables[0]);
- dev->dev.archdata.dma_offset = pe->tce_bypass_base;
-
- if ((pe->flags & PNV_IODA_PE_BUS_ALL) && dev->subordinate)
- pnv_ioda_setup_bus_dma(pe, dev->subordinate);
- }
-}
-
static inline __be64 __iomem *pnv_ioda_get_inval_reg(struct pnv_phb *phb,
bool real_mode)
{
@@ -2285,6 +1772,7 @@
pe->table_group.tce32_size = tbl->it_size << tbl->it_page_shift;
iommu_init_table(tbl, phb->hose->node, 0, 0);
+ pe->dma_setup_done = true;
return;
fail:
/* XXX Failure: Try to fallback to 64-bit only ? */
@@ -2474,7 +1962,6 @@
return 0;
}
-#if defined(CONFIG_IOMMU_API) || defined(CONFIG_PCI_IOV)
static long pnv_pci_ioda2_unset_window(struct iommu_table_group *table_group,
int num)
{
@@ -2498,7 +1985,6 @@
return ret;
}
-#endif
#ifdef CONFIG_IOMMU_API
unsigned long pnv_pci_ioda2_get_table_size(__u32 page_shift,
@@ -2547,6 +2033,19 @@
return ret;
}
+static void pnv_ioda_setup_bus_dma(struct pnv_ioda_pe *pe, struct pci_bus *bus)
+{
+ struct pci_dev *dev;
+
+ list_for_each_entry(dev, &bus->devices, bus_list) {
+ set_iommu_table_base(&dev->dev, pe->table_group.tables[0]);
+ dev->dev.archdata.dma_offset = pe->tce_bypass_base;
+
+ if ((pe->flags & PNV_IODA_PE_BUS_ALL) && dev->subordinate)
+ pnv_ioda_setup_bus_dma(pe, dev->subordinate);
+ }
+}
+
static void pnv_ioda2_take_ownership(struct iommu_table_group *table_group)
{
struct pnv_ioda_pe *pe = container_of(table_group, struct pnv_ioda_pe,
@@ -2583,14 +2082,11 @@
};
#endif
-static void pnv_pci_ioda2_setup_dma_pe(struct pnv_phb *phb,
- struct pnv_ioda_pe *pe)
+void pnv_pci_ioda2_setup_dma_pe(struct pnv_phb *phb,
+ struct pnv_ioda_pe *pe)
{
int64_t rc;
- if (!pnv_pci_ioda_pe_dma_weight(pe))
- return;
-
/* TVE #1 is selected by PCI address bit 59 */
pe->tce_bypass_base = 1ull << 59;
@@ -2615,6 +2111,7 @@
iommu_register_group(&pe->table_group, phb->hose->global_number,
pe->pe_number);
#endif
+ pe->dma_setup_done = true;
}
int64_t pnv_opal_pci_msi_eoi(struct irq_chip *chip, unsigned int hw_irq)
@@ -2763,118 +2260,6 @@
count, phb->msi_base);
}
-#ifdef CONFIG_PCI_IOV
-static void pnv_pci_ioda_fixup_iov_resources(struct pci_dev *pdev)
-{
- struct pci_controller *hose = pci_bus_to_host(pdev->bus);
- struct pnv_phb *phb = hose->private_data;
- const resource_size_t gate = phb->ioda.m64_segsize >> 2;
- struct resource *res;
- int i;
- resource_size_t size, total_vf_bar_sz;
- struct pci_dn *pdn;
- int mul, total_vfs;
-
- pdn = pci_get_pdn(pdev);
- pdn->vfs_expanded = 0;
- pdn->m64_single_mode = false;
-
- total_vfs = pci_sriov_get_totalvfs(pdev);
- mul = phb->ioda.total_pe_num;
- total_vf_bar_sz = 0;
-
- for (i = 0; i < PCI_SRIOV_NUM_BARS; i++) {
- res = &pdev->resource[i + PCI_IOV_RESOURCES];
- if (!res->flags || res->parent)
- continue;
- if (!pnv_pci_is_m64_flags(res->flags)) {
- dev_warn(&pdev->dev, "Don't support SR-IOV with"
- " non M64 VF BAR%d: %pR. \n",
- i, res);
- goto truncate_iov;
- }
-
- total_vf_bar_sz += pci_iov_resource_size(pdev,
- i + PCI_IOV_RESOURCES);
-
- /*
- * If bigger than quarter of M64 segment size, just round up
- * power of two.
- *
- * Generally, one M64 BAR maps one IOV BAR. To avoid conflict
- * with other devices, IOV BAR size is expanded to be
- * (total_pe * VF_BAR_size). When VF_BAR_size is half of M64
- * segment size , the expanded size would equal to half of the
- * whole M64 space size, which will exhaust the M64 Space and
- * limit the system flexibility. This is a design decision to
- * set the boundary to quarter of the M64 segment size.
- */
- if (total_vf_bar_sz > gate) {
- mul = roundup_pow_of_two(total_vfs);
- dev_info(&pdev->dev,
- "VF BAR Total IOV size %llx > %llx, roundup to %d VFs\n",
- total_vf_bar_sz, gate, mul);
- pdn->m64_single_mode = true;
- break;
- }
- }
-
- for (i = 0; i < PCI_SRIOV_NUM_BARS; i++) {
- res = &pdev->resource[i + PCI_IOV_RESOURCES];
- if (!res->flags || res->parent)
- continue;
-
- size = pci_iov_resource_size(pdev, i + PCI_IOV_RESOURCES);
- /*
- * On PHB3, the minimum size alignment of M64 BAR in single
- * mode is 32MB.
- */
- if (pdn->m64_single_mode && (size < SZ_32M))
- goto truncate_iov;
- dev_dbg(&pdev->dev, " Fixing VF BAR%d: %pR to\n", i, res);
- res->end = res->start + size * mul - 1;
- dev_dbg(&pdev->dev, " %pR\n", res);
- dev_info(&pdev->dev, "VF BAR%d: %pR (expanded to %d VFs for PE alignment)",
- i, res, mul);
- }
- pdn->vfs_expanded = mul;
-
- return;
-
-truncate_iov:
- /* To save MMIO space, IOV BAR is truncated. */
- for (i = 0; i < PCI_SRIOV_NUM_BARS; i++) {
- res = &pdev->resource[i + PCI_IOV_RESOURCES];
- res->flags = 0;
- res->end = res->start - 1;
- }
-}
-
-static void pnv_pci_ioda_fixup_iov(struct pci_dev *pdev)
-{
- if (WARN_ON(pci_dev_is_added(pdev)))
- return;
-
- if (pdev->is_virtfn) {
- struct pnv_ioda_pe *pe = pnv_ioda_get_pe(pdev);
-
- /*
- * VF PEs are single-device PEs so their pdev pointer needs to
- * be set. The pdev doesn't exist when the PE is allocated (in
- * (pcibios_sriov_enable()) so we fix it up here.
- */
- pe->pdev = pdev;
- WARN_ON(!(pe->flags & PNV_IODA_PE_VF));
- } else if (pdev->is_physfn) {
- /*
- * For PFs adjust their allocated IOV resources to match what
- * the PHB can support using it's M64 BAR table.
- */
- pnv_pci_ioda_fixup_iov_resources(pdev);
- }
-}
-#endif /* CONFIG_PCI_IOV */
-
static void pnv_ioda_setup_pe_res(struct pnv_ioda_pe *pe,
struct resource *res)
{
@@ -3101,10 +2486,9 @@
static resource_size_t pnv_pci_window_alignment(struct pci_bus *bus,
unsigned long type)
{
- struct pci_dev *bridge;
- struct pci_controller *hose = pci_bus_to_host(bus);
- struct pnv_phb *phb = hose->private_data;
+ struct pnv_phb *phb = pci_bus_to_pnvhb(bus);
int num_pci_bridges = 0;
+ struct pci_dev *bridge;
bridge = bus->self;
while (bridge) {
@@ -3190,8 +2574,6 @@
static void pnv_pci_configure_bus(struct pci_bus *bus)
{
- struct pci_controller *hose = pci_bus_to_host(bus);
- struct pnv_phb *phb = hose->private_data;
struct pci_dev *bridge = bus->self;
struct pnv_ioda_pe *pe;
bool all = (bridge && pci_pcie_type(bridge) == PCI_EXP_TYPE_PCI_BRIDGE);
@@ -3215,17 +2597,6 @@
return;
pnv_ioda_setup_pe_seg(pe);
- switch (phb->type) {
- case PNV_PHB_IODA1:
- pnv_pci_ioda1_setup_dma_pe(phb, pe);
- break;
- case PNV_PHB_IODA2:
- pnv_pci_ioda2_setup_dma_pe(phb, pe);
- break;
- default:
- pr_warn("%s: No DMA for PHB#%x (type %d)\n",
- __func__, phb->hose->global_number, phb->type);
- }
}
static resource_size_t pnv_pci_default_alignment(void)
@@ -3233,49 +2604,12 @@
return PAGE_SIZE;
}
-#ifdef CONFIG_PCI_IOV
-static resource_size_t pnv_pci_iov_resource_alignment(struct pci_dev *pdev,
- int resno)
-{
- struct pci_controller *hose = pci_bus_to_host(pdev->bus);
- struct pnv_phb *phb = hose->private_data;
- struct pci_dn *pdn = pci_get_pdn(pdev);
- resource_size_t align;
-
- /*
- * On PowerNV platform, IOV BAR is mapped by M64 BAR to enable the
- * SR-IOV. While from hardware perspective, the range mapped by M64
- * BAR should be size aligned.
- *
- * When IOV BAR is mapped with M64 BAR in Single PE mode, the extra
- * powernv-specific hardware restriction is gone. But if just use the
- * VF BAR size as the alignment, PF BAR / VF BAR may be allocated with
- * in one segment of M64 #15, which introduces the PE conflict between
- * PF and VF. Based on this, the minimum alignment of an IOV BAR is
- * m64_segsize.
- *
- * This function returns the total IOV BAR size if M64 BAR is in
- * Shared PE mode or just VF BAR size if not.
- * If the M64 BAR is in Single PE mode, return the VF BAR size or
- * M64 segment size if IOV BAR size is less.
- */
- align = pci_iov_resource_size(pdev, resno);
- if (!pdn->vfs_expanded)
- return align;
- if (pdn->m64_single_mode)
- return max(align, (resource_size_t)phb->ioda.m64_segsize);
-
- return pdn->vfs_expanded * align;
-}
-#endif /* CONFIG_PCI_IOV */
-
/* Prevent enabling devices for which we couldn't properly
* assign a PE
*/
static bool pnv_pci_enable_device_hook(struct pci_dev *dev)
{
- struct pci_controller *hose = pci_bus_to_host(dev->bus);
- struct pnv_phb *phb = hose->private_data;
+ struct pnv_phb *phb = pci_bus_to_pnvhb(dev->bus);
struct pci_dn *pdn;
/* The function is probably called while the PEs have
@@ -3346,11 +2680,10 @@
static void pnv_pci_ioda1_release_pe_dma(struct pnv_ioda_pe *pe)
{
- unsigned int weight = pnv_pci_ioda_pe_dma_weight(pe);
struct iommu_table *tbl = pe->table_group.tables[0];
int64_t rc;
- if (!weight)
+ if (!pe->dma_setup_done)
return;
rc = pnv_pci_ioda1_unset_window(&pe->table_group, 0);
@@ -3367,22 +2700,17 @@
iommu_tce_table_put(tbl);
}
-static void pnv_pci_ioda2_release_pe_dma(struct pnv_ioda_pe *pe)
+void pnv_pci_ioda2_release_pe_dma(struct pnv_ioda_pe *pe)
{
struct iommu_table *tbl = pe->table_group.tables[0];
- unsigned int weight = pnv_pci_ioda_pe_dma_weight(pe);
-#ifdef CONFIG_IOMMU_API
int64_t rc;
-#endif
- if (!weight)
+ if (pe->dma_setup_done)
return;
-#ifdef CONFIG_IOMMU_API
rc = pnv_pci_ioda2_unset_window(&pe->table_group, 0);
if (rc)
pe_warn(pe, "OPAL error %lld release DMA window\n", rc);
-#endif
pnv_pci_ioda2_set_bypass(pe, false);
if (pe->table_group.group) {
@@ -3405,14 +2733,8 @@
if (map[idx] != pe->pe_number)
continue;
- if (win == OPAL_M64_WINDOW_TYPE)
- rc = opal_pci_map_pe_mmio_window(phb->opal_id,
- phb->ioda.reserved_pe_idx, win,
- idx / PNV_IODA1_M64_SEGS,
- idx % PNV_IODA1_M64_SEGS);
- else
- rc = opal_pci_map_pe_mmio_window(phb->opal_id,
- phb->ioda.reserved_pe_idx, win, 0, idx);
+ rc = opal_pci_map_pe_mmio_window(phb->opal_id,
+ phb->ioda.reserved_pe_idx, win, 0, idx);
if (rc != OPAL_SUCCESS)
pe_warn(pe, "Error %lld unmapping (%d) segment#%d\n",
@@ -3431,8 +2753,7 @@
phb->ioda.io_segmap);
pnv_ioda_free_pe_seg(pe, OPAL_M32_WINDOW_TYPE,
phb->ioda.m32_segmap);
- pnv_ioda_free_pe_seg(pe, OPAL_M64_WINDOW_TYPE,
- phb->ioda.m64_segmap);
+ /* M64 is pre-configured by pnv_ioda1_init_m64() */
} else if (phb->type == PNV_PHB_IODA2) {
pnv_ioda_free_pe_seg(pe, OPAL_M32_WINDOW_TYPE,
phb->ioda.m32_segmap);
@@ -3488,17 +2809,27 @@
static void pnv_pci_release_device(struct pci_dev *pdev)
{
- struct pci_controller *hose = pci_bus_to_host(pdev->bus);
- struct pnv_phb *phb = hose->private_data;
+ struct pnv_phb *phb = pci_bus_to_pnvhb(pdev->bus);
struct pci_dn *pdn = pci_get_pdn(pdev);
struct pnv_ioda_pe *pe;
+ /* The VF PE state is torn down when sriov_disable() is called */
if (pdev->is_virtfn)
return;
if (!pdn || pdn->pe_number == IODA_INVALID_PE)
return;
+#ifdef CONFIG_PCI_IOV
+ /*
+ * FIXME: Try move this to sriov_disable(). It's here since we allocate
+ * the iov state at probe time since we need to fiddle with the IOV
+ * resources.
+ */
+ if (pdev->is_physfn)
+ kfree(pdev->dev.archdata.iov_data);
+#endif
+
/*
* PCI hotplug can happen as part of EEH error recovery. The @pdn
* isn't removed and added afterwards in this scenario. We should
@@ -3534,8 +2865,7 @@
static void pnv_pci_ioda_dma_bus_setup(struct pci_bus *bus)
{
- struct pci_controller *hose = bus->sysdata;
- struct pnv_phb *phb = hose->private_data;
+ struct pnv_phb *phb = pci_bus_to_pnvhb(bus);
struct pnv_ioda_pe *pe;
list_for_each_entry(pe, &phb->ioda.pe_list, list) {
@@ -3760,7 +3090,7 @@
pnv_ioda_reserve_pe(phb, phb->ioda.root_pe_idx);
} else {
/* otherwise just allocate one */
- root_pe = pnv_ioda_alloc_pe(phb);
+ root_pe = pnv_ioda_alloc_pe(phb, 1);
phb->ioda.root_pe_idx = root_pe->pe_number;
}
@@ -3873,8 +3203,7 @@
static void pnv_npu2_opencapi_cfg_size_fixup(struct pci_dev *dev)
{
- struct pci_controller *hose = pci_bus_to_host(dev->bus);
- struct pnv_phb *phb = hose->private_data;
+ struct pnv_phb *phb = pci_bus_to_pnvhb(dev->bus);
if (!machine_is(powernv))
return;
diff --git a/arch/powerpc/platforms/powernv/pci-sriov.c b/arch/powerpc/platforms/powernv/pci-sriov.c
new file mode 100644
index 0000000..c4434f2
--- /dev/null
+++ b/arch/powerpc/platforms/powernv/pci-sriov.c
@@ -0,0 +1,766 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+
+#include <linux/kernel.h>
+#include <linux/ioport.h>
+#include <linux/bitmap.h>
+#include <linux/pci.h>
+
+#include <asm/opal.h>
+
+#include "pci.h"
+
+/* for pci_dev_is_added() */
+#include "../../../../drivers/pci/pci.h"
+
+/*
+ * The majority of the complexity in supporting SR-IOV on PowerNV comes from
+ * the need to put the MMIO space for each VF into a separate PE. Internally
+ * the PHB maps MMIO addresses to a specific PE using the "Memory BAR Table".
+ * The MBT historically only applied to the 64bit MMIO window of the PHB
+ * so it's common to see it referred to as the "M64BT".
+ *
+ * An MBT entry stores the mapped range as an <base>,<mask> pair. This forces
+ * the address range that we want to map to be power-of-two sized and aligned.
+ * For conventional PCI devices this isn't really an issue since PCI device BARs
+ * have the same requirement.
+ *
+ * For a SR-IOV BAR things are a little more awkward since size and alignment
+ * are not coupled. The alignment is set based on the the per-VF BAR size, but
+ * the total BAR area is: number-of-vfs * per-vf-size. The number of VFs
+ * isn't necessarily a power of two, so neither is the total size. To fix that
+ * we need to finesse (read: hack) the Linux BAR allocator so that it will
+ * allocate the SR-IOV BARs in a way that lets us map them using the MBT.
+ *
+ * The changes to size and alignment that we need to do depend on the "mode"
+ * of MBT entry that we use. We only support SR-IOV on PHB3 (IODA2) and above,
+ * so as a baseline we can assume that we have the following BAR modes
+ * available:
+ *
+ * NB: $PE_COUNT is the number of PEs that the PHB supports.
+ *
+ * a) A segmented BAR that splits the mapped range into $PE_COUNT equally sized
+ * segments. The n'th segment is mapped to the n'th PE.
+ * b) An un-segmented BAR that maps the whole address range to a specific PE.
+ *
+ *
+ * We prefer to use mode a) since it only requires one MBT entry per SR-IOV BAR
+ * For comparison b) requires one entry per-VF per-BAR, or:
+ * (num-vfs * num-sriov-bars) in total. To use a) we need the size of each segment
+ * to equal the size of the per-VF BAR area. So:
+ *
+ * new_size = per-vf-size * number-of-PEs
+ *
+ * The alignment for the SR-IOV BAR also needs to be changed from per-vf-size
+ * to "new_size", calculated above. Implementing this is a convoluted process
+ * which requires several hooks in the PCI core:
+ *
+ * 1. In pcibios_add_device() we call pnv_pci_ioda_fixup_iov().
+ *
+ * At this point the device has been probed and the device's BARs are sized,
+ * but no resource allocations have been done. The SR-IOV BARs are sized
+ * based on the maximum number of VFs supported by the device and we need
+ * to increase that to new_size.
+ *
+ * 2. Later, when Linux actually assigns resources it tries to make the resource
+ * allocations for each PCI bus as compact as possible. As a part of that it
+ * sorts the BARs on a bus by their required alignment, which is calculated
+ * using pci_resource_alignment().
+ *
+ * For IOV resources this goes:
+ * pci_resource_alignment()
+ * pci_sriov_resource_alignment()
+ * pcibios_sriov_resource_alignment()
+ * pnv_pci_iov_resource_alignment()
+ *
+ * Our hook overrides the default alignment, equal to the per-vf-size, with
+ * new_size computed above.
+ *
+ * 3. When userspace enables VFs for a device:
+ *
+ * sriov_enable()
+ * pcibios_sriov_enable()
+ * pnv_pcibios_sriov_enable()
+ *
+ * This is where we actually allocate PE numbers for each VF and setup the
+ * MBT mapping for each SR-IOV BAR. In steps 1) and 2) we setup an "arena"
+ * where each MBT segment is equal in size to the VF BAR so we can shift
+ * around the actual SR-IOV BAR location within this arena. We need this
+ * ability because the PE space is shared by all devices on the same PHB.
+ * When using mode a) described above segment 0 in maps to PE#0 which might
+ * be already being used by another device on the PHB.
+ *
+ * As a result we need allocate a contigious range of PE numbers, then shift
+ * the address programmed into the SR-IOV BAR of the PF so that the address
+ * of VF0 matches up with the segment corresponding to the first allocated
+ * PE number. This is handled in pnv_pci_vf_resource_shift().
+ *
+ * Once all that is done we return to the PCI core which then enables VFs,
+ * scans them and creates pci_devs for each. The init process for a VF is
+ * largely the same as a normal device, but the VF is inserted into the IODA
+ * PE that we allocated for it rather than the PE associated with the bus.
+ *
+ * 4. When userspace disables VFs we unwind the above in
+ * pnv_pcibios_sriov_disable(). Fortunately this is relatively simple since
+ * we don't need to validate anything, just tear down the mappings and
+ * move SR-IOV resource back to its "proper" location.
+ *
+ * That's how mode a) works. In theory mode b) (single PE mapping) is less work
+ * since we can map each individual VF with a separate BAR. However, there's a
+ * few limitations:
+ *
+ * 1) For IODA2 mode b) has a minimum alignment requirement of 32MB. This makes
+ * it only usable for devices with very large per-VF BARs. Such devices are
+ * similar to Big Foot. They definitely exist, but I've never seen one.
+ *
+ * 2) The number of MBT entries that we have is limited. PHB3 and PHB4 only
+ * 16 total and some are needed for. Most SR-IOV capable network cards can support
+ * more than 16 VFs on each port.
+ *
+ * We use b) when using a) would use more than 1/4 of the entire 64 bit MMIO
+ * window of the PHB.
+ *
+ *
+ *
+ * PHB4 (IODA3) added a few new features that would be useful for SR-IOV. It
+ * allowed the MBT to map 32bit MMIO space in addition to 64bit which allows
+ * us to support SR-IOV BARs in the 32bit MMIO window. This is useful since
+ * the Linux BAR allocation will place any BAR marked as non-prefetchable into
+ * the non-prefetchable bridge window, which is 32bit only. It also added two
+ * new modes:
+ *
+ * c) A segmented BAR similar to a), but each segment can be individually
+ * mapped to any PE. This is matches how the 32bit MMIO window worked on
+ * IODA1&2.
+ *
+ * d) A segmented BAR with 8, 64, or 128 segments. This works similarly to a),
+ * but with fewer segments and configurable base PE.
+ *
+ * i.e. The n'th segment maps to the (n + base)'th PE.
+ *
+ * The base PE is also required to be a multiple of the window size.
+ *
+ * Unfortunately, the OPAL API doesn't currently (as of skiboot v6.6) allow us
+ * to exploit any of the IODA3 features.
+ */
+
+static void pnv_pci_ioda_fixup_iov_resources(struct pci_dev *pdev)
+{
+ struct pnv_phb *phb = pci_bus_to_pnvhb(pdev->bus);
+ struct resource *res;
+ int i;
+ resource_size_t vf_bar_sz;
+ struct pnv_iov_data *iov;
+ int mul;
+
+ iov = kzalloc(sizeof(*iov), GFP_KERNEL);
+ if (!iov)
+ goto disable_iov;
+ pdev->dev.archdata.iov_data = iov;
+ mul = phb->ioda.total_pe_num;
+
+ for (i = 0; i < PCI_SRIOV_NUM_BARS; i++) {
+ res = &pdev->resource[i + PCI_IOV_RESOURCES];
+ if (!res->flags || res->parent)
+ continue;
+ if (!pnv_pci_is_m64_flags(res->flags)) {
+ dev_warn(&pdev->dev, "Don't support SR-IOV with non M64 VF BAR%d: %pR. \n",
+ i, res);
+ goto disable_iov;
+ }
+
+ vf_bar_sz = pci_iov_resource_size(pdev, i + PCI_IOV_RESOURCES);
+
+ /*
+ * Generally, one segmented M64 BAR maps one IOV BAR. However,
+ * if a VF BAR is too large we end up wasting a lot of space.
+ * If each VF needs more than 1/4 of the default m64 segment
+ * then each VF BAR should be mapped in single-PE mode to reduce
+ * the amount of space required. This does however limit the
+ * number of VFs we can support.
+ *
+ * The 1/4 limit is arbitrary and can be tweaked.
+ */
+ if (vf_bar_sz > (phb->ioda.m64_segsize >> 2)) {
+ /*
+ * On PHB3, the minimum size alignment of M64 BAR in
+ * single mode is 32MB. If this VF BAR is smaller than
+ * 32MB, but still too large for a segmented window
+ * then we can't map it and need to disable SR-IOV for
+ * this device.
+ */
+ if (vf_bar_sz < SZ_32M) {
+ pci_err(pdev, "VF BAR%d: %pR can't be mapped in single PE mode\n",
+ i, res);
+ goto disable_iov;
+ }
+
+ iov->m64_single_mode[i] = true;
+ continue;
+ }
+
+ /*
+ * This BAR can be mapped with one segmented window, so adjust
+ * te resource size to accommodate.
+ */
+ pci_dbg(pdev, " Fixing VF BAR%d: %pR to\n", i, res);
+ res->end = res->start + vf_bar_sz * mul - 1;
+ pci_dbg(pdev, " %pR\n", res);
+
+ pci_info(pdev, "VF BAR%d: %pR (expanded to %d VFs for PE alignment)",
+ i, res, mul);
+
+ iov->need_shift = true;
+ }
+
+ return;
+
+disable_iov:
+ /* Save ourselves some MMIO space by disabling the unusable BARs */
+ for (i = 0; i < PCI_SRIOV_NUM_BARS; i++) {
+ res = &pdev->resource[i + PCI_IOV_RESOURCES];
+ res->flags = 0;
+ res->end = res->start - 1;
+ }
+
+ pdev->dev.archdata.iov_data = NULL;
+ kfree(iov);
+}
+
+void pnv_pci_ioda_fixup_iov(struct pci_dev *pdev)
+{
+ if (WARN_ON(pci_dev_is_added(pdev)))
+ return;
+
+ if (pdev->is_virtfn) {
+ struct pnv_ioda_pe *pe = pnv_ioda_get_pe(pdev);
+
+ /*
+ * VF PEs are single-device PEs so their pdev pointer needs to
+ * be set. The pdev doesn't exist when the PE is allocated (in
+ * (pcibios_sriov_enable()) so we fix it up here.
+ */
+ pe->pdev = pdev;
+ WARN_ON(!(pe->flags & PNV_IODA_PE_VF));
+ } else if (pdev->is_physfn) {
+ /*
+ * For PFs adjust their allocated IOV resources to match what
+ * the PHB can support using it's M64 BAR table.
+ */
+ pnv_pci_ioda_fixup_iov_resources(pdev);
+ }
+}
+
+resource_size_t pnv_pci_iov_resource_alignment(struct pci_dev *pdev,
+ int resno)
+{
+ resource_size_t align = pci_iov_resource_size(pdev, resno);
+ struct pnv_phb *phb = pci_bus_to_pnvhb(pdev->bus);
+ struct pnv_iov_data *iov = pnv_iov_get(pdev);
+
+ /*
+ * iov can be null if we have an SR-IOV device with IOV BAR that can't
+ * be placed in the m64 space (i.e. The BAR is 32bit or non-prefetch).
+ * In that case we don't allow VFs to be enabled since one of their
+ * BARs would not be placed in the correct PE.
+ */
+ if (!iov)
+ return align;
+
+ /*
+ * If we're using single mode then we can just use the native VF BAR
+ * alignment. We validated that it's possible to use a single PE
+ * window above when we did the fixup.
+ */
+ if (iov->m64_single_mode[resno - PCI_IOV_RESOURCES])
+ return align;
+
+ /*
+ * On PowerNV platform, IOV BAR is mapped by M64 BAR to enable the
+ * SR-IOV. While from hardware perspective, the range mapped by M64
+ * BAR should be size aligned.
+ *
+ * This function returns the total IOV BAR size if M64 BAR is in
+ * Shared PE mode or just VF BAR size if not.
+ * If the M64 BAR is in Single PE mode, return the VF BAR size or
+ * M64 segment size if IOV BAR size is less.
+ */
+ return phb->ioda.total_pe_num * align;
+}
+
+static int pnv_pci_vf_release_m64(struct pci_dev *pdev, u16 num_vfs)
+{
+ struct pnv_iov_data *iov;
+ struct pnv_phb *phb;
+ int window_id;
+
+ phb = pci_bus_to_pnvhb(pdev->bus);
+ iov = pnv_iov_get(pdev);
+
+ for_each_set_bit(window_id, iov->used_m64_bar_mask, MAX_M64_BARS) {
+ opal_pci_phb_mmio_enable(phb->opal_id,
+ OPAL_M64_WINDOW_TYPE,
+ window_id,
+ 0);
+
+ clear_bit(window_id, &phb->ioda.m64_bar_alloc);
+ }
+
+ return 0;
+}
+
+
+/*
+ * PHB3 and beyond support segmented windows. The window's address range
+ * is subdivided into phb->ioda.total_pe_num segments and there's a 1-1
+ * mapping between PEs and segments.
+ */
+static int64_t pnv_ioda_map_m64_segmented(struct pnv_phb *phb,
+ int window_id,
+ resource_size_t start,
+ resource_size_t size)
+{
+ int64_t rc;
+
+ rc = opal_pci_set_phb_mem_window(phb->opal_id,
+ OPAL_M64_WINDOW_TYPE,
+ window_id,
+ start,
+ 0, /* unused */
+ size);
+ if (rc)
+ goto out;
+
+ rc = opal_pci_phb_mmio_enable(phb->opal_id,
+ OPAL_M64_WINDOW_TYPE,
+ window_id,
+ OPAL_ENABLE_M64_SPLIT);
+out:
+ if (rc)
+ pr_err("Failed to map M64 window #%d: %lld\n", window_id, rc);
+
+ return rc;
+}
+
+static int64_t pnv_ioda_map_m64_single(struct pnv_phb *phb,
+ int pe_num,
+ int window_id,
+ resource_size_t start,
+ resource_size_t size)
+{
+ int64_t rc;
+
+ /*
+ * The API for setting up m64 mmio windows seems to have been designed
+ * with P7-IOC in mind. For that chip each M64 BAR (window) had a fixed
+ * split of 8 equally sized segments each of which could individually
+ * assigned to a PE.
+ *
+ * The problem with this is that the API doesn't have any way to
+ * communicate the number of segments we want on a BAR. This wasn't
+ * a problem for p7-ioc since you didn't have a choice, but the
+ * single PE windows added in PHB3 don't map cleanly to this API.
+ *
+ * As a result we've got this slightly awkward process where we
+ * call opal_pci_map_pe_mmio_window() to put the single in single
+ * PE mode, and set the PE for the window before setting the address
+ * bounds. We need to do it this way because the single PE windows
+ * for PHB3 have different alignment requirements on PHB3.
+ */
+ rc = opal_pci_map_pe_mmio_window(phb->opal_id,
+ pe_num,
+ OPAL_M64_WINDOW_TYPE,
+ window_id,
+ 0);
+ if (rc)
+ goto out;
+
+ /*
+ * NB: In single PE mode the window needs to be aligned to 32MB
+ */
+ rc = opal_pci_set_phb_mem_window(phb->opal_id,
+ OPAL_M64_WINDOW_TYPE,
+ window_id,
+ start,
+ 0, /* ignored by FW, m64 is 1-1 */
+ size);
+ if (rc)
+ goto out;
+
+ /*
+ * Now actually enable it. We specified the BAR should be in "non-split"
+ * mode so FW will validate that the BAR is in single PE mode.
+ */
+ rc = opal_pci_phb_mmio_enable(phb->opal_id,
+ OPAL_M64_WINDOW_TYPE,
+ window_id,
+ OPAL_ENABLE_M64_NON_SPLIT);
+out:
+ if (rc)
+ pr_err("Error mapping single PE BAR\n");
+
+ return rc;
+}
+
+static int pnv_pci_alloc_m64_bar(struct pnv_phb *phb, struct pnv_iov_data *iov)
+{
+ int win;
+
+ do {
+ win = find_next_zero_bit(&phb->ioda.m64_bar_alloc,
+ phb->ioda.m64_bar_idx + 1, 0);
+
+ if (win >= phb->ioda.m64_bar_idx + 1)
+ return -1;
+ } while (test_and_set_bit(win, &phb->ioda.m64_bar_alloc));
+
+ set_bit(win, iov->used_m64_bar_mask);
+
+ return win;
+}
+
+static int pnv_pci_vf_assign_m64(struct pci_dev *pdev, u16 num_vfs)
+{
+ struct pnv_iov_data *iov;
+ struct pnv_phb *phb;
+ unsigned int win;
+ struct resource *res;
+ int i, j;
+ int64_t rc;
+ resource_size_t size, start;
+ int base_pe_num;
+
+ phb = pci_bus_to_pnvhb(pdev->bus);
+ iov = pnv_iov_get(pdev);
+
+ for (i = 0; i < PCI_SRIOV_NUM_BARS; i++) {
+ res = &pdev->resource[i + PCI_IOV_RESOURCES];
+ if (!res->flags || !res->parent)
+ continue;
+
+ /* don't need single mode? map everything in one go! */
+ if (!iov->m64_single_mode[i]) {
+ win = pnv_pci_alloc_m64_bar(phb, iov);
+ if (win < 0)
+ goto m64_failed;
+
+ size = resource_size(res);
+ start = res->start;
+
+ rc = pnv_ioda_map_m64_segmented(phb, win, start, size);
+ if (rc)
+ goto m64_failed;
+
+ continue;
+ }
+
+ /* otherwise map each VF with single PE BARs */
+ size = pci_iov_resource_size(pdev, PCI_IOV_RESOURCES + i);
+ base_pe_num = iov->vf_pe_arr[0].pe_number;
+
+ for (j = 0; j < num_vfs; j++) {
+ win = pnv_pci_alloc_m64_bar(phb, iov);
+ if (win < 0)
+ goto m64_failed;
+
+ start = res->start + size * j;
+ rc = pnv_ioda_map_m64_single(phb, win,
+ base_pe_num + j,
+ start,
+ size);
+ if (rc)
+ goto m64_failed;
+ }
+ }
+ return 0;
+
+m64_failed:
+ pnv_pci_vf_release_m64(pdev, num_vfs);
+ return -EBUSY;
+}
+
+static void pnv_ioda_release_vf_PE(struct pci_dev *pdev)
+{
+ struct pnv_phb *phb;
+ struct pnv_ioda_pe *pe, *pe_n;
+
+ phb = pci_bus_to_pnvhb(pdev->bus);
+
+ if (!pdev->is_physfn)
+ return;
+
+ /* FIXME: Use pnv_ioda_release_pe()? */
+ list_for_each_entry_safe(pe, pe_n, &phb->ioda.pe_list, list) {
+ if (pe->parent_dev != pdev)
+ continue;
+
+ pnv_pci_ioda2_release_pe_dma(pe);
+
+ /* Remove from list */
+ mutex_lock(&phb->ioda.pe_list_mutex);
+ list_del(&pe->list);
+ mutex_unlock(&phb->ioda.pe_list_mutex);
+
+ pnv_ioda_deconfigure_pe(phb, pe);
+
+ pnv_ioda_free_pe(pe);
+ }
+}
+
+static int pnv_pci_vf_resource_shift(struct pci_dev *dev, int offset)
+{
+ struct resource *res, res2;
+ struct pnv_iov_data *iov;
+ resource_size_t size;
+ u16 num_vfs;
+ int i;
+
+ if (!dev->is_physfn)
+ return -EINVAL;
+ iov = pnv_iov_get(dev);
+
+ /*
+ * "offset" is in VFs. The M64 windows are sized so that when they
+ * are segmented, each segment is the same size as the IOV BAR.
+ * Each segment is in a separate PE, and the high order bits of the
+ * address are the PE number. Therefore, each VF's BAR is in a
+ * separate PE, and changing the IOV BAR start address changes the
+ * range of PEs the VFs are in.
+ */
+ num_vfs = iov->num_vfs;
+ for (i = 0; i < PCI_SRIOV_NUM_BARS; i++) {
+ res = &dev->resource[i + PCI_IOV_RESOURCES];
+ if (!res->flags || !res->parent)
+ continue;
+ if (iov->m64_single_mode[i])
+ continue;
+
+ /*
+ * The actual IOV BAR range is determined by the start address
+ * and the actual size for num_vfs VFs BAR. This check is to
+ * make sure that after shifting, the range will not overlap
+ * with another device.
+ */
+ size = pci_iov_resource_size(dev, i + PCI_IOV_RESOURCES);
+ res2.flags = res->flags;
+ res2.start = res->start + (size * offset);
+ res2.end = res2.start + (size * num_vfs) - 1;
+
+ if (res2.end > res->end) {
+ dev_err(&dev->dev, "VF BAR%d: %pR would extend past %pR (trying to enable %d VFs shifted by %d)\n",
+ i, &res2, res, num_vfs, offset);
+ return -EBUSY;
+ }
+ }
+
+ /*
+ * Since M64 BAR shares segments among all possible 256 PEs,
+ * we have to shift the beginning of PF IOV BAR to make it start from
+ * the segment which belongs to the PE number assigned to the first VF.
+ * This creates a "hole" in the /proc/iomem which could be used for
+ * allocating other resources so we reserve this area below and
+ * release when IOV is released.
+ */
+ for (i = 0; i < PCI_SRIOV_NUM_BARS; i++) {
+ res = &dev->resource[i + PCI_IOV_RESOURCES];
+ if (!res->flags || !res->parent)
+ continue;
+ if (iov->m64_single_mode[i])
+ continue;
+
+ size = pci_iov_resource_size(dev, i + PCI_IOV_RESOURCES);
+ res2 = *res;
+ res->start += size * offset;
+
+ dev_info(&dev->dev, "VF BAR%d: %pR shifted to %pR (%sabling %d VFs shifted by %d)\n",
+ i, &res2, res, (offset > 0) ? "En" : "Dis",
+ num_vfs, offset);
+
+ if (offset < 0) {
+ devm_release_resource(&dev->dev, &iov->holes[i]);
+ memset(&iov->holes[i], 0, sizeof(iov->holes[i]));
+ }
+
+ pci_update_resource(dev, i + PCI_IOV_RESOURCES);
+
+ if (offset > 0) {
+ iov->holes[i].start = res2.start;
+ iov->holes[i].end = res2.start + size * offset - 1;
+ iov->holes[i].flags = IORESOURCE_BUS;
+ iov->holes[i].name = "pnv_iov_reserved";
+ devm_request_resource(&dev->dev, res->parent,
+ &iov->holes[i]);
+ }
+ }
+ return 0;
+}
+
+static void pnv_pci_sriov_disable(struct pci_dev *pdev)
+{
+ u16 num_vfs, base_pe;
+ struct pnv_iov_data *iov;
+
+ iov = pnv_iov_get(pdev);
+ num_vfs = iov->num_vfs;
+ base_pe = iov->vf_pe_arr[0].pe_number;
+
+ if (WARN_ON(!iov))
+ return;
+
+ /* Release VF PEs */
+ pnv_ioda_release_vf_PE(pdev);
+
+ /* Un-shift the IOV BARs if we need to */
+ if (iov->need_shift)
+ pnv_pci_vf_resource_shift(pdev, -base_pe);
+
+ /* Release M64 windows */
+ pnv_pci_vf_release_m64(pdev, num_vfs);
+}
+
+static void pnv_ioda_setup_vf_PE(struct pci_dev *pdev, u16 num_vfs)
+{
+ struct pnv_phb *phb;
+ struct pnv_ioda_pe *pe;
+ int pe_num;
+ u16 vf_index;
+ struct pnv_iov_data *iov;
+ struct pci_dn *pdn;
+
+ if (!pdev->is_physfn)
+ return;
+
+ phb = pci_bus_to_pnvhb(pdev->bus);
+ pdn = pci_get_pdn(pdev);
+ iov = pnv_iov_get(pdev);
+
+ /* Reserve PE for each VF */
+ for (vf_index = 0; vf_index < num_vfs; vf_index++) {
+ int vf_devfn = pci_iov_virtfn_devfn(pdev, vf_index);
+ int vf_bus = pci_iov_virtfn_bus(pdev, vf_index);
+ struct pci_dn *vf_pdn;
+
+ pe = &iov->vf_pe_arr[vf_index];
+ pe->phb = phb;
+ pe->flags = PNV_IODA_PE_VF;
+ pe->pbus = NULL;
+ pe->parent_dev = pdev;
+ pe->mve_number = -1;
+ pe->rid = (vf_bus << 8) | vf_devfn;
+
+ pe_num = pe->pe_number;
+ pe_info(pe, "VF %04d:%02d:%02d.%d associated with PE#%x\n",
+ pci_domain_nr(pdev->bus), pdev->bus->number,
+ PCI_SLOT(vf_devfn), PCI_FUNC(vf_devfn), pe_num);
+
+ if (pnv_ioda_configure_pe(phb, pe)) {
+ /* XXX What do we do here ? */
+ pnv_ioda_free_pe(pe);
+ pe->pdev = NULL;
+ continue;
+ }
+
+ /* Put PE to the list */
+ mutex_lock(&phb->ioda.pe_list_mutex);
+ list_add_tail(&pe->list, &phb->ioda.pe_list);
+ mutex_unlock(&phb->ioda.pe_list_mutex);
+
+ /* associate this pe to it's pdn */
+ list_for_each_entry(vf_pdn, &pdn->parent->child_list, list) {
+ if (vf_pdn->busno == vf_bus &&
+ vf_pdn->devfn == vf_devfn) {
+ vf_pdn->pe_number = pe_num;
+ break;
+ }
+ }
+
+ pnv_pci_ioda2_setup_dma_pe(phb, pe);
+ }
+}
+
+static int pnv_pci_sriov_enable(struct pci_dev *pdev, u16 num_vfs)
+{
+ struct pnv_ioda_pe *base_pe;
+ struct pnv_iov_data *iov;
+ struct pnv_phb *phb;
+ int ret;
+ u16 i;
+
+ phb = pci_bus_to_pnvhb(pdev->bus);
+ iov = pnv_iov_get(pdev);
+
+ /*
+ * There's a calls to IODA2 PE setup code littered throughout. We could
+ * probably fix that, but we'd still have problems due to the
+ * restriction inherent on IODA1 PHBs.
+ *
+ * NB: We class IODA3 as IODA2 since they're very similar.
+ */
+ if (phb->type != PNV_PHB_IODA2) {
+ pci_err(pdev, "SR-IOV is not supported on this PHB\n");
+ return -ENXIO;
+ }
+
+ if (!iov) {
+ dev_info(&pdev->dev, "don't support this SRIOV device with non 64bit-prefetchable IOV BAR\n");
+ return -ENOSPC;
+ }
+
+ /* allocate a contigious block of PEs for our VFs */
+ base_pe = pnv_ioda_alloc_pe(phb, num_vfs);
+ if (!base_pe) {
+ pci_err(pdev, "Unable to allocate PEs for %d VFs\n", num_vfs);
+ return -EBUSY;
+ }
+
+ iov->vf_pe_arr = base_pe;
+ iov->num_vfs = num_vfs;
+
+ /* Assign M64 window accordingly */
+ ret = pnv_pci_vf_assign_m64(pdev, num_vfs);
+ if (ret) {
+ dev_info(&pdev->dev, "Not enough M64 window resources\n");
+ goto m64_failed;
+ }
+
+ /*
+ * When using one M64 BAR to map one IOV BAR, we need to shift
+ * the IOV BAR according to the PE# allocated to the VFs.
+ * Otherwise, the PE# for the VF will conflict with others.
+ */
+ if (iov->need_shift) {
+ ret = pnv_pci_vf_resource_shift(pdev, base_pe->pe_number);
+ if (ret)
+ goto shift_failed;
+ }
+
+ /* Setup VF PEs */
+ pnv_ioda_setup_vf_PE(pdev, num_vfs);
+
+ return 0;
+
+shift_failed:
+ pnv_pci_vf_release_m64(pdev, num_vfs);
+
+m64_failed:
+ for (i = 0; i < num_vfs; i++)
+ pnv_ioda_free_pe(&iov->vf_pe_arr[i]);
+
+ return ret;
+}
+
+int pnv_pcibios_sriov_disable(struct pci_dev *pdev)
+{
+ pnv_pci_sriov_disable(pdev);
+
+ /* Release PCI data */
+ remove_sriov_vf_pdns(pdev);
+ return 0;
+}
+
+int pnv_pcibios_sriov_enable(struct pci_dev *pdev, u16 num_vfs)
+{
+ /* Allocate PCI data */
+ add_sriov_vf_pdns(pdev);
+
+ return pnv_pci_sriov_enable(pdev, num_vfs);
+}
diff --git a/arch/powerpc/platforms/powernv/pci.c b/arch/powerpc/platforms/powernv/pci.c
index 091fe1c..9b9bca1 100644
--- a/arch/powerpc/platforms/powernv/pci.c
+++ b/arch/powerpc/platforms/powernv/pci.c
@@ -162,8 +162,7 @@
int pnv_setup_msi_irqs(struct pci_dev *pdev, int nvec, int type)
{
- struct pci_controller *hose = pci_bus_to_host(pdev->bus);
- struct pnv_phb *phb = hose->private_data;
+ struct pnv_phb *phb = pci_bus_to_pnvhb(pdev->bus);
struct msi_desc *entry;
struct msi_msg msg;
int hwirq;
@@ -211,8 +210,7 @@
void pnv_teardown_msi_irqs(struct pci_dev *pdev)
{
- struct pci_controller *hose = pci_bus_to_host(pdev->bus);
- struct pnv_phb *phb = hose->private_data;
+ struct pnv_phb *phb = pci_bus_to_pnvhb(pdev->bus);
struct msi_desc *entry;
irq_hw_number_t hwirq;
@@ -824,10 +822,9 @@
int pnv_pci_set_tunnel_bar(struct pci_dev *dev, u64 addr, int enable)
{
- __be64 val;
- struct pci_controller *hose;
- struct pnv_phb *phb;
+ struct pnv_phb *phb = pci_bus_to_pnvhb(dev->bus);
u64 tunnel_bar;
+ __be64 val;
int rc;
if (!opal_check_token(OPAL_PCI_GET_PBCQ_TUNNEL_BAR))
@@ -835,9 +832,6 @@
if (!opal_check_token(OPAL_PCI_SET_PBCQ_TUNNEL_BAR))
return -ENXIO;
- hose = pci_bus_to_host(dev->bus);
- phb = hose->private_data;
-
mutex_lock(&tunnel_mutex);
rc = opal_pci_get_pbcq_tunnel_bar(phb->opal_id, &val);
if (rc != OPAL_SUCCESS) {
diff --git a/arch/powerpc/platforms/powernv/pci.h b/arch/powerpc/platforms/powernv/pci.h
index 51c254f2..739a0b3 100644
--- a/arch/powerpc/platforms/powernv/pci.h
+++ b/arch/powerpc/platforms/powernv/pci.h
@@ -87,7 +87,14 @@
bool tce_bypass_enabled;
uint64_t tce_bypass_base;
- /* MSIs. MVE index is identical for for 32 and 64 bit MSI
+ /*
+ * Used to track whether we've done DMA setup for this PE or not. We
+ * want to defer allocating TCE tables, etc until we've added a
+ * non-bridge device to the PE.
+ */
+ bool dma_setup_done;
+
+ /* MSIs. MVE index is identical for 32 and 64 bit MSI
* and -1 if not supported. (It's actually identical to the
* PE number)
*/
@@ -147,6 +154,7 @@
unsigned long m64_size;
unsigned long m64_segsize;
unsigned long m64_base;
+#define MAX_M64_BARS 64
unsigned long m64_bar_alloc;
/* IO ports */
@@ -187,6 +195,89 @@
u8 *diag_data;
};
+
+/* IODA PE management */
+
+static inline bool pnv_pci_is_m64(struct pnv_phb *phb, struct resource *r)
+{
+ /*
+ * WARNING: We cannot rely on the resource flags. The Linux PCI
+ * allocation code sometimes decides to put a 64-bit prefetchable
+ * BAR in the 32-bit window, so we have to compare the addresses.
+ *
+ * For simplicity we only test resource start.
+ */
+ return (r->start >= phb->ioda.m64_base &&
+ r->start < (phb->ioda.m64_base + phb->ioda.m64_size));
+}
+
+static inline bool pnv_pci_is_m64_flags(unsigned long resource_flags)
+{
+ unsigned long flags = (IORESOURCE_MEM_64 | IORESOURCE_PREFETCH);
+
+ return (resource_flags & flags) == flags;
+}
+
+int pnv_ioda_configure_pe(struct pnv_phb *phb, struct pnv_ioda_pe *pe);
+int pnv_ioda_deconfigure_pe(struct pnv_phb *phb, struct pnv_ioda_pe *pe);
+
+void pnv_pci_ioda2_setup_dma_pe(struct pnv_phb *phb, struct pnv_ioda_pe *pe);
+void pnv_pci_ioda2_release_pe_dma(struct pnv_ioda_pe *pe);
+
+struct pnv_ioda_pe *pnv_ioda_alloc_pe(struct pnv_phb *phb, int count);
+void pnv_ioda_free_pe(struct pnv_ioda_pe *pe);
+
+#ifdef CONFIG_PCI_IOV
+/*
+ * For SR-IOV we want to put each VF's MMIO resource in to a separate PE.
+ * This requires a bit of acrobatics with the MMIO -> PE configuration
+ * and this structure is used to keep track of it all.
+ */
+struct pnv_iov_data {
+ /* number of VFs enabled */
+ u16 num_vfs;
+
+ /* pointer to the array of VF PEs. num_vfs long*/
+ struct pnv_ioda_pe *vf_pe_arr;
+
+ /* Did we map the VF BAR with single-PE IODA BARs? */
+ bool m64_single_mode[PCI_SRIOV_NUM_BARS];
+
+ /*
+ * True if we're using any segmented windows. In that case we need
+ * shift the start of the IOV resource the segment corresponding to
+ * the allocated PE.
+ */
+ bool need_shift;
+
+ /*
+ * Bit mask used to track which m64 windows are used to map the
+ * SR-IOV BARs for this device.
+ */
+ DECLARE_BITMAP(used_m64_bar_mask, MAX_M64_BARS);
+
+ /*
+ * If we map the SR-IOV BARs with a segmented window then
+ * parts of that window will be "claimed" by other PEs.
+ *
+ * "holes" here is used to reserve the leading portion
+ * of the window that is used by other (non VF) PEs.
+ */
+ struct resource holes[PCI_SRIOV_NUM_BARS];
+};
+
+static inline struct pnv_iov_data *pnv_iov_get(struct pci_dev *pdev)
+{
+ return pdev->dev.archdata.iov_data;
+}
+
+void pnv_pci_ioda_fixup_iov(struct pci_dev *pdev);
+resource_size_t pnv_pci_iov_resource_alignment(struct pci_dev *pdev, int resno);
+
+int pnv_pcibios_sriov_enable(struct pci_dev *pdev, u16 num_vfs);
+int pnv_pcibios_sriov_disable(struct pci_dev *pdev);
+#endif /* CONFIG_PCI_IOV */
+
extern struct pci_ops pnv_pci_ops;
void pnv_pci_dump_phb_diag_data(struct pci_controller *hose,
@@ -260,4 +351,14 @@
extern unsigned long pnv_ioda_parse_tce_sizes(struct pnv_phb *phb);
+static inline struct pnv_phb *pci_bus_to_pnvhb(struct pci_bus *bus)
+{
+ struct pci_controller *hose = bus->sysdata;
+
+ if (hose)
+ return hose->private_data;
+
+ return NULL;
+}
+
#endif /* __POWERNV_PCI_H */
diff --git a/arch/powerpc/platforms/powernv/setup.c b/arch/powerpc/platforms/powernv/setup.c
index 3bc188d..7fcb886 100644
--- a/arch/powerpc/platforms/powernv/setup.c
+++ b/arch/powerpc/platforms/powernv/setup.c
@@ -399,7 +399,15 @@
#ifdef CONFIG_MEMORY_HOTPLUG_SPARSE
static unsigned long pnv_memory_block_size(void)
{
- return 256UL * 1024 * 1024;
+ /*
+ * We map the kernel linear region with 1GB large pages on radix. For
+ * memory hot unplug to work our memory block size must be at least
+ * this size.
+ */
+ if (radix_enabled())
+ return radix_mem_block_size;
+ else
+ return 256UL * 1024 * 1024;
}
#endif
diff --git a/arch/powerpc/platforms/pseries/Kconfig b/arch/powerpc/platforms/pseries/Kconfig
index 24c1836..5e037df 100644
--- a/arch/powerpc/platforms/pseries/Kconfig
+++ b/arch/powerpc/platforms/pseries/Kconfig
@@ -25,15 +25,22 @@
select SWIOTLB
default y
+config PARAVIRT_SPINLOCKS
+ bool
+
config PPC_SPLPAR
- depends on PPC_PSERIES
bool "Support for shared-processor logical partitions"
+ depends on PPC_PSERIES
+ select PARAVIRT_SPINLOCKS if PPC_QUEUED_SPINLOCKS
+ default y
help
Enabling this option will make the kernel run more efficiently
on logically-partitioned pSeries systems which use shared
processors, that is, which share physical processors between
two or more partitions.
+ Say Y if you are unsure.
+
config DTL
bool "Dispatch Trace Log"
depends on PPC_SPLPAR && DEBUG_FS
diff --git a/arch/powerpc/platforms/pseries/eeh_pseries.c b/arch/powerpc/platforms/pseries/eeh_pseries.c
index ace117f..cb2d9a9 100644
--- a/arch/powerpc/platforms/pseries/eeh_pseries.c
+++ b/arch/powerpc/platforms/pseries/eeh_pseries.c
@@ -24,6 +24,7 @@
#include <linux/sched.h>
#include <linux/seq_file.h>
#include <linux/spinlock.h>
+#include <linux/crash_dump.h>
#include <asm/eeh.h>
#include <asm/eeh_event.h>
@@ -32,6 +33,8 @@
#include <asm/ppc-pci.h>
#include <asm/rtas.h>
+static int pseries_eeh_get_pe_addr(struct pci_dn *pdn);
+
/* RTAS tokens */
static int ibm_set_eeh_option;
static int ibm_set_slot_reset;
@@ -52,8 +55,6 @@
dev_dbg(&pdev->dev, "EEH: Setting up device\n");
#ifdef CONFIG_PCI_IOV
if (pdev->is_virtfn) {
- struct pci_dn *physfn_pdn;
-
pdn->device_id = pdev->device;
pdn->vendor_id = pdev->vendor;
pdn->class_code = pdev->class;
@@ -63,23 +64,172 @@
* completion from platform.
*/
pdn->last_allow_rc = 0;
- physfn_pdn = pci_get_pdn(pdev->physfn);
- pdn->pe_number = physfn_pdn->pe_num_map[pdn->vf_index];
}
#endif
pseries_eeh_init_edev(pdn);
#ifdef CONFIG_PCI_IOV
if (pdev->is_virtfn) {
+ /*
+ * FIXME: This really should be handled by choosing the right
+ * parent PE in in pseries_eeh_init_edev().
+ */
+ struct eeh_pe *physfn_pe = pci_dev_to_eeh_dev(pdev->physfn)->pe;
struct eeh_dev *edev = pdn_to_eeh_dev(pdn);
edev->pe_config_addr = (pdn->busno << 16) | (pdn->devfn << 8);
- eeh_rmv_from_parent_pe(edev); /* Remove as it is adding to bus pe */
- eeh_add_to_parent_pe(edev); /* Add as VF PE type */
+ eeh_pe_tree_remove(edev); /* Remove as it is adding to bus pe */
+ eeh_pe_tree_insert(edev, physfn_pe); /* Add as VF PE type */
}
#endif
eeh_probe_device(pdev);
}
+
+/**
+ * pseries_eeh_get_config_addr - Retrieve config address
+ *
+ * Retrieve the assocated config address. Actually, there're 2 RTAS
+ * function calls dedicated for the purpose. We need implement
+ * it through the new function and then the old one. Besides,
+ * you should make sure the config address is figured out from
+ * FDT node before calling the function.
+ *
+ * It's notable that zero'ed return value means invalid PE config
+ * address.
+ */
+static int pseries_eeh_get_config_addr(struct pci_controller *phb, int config_addr)
+{
+ int ret = 0;
+ int rets[3];
+
+ if (ibm_get_config_addr_info2 != RTAS_UNKNOWN_SERVICE) {
+ /*
+ * First of all, we need to make sure there has one PE
+ * associated with the device. Otherwise, PE address is
+ * meaningless.
+ */
+ ret = rtas_call(ibm_get_config_addr_info2, 4, 2, rets,
+ config_addr, BUID_HI(phb->buid),
+ BUID_LO(phb->buid), 1);
+ if (ret || (rets[0] == 0))
+ return 0;
+
+ /* Retrieve the associated PE config address */
+ ret = rtas_call(ibm_get_config_addr_info2, 4, 2, rets,
+ config_addr, BUID_HI(phb->buid),
+ BUID_LO(phb->buid), 0);
+ if (ret) {
+ pr_warn("%s: Failed to get address for PHB#%x-PE#%x\n",
+ __func__, phb->global_number, config_addr);
+ return 0;
+ }
+
+ return rets[0];
+ }
+
+ if (ibm_get_config_addr_info != RTAS_UNKNOWN_SERVICE) {
+ ret = rtas_call(ibm_get_config_addr_info, 4, 2, rets,
+ config_addr, BUID_HI(phb->buid),
+ BUID_LO(phb->buid), 0);
+ if (ret) {
+ pr_warn("%s: Failed to get address for PHB#%x-PE#%x\n",
+ __func__, phb->global_number, config_addr);
+ return 0;
+ }
+
+ return rets[0];
+ }
+
+ return ret;
+}
+
+/**
+ * pseries_eeh_phb_reset - Reset the specified PHB
+ * @phb: PCI controller
+ * @config_adddr: the associated config address
+ * @option: reset option
+ *
+ * Reset the specified PHB/PE
+ */
+static int pseries_eeh_phb_reset(struct pci_controller *phb, int config_addr, int option)
+{
+ int ret;
+
+ /* Reset PE through RTAS call */
+ ret = rtas_call(ibm_set_slot_reset, 4, 1, NULL,
+ config_addr, BUID_HI(phb->buid),
+ BUID_LO(phb->buid), option);
+
+ /* If fundamental-reset not supported, try hot-reset */
+ if (option == EEH_RESET_FUNDAMENTAL &&
+ ret == -8) {
+ option = EEH_RESET_HOT;
+ ret = rtas_call(ibm_set_slot_reset, 4, 1, NULL,
+ config_addr, BUID_HI(phb->buid),
+ BUID_LO(phb->buid), option);
+ }
+
+ /* We need reset hold or settlement delay */
+ if (option == EEH_RESET_FUNDAMENTAL ||
+ option == EEH_RESET_HOT)
+ msleep(EEH_PE_RST_HOLD_TIME);
+ else
+ msleep(EEH_PE_RST_SETTLE_TIME);
+
+ return ret;
+}
+
+/**
+ * pseries_eeh_phb_configure_bridge - Configure PCI bridges in the indicated PE
+ * @phb: PCI controller
+ * @config_adddr: the associated config address
+ *
+ * The function will be called to reconfigure the bridges included
+ * in the specified PE so that the mulfunctional PE would be recovered
+ * again.
+ */
+static int pseries_eeh_phb_configure_bridge(struct pci_controller *phb, int config_addr)
+{
+ int ret;
+ /* Waiting 0.2s maximum before skipping configuration */
+ int max_wait = 200;
+
+ while (max_wait > 0) {
+ ret = rtas_call(ibm_configure_pe, 3, 1, NULL,
+ config_addr, BUID_HI(phb->buid),
+ BUID_LO(phb->buid));
+
+ if (!ret)
+ return ret;
+ if (ret < 0)
+ break;
+
+ /*
+ * If RTAS returns a delay value that's above 100ms, cut it
+ * down to 100ms in case firmware made a mistake. For more
+ * on how these delay values work see rtas_busy_delay_time
+ */
+ if (ret > RTAS_EXTENDED_DELAY_MIN+2 &&
+ ret <= RTAS_EXTENDED_DELAY_MAX)
+ ret = RTAS_EXTENDED_DELAY_MIN+2;
+
+ max_wait -= rtas_busy_delay_time(ret);
+
+ if (max_wait < 0)
+ break;
+
+ rtas_busy_delay(ret);
+ }
+
+ pr_warn("%s: Unable to configure bridge PHB#%x-PE#%x (%d)\n",
+ __func__, phb->global_number, config_addr, ret);
+ /* PAPR defines -3 as "Parameter Error" for this function: */
+ if (ret == -3)
+ return -EINVAL;
+ else
+ return -EIO;
+}
+
/*
* Buffer for reporting slot-error-detail rtas calls. Its here
* in BSS, and not dynamically alloced, so that it ends up in
@@ -96,6 +246,10 @@
*/
static int pseries_eeh_init(void)
{
+ struct pci_controller *phb;
+ struct pci_dn *pdn;
+ int addr, config_addr;
+
/* figure out EEH RTAS function call tokens */
ibm_set_eeh_option = rtas_token("ibm,set-eeh-option");
ibm_set_slot_reset = rtas_token("ibm,set-slot-reset");
@@ -148,6 +302,22 @@
/* Set EEH machine dependent code */
ppc_md.pcibios_bus_add_device = pseries_pcibios_bus_add_device;
+ if (is_kdump_kernel() || reset_devices) {
+ pr_info("Issue PHB reset ...\n");
+ list_for_each_entry(phb, &hose_list, list_node) {
+ pdn = list_first_entry(&PCI_DN(phb->dn)->child_list, struct pci_dn, list);
+ addr = (pdn->busno << 16) | (pdn->devfn << 8);
+ config_addr = pseries_eeh_get_config_addr(phb, addr);
+ /* invalid PE config addr */
+ if (config_addr == 0)
+ continue;
+
+ pseries_eeh_phb_reset(phb, config_addr, EEH_RESET_FUNDAMENTAL);
+ pseries_eeh_phb_reset(phb, config_addr, EEH_RESET_DEACTIVATE);
+ pseries_eeh_phb_configure_bridge(phb, config_addr);
+ }
+ }
+
return 0;
}
@@ -221,6 +391,43 @@
}
/**
+ * pseries_eeh_pe_get_parent - Retrieve the parent PE
+ * @edev: EEH device
+ *
+ * The whole PEs existing in the system are organized as hierarchy
+ * tree. The function is used to retrieve the parent PE according
+ * to the parent EEH device.
+ */
+static struct eeh_pe *pseries_eeh_pe_get_parent(struct eeh_dev *edev)
+{
+ struct eeh_dev *parent;
+ struct pci_dn *pdn = eeh_dev_to_pdn(edev);
+
+ /*
+ * It might have the case for the indirect parent
+ * EEH device already having associated PE, but
+ * the direct parent EEH device doesn't have yet.
+ */
+ if (edev->physfn)
+ pdn = pci_get_pdn(edev->physfn);
+ else
+ pdn = pdn ? pdn->parent : NULL;
+ while (pdn) {
+ /* We're poking out of PCI territory */
+ parent = pdn_to_eeh_dev(pdn);
+ if (!parent)
+ return NULL;
+
+ if (parent->pe)
+ return parent->pe;
+
+ pdn = pdn->parent;
+ }
+
+ return NULL;
+}
+
+/**
* pseries_eeh_init_edev - initialise the eeh_dev and eeh_pe for a pci_dn
*
* @pdn: PCI device node
@@ -275,12 +482,11 @@
* correctly reflects that current device is root port
* or PCIe switch downstream port.
*/
- edev->class_code = pdn->class_code;
edev->pcix_cap = pseries_eeh_find_cap(pdn, PCI_CAP_ID_PCIX);
edev->pcie_cap = pseries_eeh_find_cap(pdn, PCI_CAP_ID_EXP);
edev->aer_cap = pseries_eeh_find_ecap(pdn, PCI_EXT_CAP_ID_ERR);
edev->mode &= 0xFFFFFF00;
- if ((edev->class_code >> 8) == PCI_CLASS_BRIDGE_PCI) {
+ if ((pdn->class_code >> 8) == PCI_CLASS_BRIDGE_PCI) {
edev->mode |= EEH_DEV_BRIDGE;
if (edev->pcie_cap) {
rtas_read_config(pdn, edev->pcie_cap + PCI_EXP_FLAGS,
@@ -304,8 +510,10 @@
if (ret) {
eeh_edev_dbg(edev, "EEH failed to enable on device (code %d)\n", ret);
} else {
+ struct eeh_pe *parent;
+
/* Retrieve PE address */
- edev->pe_config_addr = eeh_ops->get_pe_addr(&pe);
+ edev->pe_config_addr = pseries_eeh_get_pe_addr(pdn);
pe.addr = edev->pe_config_addr;
/* Some older systems (Power4) allow the ibm,set-eeh-option
@@ -316,16 +524,19 @@
if (ret > 0 && ret != EEH_STATE_NOT_SUPPORT)
enable = 1;
- if (enable) {
+ /*
+ * This device doesn't support EEH, but it may have an
+ * EEH parent. In this case any error on the device will
+ * freeze the PE of it's upstream bridge, so added it to
+ * the upstream PE.
+ */
+ parent = pseries_eeh_pe_get_parent(edev);
+ if (parent && !enable)
+ edev->pe_config_addr = parent->addr;
+
+ if (enable || parent) {
eeh_add_flag(EEH_ENABLED);
- eeh_add_to_parent_pe(edev);
- } else if (pdn->parent && pdn_to_eeh_dev(pdn->parent) &&
- (pdn_to_eeh_dev(pdn->parent))->pe) {
- /* This device doesn't support EEH, but it may have an
- * EEH parent, in which case we mark it as supported.
- */
- edev->pe_config_addr = pdn_to_eeh_dev(pdn->parent)->pe_config_addr;
- eeh_add_to_parent_pe(edev);
+ eeh_pe_tree_insert(edev, parent);
}
eeh_edev_dbg(edev, "EEH is %s on device (code %d)\n",
(enable ? "enabled" : "unsupported"), ret);
@@ -435,8 +646,10 @@
* It's notable that zero'ed return value means invalid PE config
* address.
*/
-static int pseries_eeh_get_pe_addr(struct eeh_pe *pe)
+static int pseries_eeh_get_pe_addr(struct pci_dn *pdn)
{
+ int config_addr = rtas_config_addr(pdn->busno, pdn->devfn, 0);
+ unsigned long buid = pdn->phb->buid;
int ret = 0;
int rets[3];
@@ -447,18 +660,16 @@
* meaningless.
*/
ret = rtas_call(ibm_get_config_addr_info2, 4, 2, rets,
- pe->config_addr, BUID_HI(pe->phb->buid),
- BUID_LO(pe->phb->buid), 1);
+ config_addr, BUID_HI(buid), BUID_LO(buid), 1);
if (ret || (rets[0] == 0))
return 0;
/* Retrieve the associated PE config address */
ret = rtas_call(ibm_get_config_addr_info2, 4, 2, rets,
- pe->config_addr, BUID_HI(pe->phb->buid),
- BUID_LO(pe->phb->buid), 0);
+ config_addr, BUID_HI(buid), BUID_LO(buid), 0);
if (ret) {
pr_warn("%s: Failed to get address for PHB#%x-PE#%x\n",
- __func__, pe->phb->global_number, pe->config_addr);
+ __func__, pdn->phb->global_number, config_addr);
return 0;
}
@@ -467,11 +678,10 @@
if (ibm_get_config_addr_info != RTAS_UNKNOWN_SERVICE) {
ret = rtas_call(ibm_get_config_addr_info, 4, 2, rets,
- pe->config_addr, BUID_HI(pe->phb->buid),
- BUID_LO(pe->phb->buid), 0);
+ config_addr, BUID_HI(buid), BUID_LO(buid), 0);
if (ret) {
pr_warn("%s: Failed to get address for PHB#%x-PE#%x\n",
- __func__, pe->phb->global_number, pe->config_addr);
+ __func__, pdn->phb->global_number, config_addr);
return 0;
}
@@ -569,35 +779,13 @@
static int pseries_eeh_reset(struct eeh_pe *pe, int option)
{
int config_addr;
- int ret;
/* Figure out PE address */
config_addr = pe->config_addr;
if (pe->addr)
config_addr = pe->addr;
- /* Reset PE through RTAS call */
- ret = rtas_call(ibm_set_slot_reset, 4, 1, NULL,
- config_addr, BUID_HI(pe->phb->buid),
- BUID_LO(pe->phb->buid), option);
-
- /* If fundamental-reset not supported, try hot-reset */
- if (option == EEH_RESET_FUNDAMENTAL &&
- ret == -8) {
- option = EEH_RESET_HOT;
- ret = rtas_call(ibm_set_slot_reset, 4, 1, NULL,
- config_addr, BUID_HI(pe->phb->buid),
- BUID_LO(pe->phb->buid), option);
- }
-
- /* We need reset hold or settlement delay */
- if (option == EEH_RESET_FUNDAMENTAL ||
- option == EEH_RESET_HOT)
- msleep(EEH_PE_RST_HOLD_TIME);
- else
- msleep(EEH_PE_RST_SETTLE_TIME);
-
- return ret;
+ return pseries_eeh_phb_reset(pe->phb, config_addr, option);
}
/**
@@ -641,110 +829,51 @@
* pseries_eeh_configure_bridge - Configure PCI bridges in the indicated PE
* @pe: EEH PE
*
- * The function will be called to reconfigure the bridges included
- * in the specified PE so that the mulfunctional PE would be recovered
- * again.
*/
static int pseries_eeh_configure_bridge(struct eeh_pe *pe)
{
int config_addr;
- int ret;
- /* Waiting 0.2s maximum before skipping configuration */
- int max_wait = 200;
/* Figure out the PE address */
config_addr = pe->config_addr;
if (pe->addr)
config_addr = pe->addr;
- while (max_wait > 0) {
- ret = rtas_call(ibm_configure_pe, 3, 1, NULL,
- config_addr, BUID_HI(pe->phb->buid),
- BUID_LO(pe->phb->buid));
-
- if (!ret)
- return ret;
- if (ret < 0)
- break;
-
- /*
- * If RTAS returns a delay value that's above 100ms, cut it
- * down to 100ms in case firmware made a mistake. For more
- * on how these delay values work see rtas_busy_delay_time
- */
- if (ret > RTAS_EXTENDED_DELAY_MIN+2 &&
- ret <= RTAS_EXTENDED_DELAY_MAX)
- ret = RTAS_EXTENDED_DELAY_MIN+2;
-
- max_wait -= rtas_busy_delay_time(ret);
-
- if (max_wait < 0)
- break;
-
- rtas_busy_delay(ret);
- }
-
- pr_warn("%s: Unable to configure bridge PHB#%x-PE#%x (%d)\n",
- __func__, pe->phb->global_number, pe->addr, ret);
- /* PAPR defines -3 as "Parameter Error" for this function: */
- if (ret == -3)
- return -EINVAL;
- else
- return -EIO;
+ return pseries_eeh_phb_configure_bridge(pe->phb, config_addr);
}
/**
* pseries_eeh_read_config - Read PCI config space
- * @pdn: PCI device node
- * @where: PCI address
+ * @edev: EEH device handle
+ * @where: PCI config space offset
* @size: size to read
* @val: return value
*
* Read config space from the speicifed device
*/
-static int pseries_eeh_read_config(struct pci_dn *pdn, int where, int size, u32 *val)
+static int pseries_eeh_read_config(struct eeh_dev *edev, int where, int size, u32 *val)
{
+ struct pci_dn *pdn = eeh_dev_to_pdn(edev);
+
return rtas_read_config(pdn, where, size, val);
}
/**
* pseries_eeh_write_config - Write PCI config space
- * @pdn: PCI device node
- * @where: PCI address
+ * @edev: EEH device handle
+ * @where: PCI config space offset
* @size: size to write
* @val: value to be written
*
* Write config space to the specified device
*/
-static int pseries_eeh_write_config(struct pci_dn *pdn, int where, int size, u32 val)
+static int pseries_eeh_write_config(struct eeh_dev *edev, int where, int size, u32 val)
{
+ struct pci_dn *pdn = eeh_dev_to_pdn(edev);
+
return rtas_write_config(pdn, where, size, val);
}
-static int pseries_eeh_restore_config(struct pci_dn *pdn)
-{
- struct eeh_dev *edev = pdn_to_eeh_dev(pdn);
- s64 ret = 0;
-
- if (!edev)
- return -EEXIST;
-
- /*
- * FIXME: The MPS, error routing rules, timeout setting are worthy
- * to be exported by firmware in extendible way.
- */
- if (edev->physfn)
- ret = eeh_restore_vf_config(pdn);
-
- if (ret) {
- pr_warn("%s: Can't reinit PCI dev 0x%x (%lld)\n",
- __func__, edev->pe_config_addr, ret);
- return -EIO;
- }
-
- return ret;
-}
-
#ifdef CONFIG_PCI_IOV
int pseries_send_allow_unfreeze(struct pci_dn *pdn,
u16 *vf_pe_array, int cur_vfs)
@@ -772,8 +901,8 @@
static int pseries_call_allow_unfreeze(struct eeh_dev *edev)
{
+ int cur_vfs = 0, rc = 0, vf_index, bus, devfn, vf_pe_num;
struct pci_dn *pdn, *tmp, *parent, *physfn_pdn;
- int cur_vfs = 0, rc = 0, vf_index, bus, devfn;
u16 *vf_pe_array;
vf_pe_array = kzalloc(RTAS_DATA_BUF_SIZE, GFP_KERNEL);
@@ -806,8 +935,10 @@
}
} else {
pdn = pci_get_pdn(edev->pdev);
- vf_pe_array[0] = cpu_to_be16(pdn->pe_number);
physfn_pdn = pci_get_pdn(edev->physfn);
+
+ vf_pe_num = physfn_pdn->pe_num_map[edev->vf_index];
+ vf_pe_array[0] = cpu_to_be16(vf_pe_num);
rc = pseries_send_allow_unfreeze(physfn_pdn,
vf_pe_array, 1);
pdn->last_allow_rc = rc;
@@ -818,10 +949,8 @@
return rc;
}
-static int pseries_notify_resume(struct pci_dn *pdn)
+static int pseries_notify_resume(struct eeh_dev *edev)
{
- struct eeh_dev *edev = pdn_to_eeh_dev(pdn);
-
if (!edev)
return -EEXIST;
@@ -841,7 +970,6 @@
.init = pseries_eeh_init,
.probe = pseries_eeh_probe,
.set_option = pseries_eeh_set_option,
- .get_pe_addr = pseries_eeh_get_pe_addr,
.get_state = pseries_eeh_get_state,
.reset = pseries_eeh_reset,
.get_log = pseries_eeh_get_log,
@@ -850,7 +978,7 @@
.read_config = pseries_eeh_read_config,
.write_config = pseries_eeh_write_config,
.next_error = NULL,
- .restore_config = pseries_eeh_restore_config,
+ .restore_config = NULL, /* NB: configure_bridge() does this */
#ifdef CONFIG_PCI_IOV
.notify_resume = pseries_notify_resume
#endif
diff --git a/arch/powerpc/platforms/pseries/firmware.c b/arch/powerpc/platforms/pseries/firmware.c
index 3e49cc2..4c7b7f5 100644
--- a/arch/powerpc/platforms/pseries/firmware.c
+++ b/arch/powerpc/platforms/pseries/firmware.c
@@ -65,6 +65,7 @@
{FW_FEATURE_HPT_RESIZE, "hcall-hpt-resize"},
{FW_FEATURE_BLOCK_REMOVE, "hcall-block-remove"},
{FW_FEATURE_PAPR_SCM, "hcall-scm"},
+ {FW_FEATURE_RPT_INVALIDATE, "hcall-rpt-invalidate"},
};
/* Build up the firmware features bitmask using the contents of
diff --git a/arch/powerpc/platforms/pseries/hotplug-cpu.c b/arch/powerpc/platforms/pseries/hotplug-cpu.c
index 3e8cbfe..c6e0d8a 100644
--- a/arch/powerpc/platforms/pseries/hotplug-cpu.c
+++ b/arch/powerpc/platforms/pseries/hotplug-cpu.c
@@ -35,54 +35,10 @@
#include <asm/topology.h>
#include "pseries.h"
-#include "offline_states.h"
/* This version can't take the spinlock, because it never returns */
static int rtas_stop_self_token = RTAS_UNKNOWN_SERVICE;
-static DEFINE_PER_CPU(enum cpu_state_vals, preferred_offline_state) =
- CPU_STATE_OFFLINE;
-static DEFINE_PER_CPU(enum cpu_state_vals, current_state) = CPU_STATE_OFFLINE;
-
-static enum cpu_state_vals default_offline_state = CPU_STATE_OFFLINE;
-
-static bool cede_offline_enabled __read_mostly = true;
-
-/*
- * Enable/disable cede_offline when available.
- */
-static int __init setup_cede_offline(char *str)
-{
- return (kstrtobool(str, &cede_offline_enabled) == 0);
-}
-
-__setup("cede_offline=", setup_cede_offline);
-
-enum cpu_state_vals get_cpu_current_state(int cpu)
-{
- return per_cpu(current_state, cpu);
-}
-
-void set_cpu_current_state(int cpu, enum cpu_state_vals state)
-{
- per_cpu(current_state, cpu) = state;
-}
-
-enum cpu_state_vals get_preferred_offline_state(int cpu)
-{
- return per_cpu(preferred_offline_state, cpu);
-}
-
-void set_preferred_offline_state(int cpu, enum cpu_state_vals state)
-{
- per_cpu(preferred_offline_state, cpu) = state;
-}
-
-void set_default_offline_state(int cpu)
-{
- per_cpu(preferred_offline_state, cpu) = default_offline_state;
-}
-
static void rtas_stop_self(void)
{
static struct rtas_args args;
@@ -101,9 +57,7 @@
static void pseries_mach_cpu_die(void)
{
- unsigned int cpu = smp_processor_id();
unsigned int hwcpu = hard_smp_processor_id();
- u8 cede_latency_hint = 0;
local_irq_disable();
idle_task_exit();
@@ -112,49 +66,6 @@
else
xics_teardown_cpu();
- if (get_preferred_offline_state(cpu) == CPU_STATE_INACTIVE) {
- set_cpu_current_state(cpu, CPU_STATE_INACTIVE);
- if (ppc_md.suspend_disable_cpu)
- ppc_md.suspend_disable_cpu();
-
- cede_latency_hint = 2;
-
- get_lppaca()->idle = 1;
- if (!lppaca_shared_proc(get_lppaca()))
- get_lppaca()->donate_dedicated_cpu = 1;
-
- while (get_preferred_offline_state(cpu) == CPU_STATE_INACTIVE) {
- while (!prep_irq_for_idle()) {
- local_irq_enable();
- local_irq_disable();
- }
-
- extended_cede_processor(cede_latency_hint);
- }
-
- local_irq_disable();
-
- if (!lppaca_shared_proc(get_lppaca()))
- get_lppaca()->donate_dedicated_cpu = 0;
- get_lppaca()->idle = 0;
-
- if (get_preferred_offline_state(cpu) == CPU_STATE_ONLINE) {
- unregister_slb_shadow(hwcpu);
-
- hard_irq_disable();
- /*
- * Call to start_secondary_resume() will not return.
- * Kernel stack will be reset and start_secondary()
- * will be called to continue the online operation.
- */
- start_secondary_resume();
- }
- }
-
- /* Requested state is CPU_STATE_OFFLINE at this point */
- WARN_ON(get_preferred_offline_state(cpu) != CPU_STATE_OFFLINE);
-
- set_cpu_current_state(cpu, CPU_STATE_OFFLINE);
unregister_slb_shadow(hwcpu);
rtas_stop_self();
@@ -200,24 +111,13 @@
int cpu_status = 1;
unsigned int pcpu = get_hard_smp_processor_id(cpu);
- if (get_preferred_offline_state(cpu) == CPU_STATE_INACTIVE) {
- cpu_status = 1;
- for (tries = 0; tries < 5000; tries++) {
- if (get_cpu_current_state(cpu) == CPU_STATE_INACTIVE) {
- cpu_status = 0;
- break;
- }
- msleep(1);
- }
- } else if (get_preferred_offline_state(cpu) == CPU_STATE_OFFLINE) {
+ for (tries = 0; tries < 25; tries++) {
+ cpu_status = smp_query_cpu_stopped(pcpu);
+ if (cpu_status == QCSS_STOPPED ||
+ cpu_status == QCSS_HARDWARE_ERROR)
+ break;
+ cpu_relax();
- for (tries = 0; tries < 25; tries++) {
- cpu_status = smp_query_cpu_stopped(pcpu);
- if (cpu_status == QCSS_STOPPED ||
- cpu_status == QCSS_HARDWARE_ERROR)
- break;
- cpu_relax();
- }
}
if (cpu_status != 0) {
@@ -359,28 +259,14 @@
if (get_hard_smp_processor_id(cpu) != thread)
continue;
- if (get_cpu_current_state(cpu) == CPU_STATE_OFFLINE)
+ if (!cpu_online(cpu))
break;
- if (get_cpu_current_state(cpu) == CPU_STATE_ONLINE) {
- set_preferred_offline_state(cpu,
- CPU_STATE_OFFLINE);
- cpu_maps_update_done();
- timed_topology_update(1);
- rc = device_offline(get_cpu_device(cpu));
- if (rc)
- goto out;
- cpu_maps_update_begin();
- break;
- }
-
- /*
- * The cpu is in CPU_STATE_INACTIVE.
- * Upgrade it's state to CPU_STATE_OFFLINE.
- */
- set_preferred_offline_state(cpu, CPU_STATE_OFFLINE);
- WARN_ON(plpar_hcall_norets(H_PROD, thread) != H_SUCCESS);
- __cpu_die(cpu);
+ cpu_maps_update_done();
+ rc = device_offline(get_cpu_device(cpu));
+ if (rc)
+ goto out;
+ cpu_maps_update_begin();
break;
}
if (cpu == num_possible_cpus()) {
@@ -414,10 +300,7 @@
for_each_present_cpu(cpu) {
if (get_hard_smp_processor_id(cpu) != thread)
continue;
- BUG_ON(get_cpu_current_state(cpu)
- != CPU_STATE_OFFLINE);
cpu_maps_update_done();
- timed_topology_update(1);
find_and_online_cpu_nid(cpu);
rc = device_online(get_cpu_device(cpu));
if (rc) {
@@ -854,7 +737,6 @@
parent = of_find_node_by_path("/cpus");
if (!parent) {
pr_warn("Could not find CPU root node in device tree\n");
- kfree(cpu_drcs);
return -1;
}
@@ -896,25 +778,6 @@
return rc;
}
-int dlpar_cpu_readd(int cpu)
-{
- struct device_node *dn;
- struct device *dev;
- u32 drc_index;
- int rc;
-
- dev = get_cpu_device(cpu);
- dn = dev->of_node;
-
- rc = of_property_read_u32(dn, "ibm,my-drc-index", &drc_index);
-
- rc = dlpar_cpu_remove_by_index(drc_index);
- if (!rc)
- rc = dlpar_cpu_add(drc_index);
-
- return rc;
-}
-
int dlpar_cpu(struct pseries_hp_errorlog *hp_elog)
{
u32 count, drc_index;
@@ -1013,27 +876,8 @@
.notifier_call = pseries_smp_notifier,
};
-#define MAX_CEDE_LATENCY_LEVELS 4
-#define CEDE_LATENCY_PARAM_LENGTH 10
-#define CEDE_LATENCY_PARAM_MAX_LENGTH \
- (MAX_CEDE_LATENCY_LEVELS * CEDE_LATENCY_PARAM_LENGTH * sizeof(char))
-#define CEDE_LATENCY_TOKEN 45
-
-static char cede_parameters[CEDE_LATENCY_PARAM_MAX_LENGTH];
-
-static int parse_cede_parameters(void)
-{
- memset(cede_parameters, 0, CEDE_LATENCY_PARAM_MAX_LENGTH);
- return rtas_call(rtas_token("ibm,get-system-parameter"), 3, 1,
- NULL,
- CEDE_LATENCY_TOKEN,
- __pa(cede_parameters),
- CEDE_LATENCY_PARAM_MAX_LENGTH);
-}
-
static int __init pseries_cpu_hotplug_init(void)
{
- int cpu;
int qcss_tok;
#ifdef CONFIG_ARCH_CPU_PROBE_RELEASE
@@ -1056,16 +900,8 @@
smp_ops->cpu_die = pseries_cpu_die;
/* Processors can be added/removed only on LPAR */
- if (firmware_has_feature(FW_FEATURE_LPAR)) {
+ if (firmware_has_feature(FW_FEATURE_LPAR))
of_reconfig_notifier_register(&pseries_smp_nb);
- cpu_maps_update_begin();
- if (cede_offline_enabled && parse_cede_parameters() == 0) {
- default_offline_state = CPU_STATE_INACTIVE;
- for_each_online_cpu(cpu)
- set_default_offline_state(cpu);
- }
- cpu_maps_update_done();
- }
return 0;
}
diff --git a/arch/powerpc/platforms/pseries/hotplug-memory.c b/arch/powerpc/platforms/pseries/hotplug-memory.c
index 5ace2f9..5d545b7 100644
--- a/arch/powerpc/platforms/pseries/hotplug-memory.c
+++ b/arch/powerpc/platforms/pseries/hotplug-memory.c
@@ -22,12 +22,10 @@
#include <asm/drmem.h>
#include "pseries.h"
-static bool rtas_hp_event;
-
unsigned long pseries_memory_block_size(void)
{
struct device_node *np;
- unsigned int memblock_size = MIN_MEMORY_BLOCK_SIZE;
+ u64 memblock_size = MIN_MEMORY_BLOCK_SIZE;
struct resource r;
np = of_find_node_by_path("/ibm,dynamic-reconfiguration-memory");
@@ -487,40 +485,6 @@
return rc;
}
-static int dlpar_memory_readd_by_index(u32 drc_index)
-{
- struct drmem_lmb *lmb;
- int lmb_found;
- int rc;
-
- pr_info("Attempting to update LMB, drc index %x\n", drc_index);
-
- lmb_found = 0;
- for_each_drmem_lmb(lmb) {
- if (lmb->drc_index == drc_index) {
- lmb_found = 1;
- rc = dlpar_remove_lmb(lmb);
- if (!rc) {
- rc = dlpar_add_lmb(lmb);
- if (rc)
- dlpar_release_drc(lmb->drc_index);
- }
- break;
- }
- }
-
- if (!lmb_found)
- rc = -EINVAL;
-
- if (rc)
- pr_info("Failed to update memory at %llx\n",
- lmb->base_addr);
- else
- pr_info("Memory at %llx was updated\n", lmb->base_addr);
-
- return rc;
-}
-
static int dlpar_memory_remove_by_ic(u32 lmbs_to_remove, u32 drc_index)
{
struct drmem_lmb *lmb, *start_lmb, *end_lmb;
@@ -617,10 +581,6 @@
{
return -EOPNOTSUPP;
}
-static int dlpar_memory_readd_by_index(u32 drc_index)
-{
- return -EOPNOTSUPP;
-}
static int dlpar_memory_remove_by_ic(u32 lmbs_to_remove, u32 drc_index)
{
@@ -903,21 +863,14 @@
}
break;
- case PSERIES_HP_ELOG_ACTION_READD:
- drc_index = hp_elog->_drc_u.drc_index;
- rc = dlpar_memory_readd_by_index(drc_index);
- break;
default:
pr_err("Invalid action (%d) specified\n", hp_elog->action);
rc = -EINVAL;
break;
}
- if (!rc) {
- rtas_hp_event = true;
+ if (!rc)
rc = drmem_update_dt();
- rtas_hp_event = false;
- }
unlock_device_hotplug();
return rc;
@@ -953,60 +906,6 @@
return (ret < 0) ? -EINVAL : 0;
}
-static int pseries_update_drconf_memory(struct of_reconfig_data *pr)
-{
- struct of_drconf_cell_v1 *new_drmem, *old_drmem;
- unsigned long memblock_size;
- u32 entries;
- __be32 *p;
- int i, rc = -EINVAL;
-
- if (rtas_hp_event)
- return 0;
-
- memblock_size = pseries_memory_block_size();
- if (!memblock_size)
- return -EINVAL;
-
- if (!pr->old_prop)
- return 0;
-
- p = (__be32 *) pr->old_prop->value;
- if (!p)
- return -EINVAL;
-
- /* The first int of the property is the number of lmb's described
- * by the property. This is followed by an array of of_drconf_cell
- * entries. Get the number of entries and skip to the array of
- * of_drconf_cell's.
- */
- entries = be32_to_cpu(*p++);
- old_drmem = (struct of_drconf_cell_v1 *)p;
-
- p = (__be32 *)pr->prop->value;
- p++;
- new_drmem = (struct of_drconf_cell_v1 *)p;
-
- for (i = 0; i < entries; i++) {
- if ((be32_to_cpu(old_drmem[i].flags) & DRCONF_MEM_ASSIGNED) &&
- (!(be32_to_cpu(new_drmem[i].flags) & DRCONF_MEM_ASSIGNED))) {
- rc = pseries_remove_memblock(
- be64_to_cpu(old_drmem[i].base_addr),
- memblock_size);
- break;
- } else if ((!(be32_to_cpu(old_drmem[i].flags) &
- DRCONF_MEM_ASSIGNED)) &&
- (be32_to_cpu(new_drmem[i].flags) &
- DRCONF_MEM_ASSIGNED)) {
- rc = memblock_add(be64_to_cpu(old_drmem[i].base_addr),
- memblock_size);
- rc = (rc < 0) ? -EINVAL : 0;
- break;
- }
- }
- return rc;
-}
-
static int pseries_memory_notifier(struct notifier_block *nb,
unsigned long action, void *data)
{
@@ -1020,10 +919,6 @@
case OF_RECONFIG_DETACH_NODE:
err = pseries_remove_mem_node(rd->dn);
break;
- case OF_RECONFIG_UPDATE_PROPERTY:
- if (!strcmp(rd->prop->name, "ibm,dynamic-memory"))
- err = pseries_update_drconf_memory(rd);
- break;
}
return notifier_from_errno(err);
}
diff --git a/arch/powerpc/platforms/pseries/hvcserver.c b/arch/powerpc/platforms/pseries/hvcserver.c
index 267139b..96e18d3 100644
--- a/arch/powerpc/platforms/pseries/hvcserver.c
+++ b/arch/powerpc/platforms/pseries/hvcserver.c
@@ -45,7 +45,7 @@
case H_LONG_BUSY_ORDER_10_SEC:
case H_LONG_BUSY_ORDER_100_SEC:
return -EBUSY;
- case H_FUNCTION: /* fall through */
+ case H_FUNCTION:
default:
return -EPERM;
}
diff --git a/arch/powerpc/platforms/pseries/lpar.c b/arch/powerpc/platforms/pseries/lpar.c
index f71ff2c..baf24ea 100644
--- a/arch/powerpc/platforms/pseries/lpar.c
+++ b/arch/powerpc/platforms/pseries/lpar.c
@@ -1681,9 +1681,11 @@
if (table_size)
flags |= PROC_TABLE_NEW;
- if (radix_enabled())
- flags |= PROC_TABLE_RADIX | PROC_TABLE_GTSE;
- else
+ if (radix_enabled()) {
+ flags |= PROC_TABLE_RADIX;
+ if (mmu_has_feature(MMU_FTR_GTSE))
+ flags |= PROC_TABLE_GTSE;
+ } else
flags |= PROC_TABLE_HPT_SLB;
for (;;) {
rc = plpar_hcall_norets(H_REGISTER_PROC_TBL, flags, base,
diff --git a/arch/powerpc/platforms/pseries/mobility.c b/arch/powerpc/platforms/pseries/mobility.c
index 10d9829..d6f4162 100644
--- a/arch/powerpc/platforms/pseries/mobility.c
+++ b/arch/powerpc/platforms/pseries/mobility.c
@@ -6,6 +6,9 @@
* Copyright (C) 2010 IBM Corporation
*/
+
+#define pr_fmt(fmt) "mobility: " fmt
+
#include <linux/cpu.h>
#include <linux/kernel.h>
#include <linux/kobject.h>
@@ -64,6 +67,8 @@
if (!dn)
return -ENOENT;
+ pr_debug("removing node %pOFfp\n", dn);
+
dlpar_detach_node(dn);
of_node_put(dn);
return 0;
@@ -122,6 +127,7 @@
}
if (!more) {
+ pr_debug("updating node %pOF property %s\n", dn, name);
of_update_property(dn, new_prop);
*prop = NULL;
}
@@ -240,33 +246,12 @@
if (rc)
dlpar_free_cc_nodes(dn);
+ pr_debug("added node %pOFfp\n", dn);
+
of_node_put(parent_dn);
return rc;
}
-static void prrn_update_node(__be32 phandle)
-{
- struct pseries_hp_errorlog hp_elog;
- struct device_node *dn;
-
- /*
- * If a node is found from a the given phandle, the phandle does not
- * represent the drc index of an LMB and we can ignore.
- */
- dn = of_find_node_by_phandle(be32_to_cpu(phandle));
- if (dn) {
- of_node_put(dn);
- return;
- }
-
- hp_elog.resource = PSERIES_HP_ELOG_RESOURCE_MEM;
- hp_elog.action = PSERIES_HP_ELOG_ACTION_READD;
- hp_elog.id_type = PSERIES_HP_ELOG_ID_DRC_INDEX;
- hp_elog._drc_u.drc_index = phandle;
-
- handle_dlpar_errorlog(&hp_elog);
-}
-
int pseries_devicetree_update(s32 scope)
{
char *rtas_buf;
@@ -305,10 +290,6 @@
break;
case UPDATE_DT_NODE:
update_dt_node(phandle, scope);
-
- if (scope == PRRN_SCOPE)
- prrn_update_node(phandle);
-
break;
case ADD_DT_NODE:
drc_index = *data++;
@@ -388,8 +369,6 @@
if (rc)
return rc;
- stop_topology_update();
-
do {
rc = rtas_ibm_suspend_me(streamid);
if (rc == -EAGAIN)
@@ -401,8 +380,6 @@
post_mobility_fixup();
- start_topology_update();
-
return count;
}
@@ -427,11 +404,11 @@
rc = sysfs_create_file(mobility_kobj, &class_attr_migration.attr);
if (rc)
- pr_err("mobility: unable to create migration sysfs file (%d)\n", rc);
+ pr_err("unable to create migration sysfs file (%d)\n", rc);
rc = sysfs_create_file(mobility_kobj, &class_attr_api_version.attr.attr);
if (rc)
- pr_err("mobility: unable to create api_version sysfs file (%d)\n", rc);
+ pr_err("unable to create api_version sysfs file (%d)\n", rc);
return 0;
}
diff --git a/arch/powerpc/platforms/pseries/offline_states.h b/arch/powerpc/platforms/pseries/offline_states.h
deleted file mode 100644
index 51414ae..0000000
--- a/arch/powerpc/platforms/pseries/offline_states.h
+++ /dev/null
@@ -1,38 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-#ifndef _OFFLINE_STATES_H_
-#define _OFFLINE_STATES_H_
-
-/* Cpu offline states go here */
-enum cpu_state_vals {
- CPU_STATE_OFFLINE,
- CPU_STATE_INACTIVE,
- CPU_STATE_ONLINE,
- CPU_MAX_OFFLINE_STATES
-};
-
-#ifdef CONFIG_HOTPLUG_CPU
-extern enum cpu_state_vals get_cpu_current_state(int cpu);
-extern void set_cpu_current_state(int cpu, enum cpu_state_vals state);
-extern void set_preferred_offline_state(int cpu, enum cpu_state_vals state);
-extern void set_default_offline_state(int cpu);
-#else
-static inline enum cpu_state_vals get_cpu_current_state(int cpu)
-{
- return CPU_STATE_ONLINE;
-}
-
-static inline void set_cpu_current_state(int cpu, enum cpu_state_vals state)
-{
-}
-
-static inline void set_preferred_offline_state(int cpu, enum cpu_state_vals state)
-{
-}
-
-static inline void set_default_offline_state(int cpu)
-{
-}
-#endif
-
-extern enum cpu_state_vals get_preferred_offline_state(int cpu);
-#endif
diff --git a/arch/powerpc/platforms/pseries/papr_scm.c b/arch/powerpc/platforms/pseries/papr_scm.c
index 9c56907..f439f0d 100644
--- a/arch/powerpc/platforms/pseries/papr_scm.c
+++ b/arch/powerpc/platforms/pseries/papr_scm.c
@@ -13,9 +13,11 @@
#include <linux/platform_device.h>
#include <linux/delay.h>
#include <linux/seq_buf.h>
+#include <linux/nd.h>
#include <asm/plpar_wrappers.h>
#include <asm/papr_pdsm.h>
+#include <asm/mce.h>
#define BIND_ANY_ADDR (~0ul)
@@ -62,6 +64,26 @@
PAPR_PMEM_HEALTH_FATAL | \
PAPR_PMEM_HEALTH_UNHEALTHY)
+#define PAPR_SCM_PERF_STATS_EYECATCHER __stringify(SCMSTATS)
+#define PAPR_SCM_PERF_STATS_VERSION 0x1
+
+/* Struct holding a single performance metric */
+struct papr_scm_perf_stat {
+ u8 stat_id[8];
+ __be64 stat_val;
+} __packed;
+
+/* Struct exchanged between kernel and PHYP for fetching drc perf stats */
+struct papr_scm_perf_stats {
+ u8 eye_catcher[8];
+ /* Should be PAPR_SCM_PERF_STATS_VERSION */
+ __be32 stats_version;
+ /* Number of stats following */
+ __be32 num_statistics;
+ /* zero or more performance matrics */
+ struct papr_scm_perf_stat scm_statistic[];
+} __packed;
+
/* private struct associated with each region */
struct papr_scm_priv {
struct platform_device *pdev;
@@ -80,6 +102,7 @@
struct resource res;
struct nd_region *region;
struct nd_interleave_set nd_set;
+ struct list_head region_list;
/* Protect dimm health data from concurrent read/writes */
struct mutex health_mutex;
@@ -89,8 +112,14 @@
/* Health information for the dimm */
u64 health_bitmap;
+
+ /* length of the stat buffer as expected by phyp */
+ size_t stat_buffer_len;
};
+static LIST_HEAD(papr_nd_regions);
+static DEFINE_MUTEX(papr_ndr_lock);
+
static int drc_pmem_bind(struct papr_scm_priv *p)
{
unsigned long ret[PLPAR_HCALL_BUFSIZE];
@@ -195,6 +224,79 @@
}
/*
+ * Query the Dimm performance stats from PHYP and copy them (if returned) to
+ * provided struct papr_scm_perf_stats instance 'stats' that can hold atleast
+ * (num_stats + header) bytes.
+ * - If buff_stats == NULL the return value is the size in byes of the buffer
+ * needed to hold all supported performance-statistics.
+ * - If buff_stats != NULL and num_stats == 0 then we copy all known
+ * performance-statistics to 'buff_stat' and expect to be large enough to
+ * hold them.
+ * - if buff_stats != NULL and num_stats > 0 then copy the requested
+ * performance-statistics to buff_stats.
+ */
+static ssize_t drc_pmem_query_stats(struct papr_scm_priv *p,
+ struct papr_scm_perf_stats *buff_stats,
+ unsigned int num_stats)
+{
+ unsigned long ret[PLPAR_HCALL_BUFSIZE];
+ size_t size;
+ s64 rc;
+
+ /* Setup the out buffer */
+ if (buff_stats) {
+ memcpy(buff_stats->eye_catcher,
+ PAPR_SCM_PERF_STATS_EYECATCHER, 8);
+ buff_stats->stats_version =
+ cpu_to_be32(PAPR_SCM_PERF_STATS_VERSION);
+ buff_stats->num_statistics =
+ cpu_to_be32(num_stats);
+
+ /*
+ * Calculate the buffer size based on num-stats provided
+ * or use the prefetched max buffer length
+ */
+ if (num_stats)
+ /* Calculate size from the num_stats */
+ size = sizeof(struct papr_scm_perf_stats) +
+ num_stats * sizeof(struct papr_scm_perf_stat);
+ else
+ size = p->stat_buffer_len;
+ } else {
+ /* In case of no out buffer ignore the size */
+ size = 0;
+ }
+
+ /* Do the HCALL asking PHYP for info */
+ rc = plpar_hcall(H_SCM_PERFORMANCE_STATS, ret, p->drc_index,
+ buff_stats ? virt_to_phys(buff_stats) : 0,
+ size);
+
+ /* Check if the error was due to an unknown stat-id */
+ if (rc == H_PARTIAL) {
+ dev_err(&p->pdev->dev,
+ "Unknown performance stats, Err:0x%016lX\n", ret[0]);
+ return -ENOENT;
+ } else if (rc != H_SUCCESS) {
+ dev_err(&p->pdev->dev,
+ "Failed to query performance stats, Err:%lld\n", rc);
+ return -EIO;
+
+ } else if (!size) {
+ /* Handle case where stat buffer size was requested */
+ dev_dbg(&p->pdev->dev,
+ "Performance stats size %ld\n", ret[0]);
+ return ret[0];
+ }
+
+ /* Successfully fetched the requested stats from phyp */
+ dev_dbg(&p->pdev->dev,
+ "Performance stats returned %d stats\n",
+ be32_to_cpu(buff_stats->num_statistics));
+ return 0;
+}
+
+/*
* Issue hcall to retrieve dimm health info and populate papr_scm_priv with the
* health information.
*/
@@ -416,6 +518,51 @@
return 0;
}
+static int papr_pdsm_fuel_gauge(struct papr_scm_priv *p,
+ union nd_pdsm_payload *payload)
+{
+ int rc, size;
+ u64 statval;
+ struct papr_scm_perf_stat *stat;
+ struct papr_scm_perf_stats *stats;
+
+ /* Silently fail if fetching performance metrics isn't supported */
+ if (!p->stat_buffer_len)
+ return 0;
+
+ /* Allocate request buffer enough to hold single performance stat */
+ size = sizeof(struct papr_scm_perf_stats) +
+ sizeof(struct papr_scm_perf_stat);
+
+ stats = kzalloc(size, GFP_KERNEL);
+ if (!stats)
+ return -ENOMEM;
+
+ stat = &stats->scm_statistic[0];
+ memcpy(&stat->stat_id, "MemLife ", sizeof(stat->stat_id));
+ stat->stat_val = 0;
+
+ /* Fetch the fuel gauge and populate it in payload */
+ rc = drc_pmem_query_stats(p, stats, 1);
+ if (rc < 0) {
+ dev_dbg(&p->pdev->dev, "Err(%d) fetching fuel gauge\n", rc);
+ goto free_stats;
+ }
+
+ statval = be64_to_cpu(stat->stat_val);
+ dev_dbg(&p->pdev->dev,
+ "Fetched fuel-gauge %llu", statval);
+ payload->health.extension_flags |=
+ PDSM_DIMM_HEALTH_RUN_GAUGE_VALID;
+ payload->health.dimm_fuel_gauge = statval;
+
+ rc = sizeof(struct nd_papr_pdsm_health);
+
+free_stats:
+ kfree(stats);
+ return rc;
+}
+
/* Fetch the DIMM health info and populate it in provided package. */
static int papr_pdsm_health(struct papr_scm_priv *p,
union nd_pdsm_payload *payload)
@@ -456,6 +603,10 @@
/* struct populated hence can release the mutex now */
mutex_unlock(&p->health_mutex);
+
+ /* Populate the fuel gauge meter in the payload */
+ papr_pdsm_fuel_gauge(p, payload);
+
rc = sizeof(struct nd_papr_pdsm_health);
out:
@@ -631,6 +782,48 @@
return 0;
}
+static ssize_t perf_stats_show(struct device *dev,
+ struct device_attribute *attr, char *buf)
+{
+ int index, rc;
+ struct seq_buf s;
+ struct papr_scm_perf_stat *stat;
+ struct papr_scm_perf_stats *stats;
+ struct nvdimm *dimm = to_nvdimm(dev);
+ struct papr_scm_priv *p = nvdimm_provider_data(dimm);
+
+ if (!p->stat_buffer_len)
+ return -ENOENT;
+
+ /* Allocate the buffer for phyp where stats are written */
+ stats = kzalloc(p->stat_buffer_len, GFP_KERNEL);
+ if (!stats)
+ return -ENOMEM;
+
+ /* Ask phyp to return all dimm perf stats */
+ rc = drc_pmem_query_stats(p, stats, 0);
+ if (rc)
+ goto free_stats;
+ /*
+ * Go through the returned output buffer and print stats and
+ * values. Since stat_id is essentially a char string of
+ * 8 bytes, simply use the string format specifier to print it.
+ */
+ seq_buf_init(&s, buf, PAGE_SIZE);
+ for (index = 0, stat = stats->scm_statistic;
+ index < be32_to_cpu(stats->num_statistics);
+ ++index, ++stat) {
+ seq_buf_printf(&s, "%.8s = 0x%016llX\n",
+ stat->stat_id,
+ be64_to_cpu(stat->stat_val));
+ }
+
+free_stats:
+ kfree(stats);
+ return rc ? rc : seq_buf_used(&s);
+}
+DEVICE_ATTR_RO(perf_stats);
+
static ssize_t flags_show(struct device *dev,
struct device_attribute *attr, char *buf)
{
@@ -676,6 +869,7 @@
/* papr_scm specific dimm attributes */
static struct attribute *papr_nd_attributes[] = {
&dev_attr_flags.attr,
+ &dev_attr_perf_stats.attr,
NULL,
};
@@ -696,6 +890,7 @@
struct nd_region_desc ndr_desc;
unsigned long dimm_flags;
int target_nid, online_nid;
+ ssize_t stat_size;
p->bus_desc.ndctl = papr_scm_ndctl;
p->bus_desc.module = THIS_MODULE;
@@ -759,6 +954,20 @@
dev_info(dev, "Region registered with target node %d and online node %d",
target_nid, online_nid);
+ mutex_lock(&papr_ndr_lock);
+ list_add_tail(&p->region_list, &papr_nd_regions);
+ mutex_unlock(&papr_ndr_lock);
+
+ /* Try retriving the stat buffer and see if its supported */
+ stat_size = drc_pmem_query_stats(p, NULL, 0);
+ if (stat_size > 0) {
+ p->stat_buffer_len = stat_size;
+ dev_dbg(&p->pdev->dev, "Max perf-stat size %lu-bytes\n",
+ p->stat_buffer_len);
+ } else {
+ dev_info(&p->pdev->dev, "Dimm performance stats unavailable\n");
+ }
+
return 0;
err: nvdimm_bus_unregister(p->bus);
@@ -766,6 +975,68 @@
return -ENXIO;
}
+static void papr_scm_add_badblock(struct nd_region *region,
+ struct nvdimm_bus *bus, u64 phys_addr)
+{
+ u64 aligned_addr = ALIGN_DOWN(phys_addr, L1_CACHE_BYTES);
+
+ if (nvdimm_bus_add_badrange(bus, aligned_addr, L1_CACHE_BYTES)) {
+ pr_err("Bad block registration for 0x%llx failed\n", phys_addr);
+ return;
+ }
+
+ pr_debug("Add memory range (0x%llx - 0x%llx) as bad range\n",
+ aligned_addr, aligned_addr + L1_CACHE_BYTES);
+
+ nvdimm_region_notify(region, NVDIMM_REVALIDATE_POISON);
+}
+
+static int handle_mce_ue(struct notifier_block *nb, unsigned long val,
+ void *data)
+{
+ struct machine_check_event *evt = data;
+ struct papr_scm_priv *p;
+ u64 phys_addr;
+ bool found = false;
+
+ if (evt->error_type != MCE_ERROR_TYPE_UE)
+ return NOTIFY_DONE;
+
+ if (list_empty(&papr_nd_regions))
+ return NOTIFY_DONE;
+
+ /*
+ * The physical address obtained here is PAGE_SIZE aligned, so get the
+ * exact address from the effective address
+ */
+ phys_addr = evt->u.ue_error.physical_address +
+ (evt->u.ue_error.effective_address & ~PAGE_MASK);
+
+ if (!evt->u.ue_error.physical_address_provided ||
+ !is_zone_device_page(pfn_to_page(phys_addr >> PAGE_SHIFT)))
+ return NOTIFY_DONE;
+
+ /* mce notifier is called from a process context, so mutex is safe */
+ mutex_lock(&papr_ndr_lock);
+ list_for_each_entry(p, &papr_nd_regions, region_list) {
+ if (phys_addr >= p->res.start && phys_addr <= p->res.end) {
+ found = true;
+ break;
+ }
+ }
+
+ if (found)
+ papr_scm_add_badblock(p->region, p->bus, phys_addr);
+
+ mutex_unlock(&papr_ndr_lock);
+
+ return found ? NOTIFY_OK : NOTIFY_DONE;
+}
+
+static struct notifier_block mce_ue_nb = {
+ .notifier_call = handle_mce_ue
+};
+
static int papr_scm_probe(struct platform_device *pdev)
{
struct device_node *dn = pdev->dev.of_node;
@@ -866,6 +1137,10 @@
{
struct papr_scm_priv *p = platform_get_drvdata(pdev);
+ mutex_lock(&papr_ndr_lock);
+ list_del(&p->region_list);
+ mutex_unlock(&papr_ndr_lock);
+
nvdimm_bus_unregister(p->bus);
drc_pmem_unbind(p);
kfree(p->bus_desc.provider_name);
@@ -876,6 +1151,7 @@
static const struct of_device_id papr_scm_match[] = {
{ .compatible = "ibm,pmemory" },
+ { .compatible = "ibm,pmemory-v2" },
{ },
};
@@ -888,7 +1164,25 @@
},
};
-module_platform_driver(papr_scm_driver);
+static int __init papr_scm_init(void)
+{
+ int ret;
+
+ ret = platform_driver_register(&papr_scm_driver);
+ if (!ret)
+ mce_register_notifier(&mce_ue_nb);
+
+ return ret;
+}
+module_init(papr_scm_init);
+
+static void __exit papr_scm_exit(void)
+{
+ mce_unregister_notifier(&mce_ue_nb);
+ platform_driver_unregister(&papr_scm_driver);
+}
+module_exit(papr_scm_exit);
+
MODULE_DEVICE_TABLE(of, papr_scm_match);
MODULE_LICENSE("GPL");
MODULE_AUTHOR("IBM Corporation");
diff --git a/arch/powerpc/platforms/pseries/pci_dlpar.c b/arch/powerpc/platforms/pseries/pci_dlpar.c
index b3a38f5..f9ae17e 100644
--- a/arch/powerpc/platforms/pseries/pci_dlpar.c
+++ b/arch/powerpc/platforms/pseries/pci_dlpar.c
@@ -34,7 +34,7 @@
pci_devs_phb_init_dynamic(phb);
/* Create EEH devices for the PHB */
- eeh_dev_phb_init_dynamic(phb);
+ eeh_phb_pe_create(phb);
if (dn->child)
pseries_eeh_init_edev_recursive(PCI_DN(dn));
diff --git a/arch/powerpc/platforms/pseries/pmem.c b/arch/powerpc/platforms/pseries/pmem.c
index f860a89..e1dc5d3 100644
--- a/arch/powerpc/platforms/pseries/pmem.c
+++ b/arch/powerpc/platforms/pseries/pmem.c
@@ -24,7 +24,6 @@
#include <asm/topology.h>
#include "pseries.h"
-#include "offline_states.h"
static struct device_node *pmem_node;
@@ -147,6 +146,12 @@
static int pseries_pmem_init(void)
{
+ /*
+ * Only supported on POWER8 and above.
+ */
+ if (!cpu_has_feature(CPU_FTR_ARCH_207S))
+ return 0;
+
pmem_node = of_find_node_by_type(NULL, "ibm,persistent-memory");
if (!pmem_node)
return 0;
diff --git a/arch/powerpc/platforms/pseries/setup.c b/arch/powerpc/platforms/pseries/setup.c
index 27094c8..2f4ee0a 100644
--- a/arch/powerpc/platforms/pseries/setup.c
+++ b/arch/powerpc/platforms/pseries/setup.c
@@ -359,7 +359,7 @@
* to ever be a problem in practice we can move this into a kernel thread to
* finish off the process later in boot.
*/
-void pseries_enable_reloc_on_exc(void)
+bool pseries_enable_reloc_on_exc(void)
{
long rc;
unsigned int delay, total_delay = 0;
@@ -370,11 +370,13 @@
if (rc == H_P2) {
pr_info("Relocation on exceptions not"
" supported\n");
+ return false;
} else if (rc != H_SUCCESS) {
pr_warn("Unable to enable relocation"
" on exceptions: %ld\n", rc);
+ return false;
}
- break;
+ return true;
}
delay = get_longbusy_msecs(rc);
@@ -383,7 +385,7 @@
pr_warn("Warning: Giving up waiting to enable "
"relocation on exceptions (%u msec)!\n",
total_delay);
- return;
+ return false;
}
mdelay(delay);
@@ -746,6 +748,11 @@
smp_init_pseries();
+ if (radix_enabled() && !mmu_has_feature(MMU_FTR_GTSE))
+ if (!firmware_has_feature(FW_FEATURE_RPT_INVALIDATE))
+ panic("BUG: Radix support requires either GTSE or RPT_INVALIDATE\n");
+
+
/* openpic global configuration register (64-bit format). */
/* openpic Interrupt Source Unit pointer (64-bit format). */
/* python0 facility area (mmio) (64-bit format) REAL address. */
@@ -772,8 +779,10 @@
if (firmware_has_feature(FW_FEATURE_LPAR)) {
vpa_init(boot_cpuid);
- if (lppaca_shared_proc(get_lppaca()))
+ if (lppaca_shared_proc(get_lppaca())) {
static_branch_enable(&shared_processor);
+ pv_spinlocks_init();
+ }
ppc_md.power_save = pseries_lpar_idle;
ppc_md.enable_pmcs = pseries_lpar_enable_pmcs;
@@ -832,12 +841,15 @@
return plpar_hcall_norets(H_SET_XDABR, dabr, dabrx);
}
-static int pseries_set_dawr(unsigned long dawr, unsigned long dawrx)
+static int pseries_set_dawr(int nr, unsigned long dawr, unsigned long dawrx)
{
/* PAPR says we can't set HYP */
dawrx &= ~DAWRX_HYP;
- return plpar_set_watchpoint0(dawr, dawrx);
+ if (nr == 0)
+ return plpar_set_watchpoint0(dawr, dawrx);
+ else
+ return plpar_set_watchpoint1(dawr, dawrx);
}
#define CMO_CHARACTERISTICS_TOKEN 44
diff --git a/arch/powerpc/platforms/pseries/smp.c b/arch/powerpc/platforms/pseries/smp.c
index 6891710..9292249 100644
--- a/arch/powerpc/platforms/pseries/smp.c
+++ b/arch/powerpc/platforms/pseries/smp.c
@@ -44,8 +44,6 @@
#include <asm/svm.h>
#include "pseries.h"
-#include "offline_states.h"
-
/*
* The Primary thread of each non-boot processor was started from the OF client
@@ -108,10 +106,7 @@
/* Fixup atomic count: it exited inside IRQ handler. */
task_thread_info(paca_ptrs[lcpu]->__current)->preempt_count = 0;
-#ifdef CONFIG_HOTPLUG_CPU
- if (get_cpu_current_state(lcpu) == CPU_STATE_INACTIVE)
- goto out;
-#endif
+
/*
* If the RTAS start-cpu token does not exist then presume the
* cpu is already spinning.
@@ -126,9 +121,6 @@
return 0;
}
-#ifdef CONFIG_HOTPLUG_CPU
-out:
-#endif
return 1;
}
@@ -143,10 +135,6 @@
vpa_init(cpu);
cpumask_clear_cpu(cpu, of_spin_mask);
-#ifdef CONFIG_HOTPLUG_CPU
- set_cpu_current_state(cpu, CPU_STATE_ONLINE);
- set_default_offline_state(cpu);
-#endif
}
static int smp_pSeries_kick_cpu(int nr)
@@ -163,20 +151,6 @@
* the processor will continue on to secondary_start
*/
paca_ptrs[nr]->cpu_start = 1;
-#ifdef CONFIG_HOTPLUG_CPU
- set_preferred_offline_state(nr, CPU_STATE_ONLINE);
-
- if (get_cpu_current_state(nr) == CPU_STATE_INACTIVE) {
- long rc;
- unsigned long hcpuid;
-
- hcpuid = get_hard_smp_processor_id(nr);
- rc = plpar_hcall_norets(H_PROD, hcpuid);
- if (rc != H_SUCCESS)
- printk(KERN_ERR "Error: Prod to wake up processor %d "
- "Ret= %ld\n", nr, rc);
- }
-#endif
return 0;
}
@@ -188,13 +162,16 @@
return 0;
}
-static void smp_pseries_cause_ipi(int cpu)
+/* Cause IPI as setup by the interrupt controller (xics or xive) */
+static void (*ic_cause_ipi)(int cpu) __ro_after_init;
+
+/* Use msgsndp doorbells target is a sibling, else use interrupt controller */
+static void dbell_or_ic_cause_ipi(int cpu)
{
- /* POWER9 should not use this handler */
if (doorbell_try_core_ipi(cpu))
return;
- icp_ops->cause_ipi(cpu);
+ ic_cause_ipi(cpu);
}
static int pseries_cause_nmi_ipi(int cpu)
@@ -218,26 +195,49 @@
return 0;
}
-static __init void pSeries_smp_probe_xics(void)
-{
- xics_smp_probe();
-
- if (cpu_has_feature(CPU_FTR_DBELL) && !is_secure_guest())
- smp_ops->cause_ipi = smp_pseries_cause_ipi;
- else
- smp_ops->cause_ipi = icp_ops->cause_ipi;
-}
-
static __init void pSeries_smp_probe(void)
{
if (xive_enabled())
- /*
- * Don't use P9 doorbells when XIVE is enabled. IPIs
- * using MMIOs should be faster
- */
xive_smp_probe();
else
- pSeries_smp_probe_xics();
+ xics_smp_probe();
+
+ /* No doorbell facility, must use the interrupt controller for IPIs */
+ if (!cpu_has_feature(CPU_FTR_DBELL))
+ return;
+
+ /* Doorbells can only be used for IPIs between SMT siblings */
+ if (!cpu_has_feature(CPU_FTR_SMT))
+ return;
+
+ if (is_kvm_guest()) {
+ /*
+ * KVM emulates doorbells by disabling FSCR[MSGP] so msgsndp
+ * faults to the hypervisor which then reads the instruction
+ * from guest memory, which tends to be slower than using XIVE.
+ */
+ if (xive_enabled())
+ return;
+
+ /*
+ * XICS hcalls aren't as fast, so we can use msgsndp (which
+ * also helps exercise KVM emulation), however KVM can't
+ * emulate secure guests because it can't read the instruction
+ * out of their memory.
+ */
+ if (is_secure_guest())
+ return;
+ }
+
+ /*
+ * Under PowerVM, FSCR[MSGP] is enabled as guest vCPU siblings are
+ * gang scheduled on the same physical core, so doorbells are always
+ * faster than the interrupt controller, and they can be used by
+ * secure guests.
+ */
+
+ ic_cause_ipi = smp_ops->cause_ipi;
+ smp_ops->cause_ipi = dbell_or_ic_cause_ipi;
}
static struct smp_ops_t pseries_smp_ops = {
diff --git a/arch/powerpc/platforms/pseries/suspend.c b/arch/powerpc/platforms/pseries/suspend.c
index 0a24a5a..81e0ac5 100644
--- a/arch/powerpc/platforms/pseries/suspend.c
+++ b/arch/powerpc/platforms/pseries/suspend.c
@@ -132,15 +132,11 @@
struct device_attribute *attr,
const char *buf, size_t count)
{
- cpumask_var_t offline_mask;
int rc;
if (!capable(CAP_SYS_ADMIN))
return -EPERM;
- if (!alloc_cpumask_var(&offline_mask, GFP_KERNEL))
- return -ENOMEM;
-
stream_id = simple_strtoul(buf, NULL, 16);
do {
@@ -149,33 +145,14 @@
ssleep(1);
} while (rc == -EAGAIN);
- if (!rc) {
- /* All present CPUs must be online */
- cpumask_andnot(offline_mask, cpu_present_mask,
- cpu_online_mask);
- rc = rtas_online_cpus_mask(offline_mask);
- if (rc) {
- pr_err("%s: Could not bring present CPUs online.\n",
- __func__);
- goto out;
- }
-
- stop_topology_update();
+ if (!rc)
rc = pm_suspend(PM_SUSPEND_MEM);
- start_topology_update();
-
- /* Take down CPUs not online prior to suspend */
- if (!rtas_offline_cpus_mask(offline_mask))
- pr_warn("%s: Could not restore CPUs to offline "
- "state.\n", __func__);
- }
stream_id = 0;
if (!rc)
rc = count;
-out:
- free_cpumask_var(offline_mask);
+
return rc;
}
diff --git a/arch/powerpc/purgatory/Makefile b/arch/powerpc/purgatory/Makefile
index 7c6d8b1..348f5958 100644
--- a/arch/powerpc/purgatory/Makefile
+++ b/arch/powerpc/purgatory/Makefile
@@ -2,11 +2,11 @@
KASAN_SANITIZE := n
-targets += trampoline.o purgatory.ro kexec-purgatory.c
+targets += trampoline_$(BITS).o purgatory.ro kexec-purgatory.c
LDFLAGS_purgatory.ro := -e purgatory_start -r --no-undefined
-$(obj)/purgatory.ro: $(obj)/trampoline.o FORCE
+$(obj)/purgatory.ro: $(obj)/trampoline_$(BITS).o FORCE
$(call if_changed,ld)
quiet_cmd_bin2c = BIN2C $@
diff --git a/arch/powerpc/purgatory/trampoline.S b/arch/powerpc/purgatory/trampoline_64.S
similarity index 70%
rename from arch/powerpc/purgatory/trampoline.S
rename to arch/powerpc/purgatory/trampoline_64.S
index a5a83c3..d956b8a 100644
--- a/arch/powerpc/purgatory/trampoline.S
+++ b/arch/powerpc/purgatory/trampoline_64.S
@@ -10,6 +10,7 @@
*/
#include <asm/asm-compat.h>
+#include <asm/crashdump-ppc64.h>
.machine ppc64
.balign 256
@@ -43,14 +44,39 @@
mr %r17,%r3 /* save cpu id to r17 */
mr %r15,%r4 /* save physical address in reg15 */
+ /* Work out where we're running */
+ bcl 20, 31, 0f
+0: mflr %r18
+
+ /*
+ * Copy BACKUP_SRC_SIZE bytes from BACKUP_SRC_START to
+ * backup_start 8 bytes at a time.
+ *
+ * Use r3 = dest, r4 = src, r5 = size, r6 = count
+ */
+ ld %r3, (backup_start - 0b)(%r18)
+ cmpdi %cr0, %r3, 0
+ beq .Lskip_copy /* skip if there is no backup region */
+ lis %r5, BACKUP_SRC_SIZE@h
+ ori %r5, %r5, BACKUP_SRC_SIZE@l
+ cmpdi %cr0, %r5, 0
+ beq .Lskip_copy /* skip if copy size is zero */
+ lis %r4, BACKUP_SRC_START@h
+ ori %r4, %r4, BACKUP_SRC_START@l
+ li %r6, 0
+.Lcopy_loop:
+ ldx %r0, %r6, %r4
+ stdx %r0, %r6, %r3
+ addi %r6, %r6, 8
+ cmpld %cr0, %r6, %r5
+ blt .Lcopy_loop
+
+.Lskip_copy:
or %r3,%r3,%r3 /* ok now to high priority, lets boot */
lis %r6,0x1
mtctr %r6 /* delay a bit for slaves to catch up */
bdnz . /* before we overwrite 0-100 again */
- bl 0f /* Work out where we're running */
-0: mflr %r18
-
/* load device-tree address */
ld %r3, (dt_offset - 0b)(%r18)
mr %r16,%r3 /* save dt address in reg16 */
@@ -61,6 +87,10 @@
li %r4,28
STWX_BE %r17,%r3,%r4 /* Store my cpu as __be32 at byte 28 */
1:
+ /* Load opal base and entry values in r8 & r9 respectively */
+ ld %r8,(opal_base - 0b)(%r18)
+ ld %r9,(opal_entry - 0b)(%r18)
+
/* load the kernel address */
ld %r4,(kernel - 0b)(%r18)
@@ -89,7 +119,6 @@
rfid /* update MSR and start kernel */
-
.balign 8
.globl kernel
kernel:
@@ -102,6 +131,23 @@
.8byte 0x0
.size dt_offset, . - dt_offset
+ .balign 8
+ .globl backup_start
+backup_start:
+ .8byte 0x0
+ .size backup_start, . - backup_start
+
+ .balign 8
+ .globl opal_base
+opal_base:
+ .8byte 0x0
+ .size opal_base, . - opal_base
+
+ .balign 8
+ .globl opal_entry
+opal_entry:
+ .8byte 0x0
+ .size opal_entry, . - opal_entry
.data
.balign 8
diff --git a/arch/powerpc/sysdev/xive/native.c b/arch/powerpc/sysdev/xive/native.c
index 71b881e..cb58ec7c 100644
--- a/arch/powerpc/sysdev/xive/native.c
+++ b/arch/powerpc/sysdev/xive/native.c
@@ -18,6 +18,7 @@
#include <linux/delay.h>
#include <linux/cpumask.h>
#include <linux/mm.h>
+#include <linux/kmemleak.h>
#include <asm/machdep.h>
#include <asm/prom.h>
@@ -647,6 +648,7 @@
pr_err("Failed to allocate provisioning page\n");
return false;
}
+ kmemleak_ignore(p);
opal_xive_donate_page(chip, __pa(p));
}
return true;
diff --git a/arch/powerpc/sysdev/xive/spapr.c b/arch/powerpc/sysdev/xive/spapr.c
index f0551a2..1e3674d 100644
--- a/arch/powerpc/sysdev/xive/spapr.c
+++ b/arch/powerpc/sysdev/xive/spapr.c
@@ -768,7 +768,7 @@
return vec5 + index;
}
-static bool xive_spapr_disabled(void)
+static bool __init xive_spapr_disabled(void)
{
const u8 *vec5_xive;
diff --git a/arch/powerpc/tools/unrel_branch_check.sh b/arch/powerpc/tools/unrel_branch_check.sh
index 7711475..6e6a30a 100755
--- a/arch/powerpc/tools/unrel_branch_check.sh
+++ b/arch/powerpc/tools/unrel_branch_check.sh
@@ -31,7 +31,10 @@
grep -v '\<__start_initialization_multiplatform>' |
grep -v -e 'b.\?.\?ctr' |
grep -v -e 'b.\?.\?lr' |
-sed 's/://' |
+sed -e 's/\bbt.\?[[:space:]]*[[:digit:]][[:digit:]]*,/beq/' \
+ -e 's/\bbf.\?[[:space:]]*[[:digit:]][[:digit:]]*,/bne/' \
+ -e 's/[[:space:]]0x/ /' \
+ -e 's/://' |
awk '{ print $1 ":" $6 ":0x" $7 ":" $8 " "}'
)
diff --git a/arch/powerpc/xmon/xmon.c b/arch/powerpc/xmon/xmon.c
index 7efe4bc..df7bca0 100644
--- a/arch/powerpc/xmon/xmon.c
+++ b/arch/powerpc/xmon/xmon.c
@@ -481,6 +481,13 @@
#endif
}
+static void xmon_touch_watchdogs(void)
+{
+ touch_softlockup_watchdog_sync();
+ rcu_cpu_stall_reset();
+ touch_nmi_watchdog();
+}
+
static int xmon_core(struct pt_regs *regs, int fromipi)
{
int cmd = 0;
@@ -718,7 +725,7 @@
else
insert_cpu_bpts();
- touch_nmi_watchdog();
+ xmon_touch_watchdogs();
local_irq_restore(flags);
return cmd != 'X' && cmd != EOF;
@@ -1593,6 +1600,7 @@
case 0x1300: ret = "(Instruction Breakpoint)"; break;
case 0x1500: ret = "(Denormalisation)"; break;
case 0x1700: ret = "(Altivec Assist)"; break;
+ case 0x3000: ret = "(System Call Vectored)"; break;
default: ret = "";
}
return ret;
@@ -1861,7 +1869,7 @@
catch_memory_errors = 1;
sync();
- if (cmd != 'i') {
+ if (cmd != 'i' || IS_ENABLED(CONFIG_PPC_BOOK3S_64)) {
for (; nflush > 0; --nflush, adrs += L1_CACHE_BYTES)
cflush((void *) adrs);
} else {
@@ -2022,6 +2030,18 @@
#endif
}
+static void dump_310_sprs(void)
+{
+#ifdef CONFIG_PPC64
+ if (!cpu_has_feature(CPU_FTR_ARCH_31))
+ return;
+
+ printf("mmcr3 = %.16lx, sier2 = %.16lx, sier3 = %.16lx\n",
+ mfspr(SPRN_MMCR3), mfspr(SPRN_SIER2), mfspr(SPRN_SIER3));
+
+#endif
+}
+
static void dump_one_spr(int spr, bool show_unimplemented)
{
unsigned long val;
@@ -2076,6 +2096,7 @@
dump_206_sprs();
dump_207_sprs();
dump_300_sprs();
+ dump_310_sprs();
return;
}
@@ -2934,11 +2955,10 @@
int nr, dotted;
unsigned long first_adr;
struct ppc_inst inst, last_inst = ppc_inst(0);
- unsigned char val[4];
dotted = 0;
- for (first_adr = adr; count > 0; --count, adr += 4) {
- nr = mread(adr, val, 4);
+ for (first_adr = adr; count > 0; --count, adr += ppc_inst_len(inst)) {
+ nr = mread_instr(adr, &inst);
if (nr == 0) {
if (praddr) {
const char *x = fault_chars[fault_type];
@@ -2946,7 +2966,6 @@
}
break;
}
- inst = ppc_inst(GETWORD(val));
if (adr > first_adr && ppc_inst_equal(inst, last_inst)) {
if (!dotted) {
printf(" ...\n");
@@ -2957,9 +2976,12 @@
dotted = 0;
last_inst = inst;
if (praddr)
- printf(REG" %.8x", adr, ppc_inst_val(inst));
+ printf(REG" %s", adr, ppc_inst_as_str(inst));
printf("\t");
- dump_func(ppc_inst_val(inst), adr);
+ if (!ppc_inst_prefixed(inst))
+ dump_func(ppc_inst_val(inst), adr);
+ else
+ dump_func(ppc_inst_as_u64(inst), adr);
printf("\n");
}
return adr - first_adr;
@@ -4256,7 +4278,7 @@
subcmd = inchar();
if (isxdigit(subcmd) || subcmd == '\n')
termch = subcmd;
- /* fall through */
+ fallthrough;
case 'f':
scanhex(&num);
if (num >= XMON_NUM_SPUS || !spu_info[num].spu) {
diff --git a/arch/riscv/Kconfig b/arch/riscv/Kconfig
index 6c4bce7..7b59055 100644
--- a/arch/riscv/Kconfig
+++ b/arch/riscv/Kconfig
@@ -13,11 +13,14 @@
config RISCV
def_bool y
select ARCH_CLOCKSOURCE_INIT
+ select ARCH_SUPPORTS_ATOMIC_RMW
select ARCH_HAS_BINFMT_FLAT
+ select ARCH_HAS_DEBUG_VM_PGTABLE
select ARCH_HAS_DEBUG_VIRTUAL if MMU
select ARCH_HAS_DEBUG_WX
select ARCH_HAS_GCOV_PROFILE_ALL
select ARCH_HAS_GIGANTIC_PAGE
+ select ARCH_HAS_KCOV
select ARCH_HAS_MMIOWB
select ARCH_HAS_PTE_SPECIAL
select ARCH_HAS_SET_DIRECT_MAP
@@ -47,6 +50,8 @@
select GENERIC_TIME_VSYSCALL if MMU && 64BIT
select HANDLE_DOMAIN_IRQ
select HAVE_ARCH_AUDITSYSCALL
+ select HAVE_ARCH_JUMP_LABEL
+ select HAVE_ARCH_JUMP_LABEL_RELATIVE
select HAVE_ARCH_KASAN if MMU && 64BIT
select HAVE_ARCH_KGDB
select HAVE_ARCH_KGDB_QXFER_PKT
@@ -54,14 +59,18 @@
select HAVE_ARCH_SECCOMP_FILTER
select HAVE_ARCH_TRACEHOOK
select HAVE_ASM_MODVERSIONS
+ select HAVE_CONTEXT_TRACKING
+ select HAVE_DEBUG_KMEMLEAK
select HAVE_DMA_CONTIGUOUS if MMU
select HAVE_EBPF_JIT if MMU
select HAVE_FUTEX_CMPXCHG if FUTEX
+ select HAVE_GCC_PLUGINS
select HAVE_GENERIC_VDSO if MMU && 64BIT
select HAVE_PCI
select HAVE_PERF_EVENTS
select HAVE_PERF_REGS
select HAVE_PERF_USER_STACK_DUMP
+ select HAVE_STACKPROTECTOR
select HAVE_SYSCALL_TRACEPOINTS
select IRQ_DOMAIN
select MODULES_USE_ELF_RELA if MODULES
@@ -179,6 +188,9 @@
default 3 if 64BIT
default 2
+config LOCKDEP_SUPPORT
+ def_bool y
+
source "arch/riscv/Kconfig.socs"
menu "Platform type"
diff --git a/arch/riscv/boot/Makefile b/arch/riscv/boot/Makefile
index 3530c59..c59fca6 100644
--- a/arch/riscv/boot/Makefile
+++ b/arch/riscv/boot/Makefile
@@ -14,6 +14,8 @@
# Based on the ia64 and arm64 boot/Makefile.
#
+KCOV_INSTRUMENT := n
+
OBJCOPYFLAGS_Image :=-O binary -R .note -R .note.gnu.build-id -R .comment -S
targets := Image loader
diff --git a/arch/riscv/configs/defconfig b/arch/riscv/configs/defconfig
index 4da4886..d58c93e 100644
--- a/arch/riscv/configs/defconfig
+++ b/arch/riscv/configs/defconfig
@@ -17,6 +17,7 @@
CONFIG_SOC_SIFIVE=y
CONFIG_SOC_VIRT=y
CONFIG_SMP=y
+CONFIG_JUMP_LABEL=y
CONFIG_MODULES=y
CONFIG_MODULE_UNLOAD=y
CONFIG_NET=y
diff --git a/arch/riscv/configs/nommu_k210_defconfig b/arch/riscv/configs/nommu_k210_defconfig
index b48138e..cd1df62 100644
--- a/arch/riscv/configs/nommu_k210_defconfig
+++ b/arch/riscv/configs/nommu_k210_defconfig
@@ -33,6 +33,7 @@
CONFIG_NR_CPUS=2
CONFIG_CMDLINE="earlycon console=ttySIF0"
CONFIG_CMDLINE_FORCE=y
+CONFIG_JUMP_LABEL=y
# CONFIG_BLOCK is not set
CONFIG_BINFMT_FLAT=y
# CONFIG_COREDUMP is not set
diff --git a/arch/riscv/configs/nommu_virt_defconfig b/arch/riscv/configs/nommu_virt_defconfig
index cf74e17..f27596e 100644
--- a/arch/riscv/configs/nommu_virt_defconfig
+++ b/arch/riscv/configs/nommu_virt_defconfig
@@ -30,6 +30,7 @@
CONFIG_SMP=y
CONFIG_CMDLINE="root=/dev/vda rw earlycon=uart8250,mmio,0x10000000,115200n8 console=ttyS0"
CONFIG_CMDLINE_FORCE=y
+CONFIG_JUMP_LABEL=y
# CONFIG_BLK_DEV_BSG is not set
CONFIG_PARTITION_ADVANCED=y
# CONFIG_MSDOS_PARTITION is not set
diff --git a/arch/riscv/configs/rv32_defconfig b/arch/riscv/configs/rv32_defconfig
index 05bbf52..3a55f0e 100644
--- a/arch/riscv/configs/rv32_defconfig
+++ b/arch/riscv/configs/rv32_defconfig
@@ -17,6 +17,7 @@
CONFIG_SOC_VIRT=y
CONFIG_ARCH_RV32I=y
CONFIG_SMP=y
+CONFIG_JUMP_LABEL=y
CONFIG_MODULES=y
CONFIG_MODULE_UNLOAD=y
CONFIG_NET=y
diff --git a/arch/riscv/include/asm/irq_work.h b/arch/riscv/include/asm/irq_work.h
new file mode 100644
index 0000000..d6c2779
--- /dev/null
+++ b/arch/riscv/include/asm/irq_work.h
@@ -0,0 +1,10 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#ifndef _ASM_RISCV_IRQ_WORK_H
+#define _ASM_RISCV_IRQ_WORK_H
+
+static inline bool arch_irq_work_has_interrupt(void)
+{
+ return true;
+}
+extern void arch_irq_work_raise(void);
+#endif /* _ASM_RISCV_IRQ_WORK_H */
diff --git a/arch/riscv/include/asm/jump_label.h b/arch/riscv/include/asm/jump_label.h
new file mode 100644
index 0000000..38af2ec
--- /dev/null
+++ b/arch/riscv/include/asm/jump_label.h
@@ -0,0 +1,60 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*
+ * Copyright (C) 2020 Emil Renner Berthing
+ *
+ * Based on arch/arm64/include/asm/jump_label.h
+ */
+#ifndef __ASM_JUMP_LABEL_H
+#define __ASM_JUMP_LABEL_H
+
+#ifndef __ASSEMBLY__
+
+#include <linux/types.h>
+#include <asm/asm.h>
+
+#define JUMP_LABEL_NOP_SIZE 4
+
+static __always_inline bool arch_static_branch(struct static_key *key,
+ bool branch)
+{
+ asm_volatile_goto(
+ " .option push \n\t"
+ " .option norelax \n\t"
+ " .option norvc \n\t"
+ "1: nop \n\t"
+ " .option pop \n\t"
+ " .pushsection __jump_table, \"aw\" \n\t"
+ " .align " RISCV_LGPTR " \n\t"
+ " .long 1b - ., %l[label] - . \n\t"
+ " " RISCV_PTR " %0 - . \n\t"
+ " .popsection \n\t"
+ : : "i"(&((char *)key)[branch]) : : label);
+
+ return false;
+label:
+ return true;
+}
+
+static __always_inline bool arch_static_branch_jump(struct static_key *key,
+ bool branch)
+{
+ asm_volatile_goto(
+ " .option push \n\t"
+ " .option norelax \n\t"
+ " .option norvc \n\t"
+ "1: jal zero, %l[label] \n\t"
+ " .option pop \n\t"
+ " .pushsection __jump_table, \"aw\" \n\t"
+ " .align " RISCV_LGPTR " \n\t"
+ " .long 1b - ., %l[label] - . \n\t"
+ " " RISCV_PTR " %0 - . \n\t"
+ " .popsection \n\t"
+ : : "i"(&((char *)key)[branch]) : : label);
+
+ return false;
+label:
+ return true;
+}
+
+#endif /* __ASSEMBLY__ */
+#endif /* __ASM_JUMP_LABEL_H */
diff --git a/arch/riscv/include/asm/mmio.h b/arch/riscv/include/asm/mmio.h
index 56053c9..aff6c33 100644
--- a/arch/riscv/include/asm/mmio.h
+++ b/arch/riscv/include/asm/mmio.h
@@ -14,12 +14,6 @@
#include <linux/types.h>
#include <asm/mmiowb.h>
-#ifndef CONFIG_MMU
-#define pgprot_noncached(x) (x)
-#define pgprot_writecombine(x) (x)
-#define pgprot_device(x) (x)
-#endif /* CONFIG_MMU */
-
/* Generic IO read/write. These perform native-endian accesses. */
#define __raw_writeb __raw_writeb
static inline void __raw_writeb(u8 val, volatile void __iomem *addr)
diff --git a/arch/riscv/include/asm/smp.h b/arch/riscv/include/asm/smp.h
index 40bb1c1..6dfd2a1 100644
--- a/arch/riscv/include/asm/smp.h
+++ b/arch/riscv/include/asm/smp.h
@@ -40,6 +40,9 @@
int riscv_hartid_to_cpuid(int hartid);
void riscv_cpuid_to_hartid_mask(const struct cpumask *in, struct cpumask *out);
+/* Secondary hart entry */
+asmlinkage void smp_callin(void);
+
/*
* Obtains the hart ID of the currently executing task. This relies on
* THREAD_INFO_IN_TASK, but we define that unconditionally.
diff --git a/arch/riscv/include/asm/stackprotector.h b/arch/riscv/include/asm/stackprotector.h
new file mode 100644
index 0000000..d95f7b2
--- /dev/null
+++ b/arch/riscv/include/asm/stackprotector.h
@@ -0,0 +1,33 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+
+#ifndef _ASM_RISCV_STACKPROTECTOR_H
+#define _ASM_RISCV_STACKPROTECTOR_H
+
+#include <linux/random.h>
+#include <linux/version.h>
+#include <asm/timex.h>
+
+extern unsigned long __stack_chk_guard;
+
+/*
+ * Initialize the stackprotector canary value.
+ *
+ * NOTE: this must only be called from functions that never return,
+ * and it must always be inlined.
+ */
+static __always_inline void boot_init_stack_canary(void)
+{
+ unsigned long canary;
+ unsigned long tsc;
+
+ /* Try to get a semi random initial value. */
+ get_random_bytes(&canary, sizeof(canary));
+ tsc = get_cycles();
+ canary += tsc + (tsc << BITS_PER_LONG/2);
+ canary ^= LINUX_VERSION_CODE;
+ canary &= CANARY_MASK;
+
+ current->stack_canary = canary;
+ __stack_chk_guard = current->stack_canary;
+}
+#endif /* _ASM_RISCV_STACKPROTECTOR_H */
diff --git a/arch/riscv/include/uapi/asm/hwcap.h b/arch/riscv/include/uapi/asm/hwcap.h
index dee98ee..46dc3f5 100644
--- a/arch/riscv/include/uapi/asm/hwcap.h
+++ b/arch/riscv/include/uapi/asm/hwcap.h
@@ -11,7 +11,7 @@
/*
* Linux saves the floating-point registers according to the ISA Linux is
* executing on, as opposed to the ISA the user program is compiled for. This
- * is necessary for a handful of esoteric use cases: for example, userpsace
+ * is necessary for a handful of esoteric use cases: for example, userspace
* threading libraries must be able to examine the actual machine state in
* order to fully reconstruct the state of a thread.
*/
diff --git a/arch/riscv/include/uapi/asm/unistd.h b/arch/riscv/include/uapi/asm/unistd.h
index 13ce76c..4b989ae 100644
--- a/arch/riscv/include/uapi/asm/unistd.h
+++ b/arch/riscv/include/uapi/asm/unistd.h
@@ -12,7 +12,7 @@
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
#ifdef __LP64__
diff --git a/arch/riscv/kernel/Makefile b/arch/riscv/kernel/Makefile
index b355cf4..a5287ab 100644
--- a/arch/riscv/kernel/Makefile
+++ b/arch/riscv/kernel/Makefile
@@ -53,4 +53,6 @@
obj-$(CONFIG_HOTPLUG_CPU) += cpu-hotplug.o
obj-$(CONFIG_KGDB) += kgdb.o
+obj-$(CONFIG_JUMP_LABEL) += jump_label.o
+
clean:
diff --git a/arch/riscv/kernel/asm-offsets.c b/arch/riscv/kernel/asm-offsets.c
index 07cb9c1..db20344 100644
--- a/arch/riscv/kernel/asm-offsets.c
+++ b/arch/riscv/kernel/asm-offsets.c
@@ -27,9 +27,6 @@
OFFSET(TASK_THREAD_S9, task_struct, thread.s[9]);
OFFSET(TASK_THREAD_S10, task_struct, thread.s[10]);
OFFSET(TASK_THREAD_S11, task_struct, thread.s[11]);
- OFFSET(TASK_THREAD_SP, task_struct, thread.sp);
- OFFSET(TASK_STACK, task_struct, stack);
- OFFSET(TASK_TI, task_struct, thread_info);
OFFSET(TASK_TI_FLAGS, task_struct, thread_info.flags);
OFFSET(TASK_TI_PREEMPT_COUNT, task_struct, thread_info.preempt_count);
OFFSET(TASK_TI_KERNEL_SP, task_struct, thread_info.kernel_sp);
diff --git a/arch/riscv/kernel/entry.S b/arch/riscv/kernel/entry.S
index cae7e6d..524d918 100644
--- a/arch/riscv/kernel/entry.S
+++ b/arch/riscv/kernel/entry.S
@@ -97,19 +97,36 @@
la gp, __global_pointer$
.option pop
- la ra, ret_from_exception
+#ifdef CONFIG_TRACE_IRQFLAGS
+ call trace_hardirqs_off
+#endif
+
+#ifdef CONFIG_CONTEXT_TRACKING
+ /* If previous state is in user mode, call context_tracking_user_exit. */
+ li a0, SR_PP
+ and a0, s1, a0
+ bnez a0, skip_context_tracking
+ call context_tracking_user_exit
+skip_context_tracking:
+#endif
+
/*
* MSB of cause differentiates between
* interrupts and exceptions
*/
bge s4, zero, 1f
+ la ra, ret_from_exception
+
/* Handle interrupts */
move a0, sp /* pt_regs */
la a1, handle_arch_irq
REG_L a1, (a1)
jr a1
1:
+#ifdef CONFIG_TRACE_IRQFLAGS
+ call trace_hardirqs_on
+#endif
/*
* Exceptions run with interrupts enabled or disabled depending on the
* state of SR_PIE in m/sstatus.
@@ -119,6 +136,7 @@
csrs CSR_STATUS, SR_IE
1:
+ la ra, ret_from_exception
/* Handle syscalls */
li t0, EXC_SYSCALL
beq s4, t0, handle_syscall
@@ -137,6 +155,17 @@
tail do_trap_unknown
handle_syscall:
+#if defined(CONFIG_TRACE_IRQFLAGS) || defined(CONFIG_CONTEXT_TRACKING)
+ /* Recover a0 - a7 for system calls */
+ REG_L a0, PT_A0(sp)
+ REG_L a1, PT_A1(sp)
+ REG_L a2, PT_A2(sp)
+ REG_L a3, PT_A3(sp)
+ REG_L a4, PT_A4(sp)
+ REG_L a5, PT_A5(sp)
+ REG_L a6, PT_A6(sp)
+ REG_L a7, PT_A7(sp)
+#endif
/* save the initial A0 value (needed in signal handlers) */
REG_S a0, PT_ORIG_A0(sp)
/*
@@ -190,6 +219,9 @@
ret_from_exception:
REG_L s0, PT_STATUS(sp)
csrc CSR_STATUS, SR_IE
+#ifdef CONFIG_TRACE_IRQFLAGS
+ call trace_hardirqs_off
+#endif
#ifdef CONFIG_RISCV_M_MODE
/* the MPP value is too large to be used as an immediate arg for addi */
li t0, SR_MPP
@@ -205,6 +237,10 @@
andi s1, s0, _TIF_WORK_MASK
bnez s1, work_pending
+#ifdef CONFIG_CONTEXT_TRACKING
+ call context_tracking_user_enter
+#endif
+
/* Save unwound kernel stack pointer in thread_info */
addi s0, sp, PT_SIZE_ON_STACK
REG_S s0, TASK_TI_KERNEL_SP(tp)
@@ -216,6 +252,16 @@
csrw CSR_SCRATCH, tp
restore_all:
+#ifdef CONFIG_TRACE_IRQFLAGS
+ REG_L s1, PT_STATUS(sp)
+ andi t0, s1, SR_PIE
+ beqz t0, 1f
+ call trace_hardirqs_on
+ j 2f
+1:
+ call trace_hardirqs_off
+2:
+#endif
REG_L a0, PT_STATUS(sp)
/*
* The current load reservation is effectively part of the processor's
@@ -389,12 +435,8 @@
lw a4, TASK_TI_CPU(a1)
sw a3, TASK_TI_CPU(a1)
sw a4, TASK_TI_CPU(a0)
-#if TASK_TI != 0
-#error "TASK_TI != 0: tp will contain a 'struct thread_info', not a 'struct task_struct' so get_current() won't work."
- addi tp, a1, TASK_TI
-#else
+ /* The offset of thread_info in task_struct is zero. */
move tp, a1
-#endif
ret
ENDPROC(__switch_to)
diff --git a/arch/riscv/kernel/head.S b/arch/riscv/kernel/head.S
index 7ed1b22..d0c5c31 100644
--- a/arch/riscv/kernel/head.S
+++ b/arch/riscv/kernel/head.S
@@ -77,10 +77,16 @@
csrw CSR_SATP, a0
.align 2
1:
- /* Set trap vector to spin forever to help debug */
- la a0, .Lsecondary_park
+ /* Set trap vector to exception handler */
+ la a0, handle_exception
csrw CSR_TVEC, a0
+ /*
+ * Set sup0 scratch register to 0, indicating to exception vector that
+ * we are presently executing in kernel.
+ */
+ csrw CSR_SCRATCH, zero
+
/* Reload the global pointer */
.option push
.option norelax
diff --git a/arch/riscv/kernel/jump_label.c b/arch/riscv/kernel/jump_label.c
new file mode 100644
index 0000000..20e0905
--- /dev/null
+++ b/arch/riscv/kernel/jump_label.c
@@ -0,0 +1,53 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (C) 2020 Emil Renner Berthing
+ *
+ * Based on arch/arm64/kernel/jump_label.c
+ */
+#include <linux/jump_label.h>
+#include <linux/kernel.h>
+#include <linux/memory.h>
+#include <linux/mutex.h>
+#include <asm/bug.h>
+#include <asm/patch.h>
+
+#define RISCV_INSN_NOP 0x00000013U
+#define RISCV_INSN_JAL 0x0000006fU
+
+void arch_jump_label_transform(struct jump_entry *entry,
+ enum jump_label_type type)
+{
+ void *addr = (void *)jump_entry_code(entry);
+ u32 insn;
+
+ if (type == JUMP_LABEL_JMP) {
+ long offset = jump_entry_target(entry) - jump_entry_code(entry);
+
+ if (WARN_ON(offset & 1 || offset < -524288 || offset >= 524288))
+ return;
+
+ insn = RISCV_INSN_JAL |
+ (((u32)offset & GENMASK(19, 12)) << (12 - 12)) |
+ (((u32)offset & GENMASK(11, 11)) << (20 - 11)) |
+ (((u32)offset & GENMASK(10, 1)) << (21 - 1)) |
+ (((u32)offset & GENMASK(20, 20)) << (31 - 20));
+ } else {
+ insn = RISCV_INSN_NOP;
+ }
+
+ mutex_lock(&text_mutex);
+ patch_text_nosync(addr, &insn, sizeof(insn));
+ mutex_unlock(&text_mutex);
+}
+
+void arch_jump_label_transform_static(struct jump_entry *entry,
+ enum jump_label_type type)
+{
+ /*
+ * We use the same instructions in the arch_static_branch and
+ * arch_static_branch_jump inline functions, so there's no
+ * need to patch them up here.
+ * The core will call arch_jump_label_transform when those
+ * instructions need to be replaced.
+ */
+}
diff --git a/arch/riscv/kernel/module.c b/arch/riscv/kernel/module.c
index 7191342..104fba8 100644
--- a/arch/riscv/kernel/module.c
+++ b/arch/riscv/kernel/module.c
@@ -263,6 +263,13 @@
return 0;
}
+static int apply_r_riscv_add64_rela(struct module *me, u32 *location,
+ Elf_Addr v)
+{
+ *(u64 *)location += (u64)v;
+ return 0;
+}
+
static int apply_r_riscv_sub32_rela(struct module *me, u32 *location,
Elf_Addr v)
{
@@ -270,6 +277,13 @@
return 0;
}
+static int apply_r_riscv_sub64_rela(struct module *me, u32 *location,
+ Elf_Addr v)
+{
+ *(u64 *)location -= (u64)v;
+ return 0;
+}
+
static int (*reloc_handlers_rela[]) (struct module *me, u32 *location,
Elf_Addr v) = {
[R_RISCV_32] = apply_r_riscv_32_rela,
@@ -290,7 +304,9 @@
[R_RISCV_RELAX] = apply_r_riscv_relax_rela,
[R_RISCV_ALIGN] = apply_r_riscv_align_rela,
[R_RISCV_ADD32] = apply_r_riscv_add32_rela,
+ [R_RISCV_ADD64] = apply_r_riscv_add64_rela,
[R_RISCV_SUB32] = apply_r_riscv_sub32_rela,
+ [R_RISCV_SUB64] = apply_r_riscv_sub64_rela,
};
int apply_relocate_add(Elf_Shdr *sechdrs, const char *strtab,
diff --git a/arch/riscv/kernel/process.c b/arch/riscv/kernel/process.c
index 31f3944..2b97c49 100644
--- a/arch/riscv/kernel/process.c
+++ b/arch/riscv/kernel/process.c
@@ -24,6 +24,12 @@
register unsigned long gp_in_global __asm__("gp");
+#ifdef CONFIG_STACKPROTECTOR
+#include <linux/stackprotector.h>
+unsigned long __stack_chk_guard __read_mostly;
+EXPORT_SYMBOL(__stack_chk_guard);
+#endif
+
extern asmlinkage void ret_from_fork(void);
extern asmlinkage void ret_from_kernel_thread(void);
diff --git a/arch/riscv/kernel/ptrace.c b/arch/riscv/kernel/ptrace.c
index 444dc7b..2d6395f 100644
--- a/arch/riscv/kernel/ptrace.c
+++ b/arch/riscv/kernel/ptrace.c
@@ -30,13 +30,10 @@
static int riscv_gpr_get(struct task_struct *target,
const struct user_regset *regset,
- unsigned int pos, unsigned int count,
- void *kbuf, void __user *ubuf)
+ struct membuf to)
{
- struct pt_regs *regs;
-
- regs = task_pt_regs(target);
- return user_regset_copyout(&pos, &count, &kbuf, &ubuf, regs, 0, -1);
+ return membuf_write(&to, task_pt_regs(target),
+ sizeof(struct user_regs_struct));
}
static int riscv_gpr_set(struct task_struct *target,
@@ -55,21 +52,13 @@
#ifdef CONFIG_FPU
static int riscv_fpr_get(struct task_struct *target,
const struct user_regset *regset,
- unsigned int pos, unsigned int count,
- void *kbuf, void __user *ubuf)
+ struct membuf to)
{
- int ret;
struct __riscv_d_ext_state *fstate = &target->thread.fstate;
- ret = user_regset_copyout(&pos, &count, &kbuf, &ubuf, fstate, 0,
- offsetof(struct __riscv_d_ext_state, fcsr));
- if (!ret) {
- ret = user_regset_copyout(&pos, &count, &kbuf, &ubuf, fstate, 0,
- offsetof(struct __riscv_d_ext_state, fcsr) +
- sizeof(fstate->fcsr));
- }
-
- return ret;
+ membuf_write(&to, fstate, offsetof(struct __riscv_d_ext_state, fcsr));
+ membuf_store(&to, fstate->fcsr);
+ return membuf_zero(&to, 4); // explicitly pad
}
static int riscv_fpr_set(struct task_struct *target,
@@ -98,8 +87,8 @@
.n = ELF_NGREG,
.size = sizeof(elf_greg_t),
.align = sizeof(elf_greg_t),
- .get = &riscv_gpr_get,
- .set = &riscv_gpr_set,
+ .regset_get = riscv_gpr_get,
+ .set = riscv_gpr_set,
},
#ifdef CONFIG_FPU
[REGSET_F] = {
@@ -107,8 +96,8 @@
.n = ELF_NFPREG,
.size = sizeof(elf_fpreg_t),
.align = sizeof(elf_fpreg_t),
- .get = &riscv_fpr_get,
- .set = &riscv_fpr_set,
+ .regset_get = riscv_fpr_get,
+ .set = riscv_fpr_set,
},
#endif
};
diff --git a/arch/riscv/kernel/smp.c b/arch/riscv/kernel/smp.c
index b1d4f45..554b0fb 100644
--- a/arch/riscv/kernel/smp.c
+++ b/arch/riscv/kernel/smp.c
@@ -16,6 +16,7 @@
#include <linux/sched.h>
#include <linux/seq_file.h>
#include <linux/delay.h>
+#include <linux/irq_work.h>
#include <asm/clint.h>
#include <asm/sbi.h>
@@ -26,6 +27,7 @@
IPI_RESCHEDULE,
IPI_CALL_FUNC,
IPI_CPU_STOP,
+ IPI_IRQ_WORK,
IPI_MAX
};
@@ -123,6 +125,13 @@
clint_clear_ipi(cpuid_to_hartid_map(smp_processor_id()));
}
+#ifdef CONFIG_IRQ_WORK
+void arch_irq_work_raise(void)
+{
+ send_ipi_single(smp_processor_id(), IPI_IRQ_WORK);
+}
+#endif
+
void handle_IPI(struct pt_regs *regs)
{
struct pt_regs *old_regs = set_irq_regs(regs);
@@ -158,6 +167,11 @@
ipi_stop();
}
+ if (ops & (1 << IPI_IRQ_WORK)) {
+ stats[IPI_IRQ_WORK]++;
+ irq_work_run();
+ }
+
BUG_ON((ops >> IPI_MAX) != 0);
/* Order data access and bit testing. */
@@ -173,6 +187,7 @@
[IPI_RESCHEDULE] = "Rescheduling interrupts",
[IPI_CALL_FUNC] = "Function call interrupts",
[IPI_CPU_STOP] = "CPU stop interrupts",
+ [IPI_IRQ_WORK] = "IRQ work interrupts",
};
void show_ipi_stats(struct seq_file *p, int prec)
diff --git a/arch/riscv/kernel/smpboot.c b/arch/riscv/kernel/smpboot.c
index 4e99227..356825a 100644
--- a/arch/riscv/kernel/smpboot.c
+++ b/arch/riscv/kernel/smpboot.c
@@ -106,7 +106,7 @@
}
}
-int start_secondary_cpu(int cpu, struct task_struct *tidle)
+static int start_secondary_cpu(int cpu, struct task_struct *tidle)
{
if (cpu_ops[cpu]->cpu_start)
return cpu_ops[cpu]->cpu_start(cpu, tidle);
@@ -121,7 +121,6 @@
ret = start_secondary_cpu(cpu, tidle);
if (!ret) {
- lockdep_assert_held(&cpu_running);
wait_for_completion_timeout(&cpu_running,
msecs_to_jiffies(1000));
@@ -146,6 +145,7 @@
asmlinkage __visible void smp_callin(void)
{
struct mm_struct *mm = &init_mm;
+ unsigned int curr_cpuid = smp_processor_id();
if (!IS_ENABLED(CONFIG_RISCV_SBI))
clint_clear_ipi(cpuid_to_hartid_map(smp_processor_id()));
@@ -154,10 +154,10 @@
mmgrab(mm);
current->active_mm = mm;
- trap_init();
- notify_cpu_starting(smp_processor_id());
- update_siblings_masks(smp_processor_id());
- set_cpu_online(smp_processor_id(), 1);
+ notify_cpu_starting(curr_cpuid);
+ update_siblings_masks(curr_cpuid);
+ set_cpu_online(curr_cpuid, 1);
+
/*
* Remote TLB flushes are ignored while the CPU is offline, so emit
* a local TLB flush right now just in case.
diff --git a/arch/riscv/kernel/traps.c b/arch/riscv/kernel/traps.c
index 7d95cce..ad14f44 100644
--- a/arch/riscv/kernel/traps.c
+++ b/arch/riscv/kernel/traps.c
@@ -174,13 +174,7 @@
}
#endif /* CONFIG_GENERIC_BUG */
+/* stvec & scratch is already set from head.S */
void trap_init(void)
{
- /*
- * Set sup0 scratch register to 0, indicating to exception vector
- * that we are presently executing in the kernel
- */
- csr_write(CSR_SCRATCH, 0);
- /* Set the exception vector address */
- csr_write(CSR_TVEC, &handle_exception);
}
diff --git a/arch/riscv/kernel/vdso/Makefile b/arch/riscv/kernel/vdso/Makefile
index e4c7c2c..478e733 100644
--- a/arch/riscv/kernel/vdso/Makefile
+++ b/arch/riscv/kernel/vdso/Makefile
@@ -16,6 +16,8 @@
# Files to link into the vdso
obj-vdso = $(patsubst %, %.o, $(vdso-syms)) note.o
+ccflags-y := -fno-stack-protector
+
ifneq ($(c-gettimeofday-y),)
CFLAGS_vgettimeofday.o += -fPIC -include $(c-gettimeofday-y)
endif
@@ -32,6 +34,7 @@
# Disable gcov profiling for VDSO code
GCOV_PROFILE := n
+KCOV_INSTRUMENT := n
# Force dependency
$(obj)/vdso.o: $(obj)/vdso.so
diff --git a/arch/riscv/kernel/vmlinux.lds.S b/arch/riscv/kernel/vmlinux.lds.S
index e6f8016..f3586e3 100644
--- a/arch/riscv/kernel/vmlinux.lds.S
+++ b/arch/riscv/kernel/vmlinux.lds.S
@@ -22,6 +22,7 @@
/* Beginning of code and text segment */
. = LOAD_OFFSET;
_start = .;
+ _stext = .;
HEAD_TEXT_SECTION
. = ALIGN(PAGE_SIZE);
@@ -54,7 +55,6 @@
. = ALIGN(SECTION_ALIGN);
.text : {
_text = .;
- _stext = .;
TEXT_TEXT
SCHED_TEXT
CPUIDLE_TEXT
diff --git a/arch/riscv/mm/Makefile b/arch/riscv/mm/Makefile
index 363ef01..c0185e5 100644
--- a/arch/riscv/mm/Makefile
+++ b/arch/riscv/mm/Makefile
@@ -5,6 +5,8 @@
CFLAGS_REMOVE_init.o = -pg
endif
+KCOV_INSTRUMENT_init.o := n
+
obj-y += init.o
obj-y += extable.o
obj-$(CONFIG_MMU) += fault.o pageattr.o
diff --git a/arch/riscv/mm/init.c b/arch/riscv/mm/init.c
index f6e6286..787c75f 100644
--- a/arch/riscv/mm/init.c
+++ b/arch/riscv/mm/init.c
@@ -541,12 +541,39 @@
}
#endif
+static void __init resource_init(void)
+{
+ struct memblock_region *region;
+
+ for_each_memblock(memory, region) {
+ struct resource *res;
+
+ res = memblock_alloc(sizeof(struct resource), SMP_CACHE_BYTES);
+ if (!res)
+ panic("%s: Failed to allocate %zu bytes\n", __func__,
+ sizeof(struct resource));
+
+ if (memblock_is_nomap(region)) {
+ res->name = "reserved";
+ res->flags = IORESOURCE_MEM;
+ } else {
+ res->name = "System RAM";
+ res->flags = IORESOURCE_SYSTEM_RAM | IORESOURCE_BUSY;
+ }
+ res->start = __pfn_to_phys(memblock_region_memory_base_pfn(region));
+ res->end = __pfn_to_phys(memblock_region_memory_end_pfn(region)) - 1;
+
+ request_resource(&iomem_resource, res);
+ }
+}
+
void __init paging_init(void)
{
setup_vm_final();
sparse_init();
setup_zero_page();
zone_sizes_init();
+ resource_init();
}
#ifdef CONFIG_SPARSEMEM_VMEMMAP
diff --git a/arch/riscv/mm/pageattr.c b/arch/riscv/mm/pageattr.c
index 289a9a5..19fecb3 100644
--- a/arch/riscv/mm/pageattr.c
+++ b/arch/riscv/mm/pageattr.c
@@ -7,6 +7,7 @@
#include <linux/pgtable.h>
#include <asm/tlbflush.h>
#include <asm/bitops.h>
+#include <asm/set_memory.h>
struct pageattr_masks {
pgprot_t set_mask;
@@ -94,7 +95,7 @@
return 0;
}
-const static struct mm_walk_ops pageattr_ops = {
+static const struct mm_walk_ops pageattr_ops = {
.pgd_entry = pageattr_pgd_entry,
.p4d_entry = pageattr_p4d_entry,
.pud_entry = pageattr_pud_entry,
diff --git a/arch/s390/kernel/ptrace.c b/arch/s390/kernel/ptrace.c
index 3c72a3b..11d2f7d 100644
--- a/arch/s390/kernel/ptrace.c
+++ b/arch/s390/kernel/ptrace.c
@@ -944,28 +944,14 @@
static int s390_regs_get(struct task_struct *target,
const struct user_regset *regset,
- unsigned int pos, unsigned int count,
- void *kbuf, void __user *ubuf)
+ struct membuf to)
{
+ unsigned pos;
if (target == current)
save_access_regs(target->thread.acrs);
- if (kbuf) {
- unsigned long *k = kbuf;
- while (count > 0) {
- *k++ = __peek_user(target, pos);
- count -= sizeof(*k);
- pos += sizeof(*k);
- }
- } else {
- unsigned long __user *u = ubuf;
- while (count > 0) {
- if (__put_user(__peek_user(target, pos), u++))
- return -EFAULT;
- count -= sizeof(*u);
- pos += sizeof(*u);
- }
- }
+ for (pos = 0; pos < sizeof(s390_regs); pos += sizeof(long))
+ membuf_store(&to, __peek_user(target, pos));
return 0;
}
@@ -1006,8 +992,8 @@
}
static int s390_fpregs_get(struct task_struct *target,
- const struct user_regset *regset, unsigned int pos,
- unsigned int count, void *kbuf, void __user *ubuf)
+ const struct user_regset *regset,
+ struct membuf to)
{
_s390_fp_regs fp_regs;
@@ -1017,8 +1003,7 @@
fp_regs.fpc = target->thread.fpu.fpc;
fpregs_store(&fp_regs, &target->thread.fpu);
- return user_regset_copyout(&pos, &count, &kbuf, &ubuf,
- &fp_regs, 0, -1);
+ return membuf_write(&to, &fp_regs, sizeof(fp_regs));
}
static int s390_fpregs_set(struct task_struct *target,
@@ -1065,20 +1050,9 @@
static int s390_last_break_get(struct task_struct *target,
const struct user_regset *regset,
- unsigned int pos, unsigned int count,
- void *kbuf, void __user *ubuf)
+ struct membuf to)
{
- if (count > 0) {
- if (kbuf) {
- unsigned long *k = kbuf;
- *k = target->thread.last_break;
- } else {
- unsigned long __user *u = ubuf;
- if (__put_user(target->thread.last_break, u))
- return -EFAULT;
- }
- }
- return 0;
+ return membuf_store(&to, target->thread.last_break);
}
static int s390_last_break_set(struct task_struct *target,
@@ -1091,16 +1065,13 @@
static int s390_tdb_get(struct task_struct *target,
const struct user_regset *regset,
- unsigned int pos, unsigned int count,
- void *kbuf, void __user *ubuf)
+ struct membuf to)
{
struct pt_regs *regs = task_pt_regs(target);
- unsigned char *data;
if (!(regs->int_code & 0x200))
return -ENODATA;
- data = target->thread.trap_tdb;
- return user_regset_copyout(&pos, &count, &kbuf, &ubuf, data, 0, 256);
+ return membuf_write(&to, target->thread.trap_tdb, 256);
}
static int s390_tdb_set(struct task_struct *target,
@@ -1113,8 +1084,7 @@
static int s390_vxrs_low_get(struct task_struct *target,
const struct user_regset *regset,
- unsigned int pos, unsigned int count,
- void *kbuf, void __user *ubuf)
+ struct membuf to)
{
__u64 vxrs[__NUM_VXRS_LOW];
int i;
@@ -1125,7 +1095,7 @@
save_fpu_regs();
for (i = 0; i < __NUM_VXRS_LOW; i++)
vxrs[i] = *((__u64 *)(target->thread.fpu.vxrs + i) + 1);
- return user_regset_copyout(&pos, &count, &kbuf, &ubuf, vxrs, 0, -1);
+ return membuf_write(&to, vxrs, sizeof(vxrs));
}
static int s390_vxrs_low_set(struct task_struct *target,
@@ -1154,18 +1124,14 @@
static int s390_vxrs_high_get(struct task_struct *target,
const struct user_regset *regset,
- unsigned int pos, unsigned int count,
- void *kbuf, void __user *ubuf)
+ struct membuf to)
{
- __vector128 vxrs[__NUM_VXRS_HIGH];
-
if (!MACHINE_HAS_VX)
return -ENODEV;
if (target == current)
save_fpu_regs();
- memcpy(vxrs, target->thread.fpu.vxrs + __NUM_VXRS_LOW, sizeof(vxrs));
-
- return user_regset_copyout(&pos, &count, &kbuf, &ubuf, vxrs, 0, -1);
+ return membuf_write(&to, target->thread.fpu.vxrs + __NUM_VXRS_LOW,
+ __NUM_VXRS_HIGH * sizeof(__vector128));
}
static int s390_vxrs_high_set(struct task_struct *target,
@@ -1187,12 +1153,9 @@
static int s390_system_call_get(struct task_struct *target,
const struct user_regset *regset,
- unsigned int pos, unsigned int count,
- void *kbuf, void __user *ubuf)
+ struct membuf to)
{
- unsigned int *data = &target->thread.system_call;
- return user_regset_copyout(&pos, &count, &kbuf, &ubuf,
- data, 0, sizeof(unsigned int));
+ return membuf_store(&to, target->thread.system_call);
}
static int s390_system_call_set(struct task_struct *target,
@@ -1207,8 +1170,7 @@
static int s390_gs_cb_get(struct task_struct *target,
const struct user_regset *regset,
- unsigned int pos, unsigned int count,
- void *kbuf, void __user *ubuf)
+ struct membuf to)
{
struct gs_cb *data = target->thread.gs_cb;
@@ -1218,8 +1180,7 @@
return -ENODATA;
if (target == current)
save_gs_cb(data);
- return user_regset_copyout(&pos, &count, &kbuf, &ubuf,
- data, 0, sizeof(struct gs_cb));
+ return membuf_write(&to, data, sizeof(struct gs_cb));
}
static int s390_gs_cb_set(struct task_struct *target,
@@ -1263,8 +1224,7 @@
static int s390_gs_bc_get(struct task_struct *target,
const struct user_regset *regset,
- unsigned int pos, unsigned int count,
- void *kbuf, void __user *ubuf)
+ struct membuf to)
{
struct gs_cb *data = target->thread.gs_bc_cb;
@@ -1272,8 +1232,7 @@
return -ENODEV;
if (!data)
return -ENODATA;
- return user_regset_copyout(&pos, &count, &kbuf, &ubuf,
- data, 0, sizeof(struct gs_cb));
+ return membuf_write(&to, data, sizeof(struct gs_cb));
}
static int s390_gs_bc_set(struct task_struct *target,
@@ -1324,8 +1283,7 @@
static int s390_runtime_instr_get(struct task_struct *target,
const struct user_regset *regset,
- unsigned int pos, unsigned int count,
- void *kbuf, void __user *ubuf)
+ struct membuf to)
{
struct runtime_instr_cb *data = target->thread.ri_cb;
@@ -1334,8 +1292,7 @@
if (!data)
return -ENODATA;
- return user_regset_copyout(&pos, &count, &kbuf, &ubuf,
- data, 0, sizeof(struct runtime_instr_cb));
+ return membuf_write(&to, data, sizeof(struct runtime_instr_cb));
}
static int s390_runtime_instr_set(struct task_struct *target,
@@ -1391,7 +1348,7 @@
.n = sizeof(s390_regs) / sizeof(long),
.size = sizeof(long),
.align = sizeof(long),
- .get = s390_regs_get,
+ .regset_get = s390_regs_get,
.set = s390_regs_set,
},
{
@@ -1399,7 +1356,7 @@
.n = sizeof(s390_fp_regs) / sizeof(long),
.size = sizeof(long),
.align = sizeof(long),
- .get = s390_fpregs_get,
+ .regset_get = s390_fpregs_get,
.set = s390_fpregs_set,
},
{
@@ -1407,7 +1364,7 @@
.n = 1,
.size = sizeof(unsigned int),
.align = sizeof(unsigned int),
- .get = s390_system_call_get,
+ .regset_get = s390_system_call_get,
.set = s390_system_call_set,
},
{
@@ -1415,7 +1372,7 @@
.n = 1,
.size = sizeof(long),
.align = sizeof(long),
- .get = s390_last_break_get,
+ .regset_get = s390_last_break_get,
.set = s390_last_break_set,
},
{
@@ -1423,7 +1380,7 @@
.n = 1,
.size = 256,
.align = 1,
- .get = s390_tdb_get,
+ .regset_get = s390_tdb_get,
.set = s390_tdb_set,
},
{
@@ -1431,7 +1388,7 @@
.n = __NUM_VXRS_LOW,
.size = sizeof(__u64),
.align = sizeof(__u64),
- .get = s390_vxrs_low_get,
+ .regset_get = s390_vxrs_low_get,
.set = s390_vxrs_low_set,
},
{
@@ -1439,7 +1396,7 @@
.n = __NUM_VXRS_HIGH,
.size = sizeof(__vector128),
.align = sizeof(__vector128),
- .get = s390_vxrs_high_get,
+ .regset_get = s390_vxrs_high_get,
.set = s390_vxrs_high_set,
},
{
@@ -1447,7 +1404,7 @@
.n = sizeof(struct gs_cb) / sizeof(__u64),
.size = sizeof(__u64),
.align = sizeof(__u64),
- .get = s390_gs_cb_get,
+ .regset_get = s390_gs_cb_get,
.set = s390_gs_cb_set,
},
{
@@ -1455,7 +1412,7 @@
.n = sizeof(struct gs_cb) / sizeof(__u64),
.size = sizeof(__u64),
.align = sizeof(__u64),
- .get = s390_gs_bc_get,
+ .regset_get = s390_gs_bc_get,
.set = s390_gs_bc_set,
},
{
@@ -1463,7 +1420,7 @@
.n = sizeof(struct runtime_instr_cb) / sizeof(__u64),
.size = sizeof(__u64),
.align = sizeof(__u64),
- .get = s390_runtime_instr_get,
+ .regset_get = s390_runtime_instr_get,
.set = s390_runtime_instr_set,
},
};
@@ -1478,28 +1435,15 @@
#ifdef CONFIG_COMPAT
static int s390_compat_regs_get(struct task_struct *target,
const struct user_regset *regset,
- unsigned int pos, unsigned int count,
- void *kbuf, void __user *ubuf)
+ struct membuf to)
{
+ unsigned n;
+
if (target == current)
save_access_regs(target->thread.acrs);
- if (kbuf) {
- compat_ulong_t *k = kbuf;
- while (count > 0) {
- *k++ = __peek_user_compat(target, pos);
- count -= sizeof(*k);
- pos += sizeof(*k);
- }
- } else {
- compat_ulong_t __user *u = ubuf;
- while (count > 0) {
- if (__put_user(__peek_user_compat(target, pos), u++))
- return -EFAULT;
- count -= sizeof(*u);
- pos += sizeof(*u);
- }
- }
+ for (n = 0; n < sizeof(s390_compat_regs); n += sizeof(compat_ulong_t))
+ membuf_store(&to, __peek_user_compat(target, n));
return 0;
}
@@ -1541,29 +1485,14 @@
static int s390_compat_regs_high_get(struct task_struct *target,
const struct user_regset *regset,
- unsigned int pos, unsigned int count,
- void *kbuf, void __user *ubuf)
+ struct membuf to)
{
compat_ulong_t *gprs_high;
+ int i;
- gprs_high = (compat_ulong_t *)
- &task_pt_regs(target)->gprs[pos / sizeof(compat_ulong_t)];
- if (kbuf) {
- compat_ulong_t *k = kbuf;
- while (count > 0) {
- *k++ = *gprs_high;
- gprs_high += 2;
- count -= sizeof(*k);
- }
- } else {
- compat_ulong_t __user *u = ubuf;
- while (count > 0) {
- if (__put_user(*gprs_high, u++))
- return -EFAULT;
- gprs_high += 2;
- count -= sizeof(*u);
- }
- }
+ gprs_high = (compat_ulong_t *)task_pt_regs(target)->gprs;
+ for (i = 0; i < NUM_GPRS; i++, gprs_high += 2)
+ membuf_store(&to, *gprs_high);
return 0;
}
@@ -1602,23 +1531,11 @@
static int s390_compat_last_break_get(struct task_struct *target,
const struct user_regset *regset,
- unsigned int pos, unsigned int count,
- void *kbuf, void __user *ubuf)
+ struct membuf to)
{
- compat_ulong_t last_break;
+ compat_ulong_t last_break = target->thread.last_break;
- if (count > 0) {
- last_break = target->thread.last_break;
- if (kbuf) {
- unsigned long *k = kbuf;
- *k = last_break;
- } else {
- unsigned long __user *u = ubuf;
- if (__put_user(last_break, u))
- return -EFAULT;
- }
- }
- return 0;
+ return membuf_store(&to, (unsigned long)last_break);
}
static int s390_compat_last_break_set(struct task_struct *target,
@@ -1635,7 +1552,7 @@
.n = sizeof(s390_compat_regs) / sizeof(compat_long_t),
.size = sizeof(compat_long_t),
.align = sizeof(compat_long_t),
- .get = s390_compat_regs_get,
+ .regset_get = s390_compat_regs_get,
.set = s390_compat_regs_set,
},
{
@@ -1643,7 +1560,7 @@
.n = sizeof(s390_fp_regs) / sizeof(compat_long_t),
.size = sizeof(compat_long_t),
.align = sizeof(compat_long_t),
- .get = s390_fpregs_get,
+ .regset_get = s390_fpregs_get,
.set = s390_fpregs_set,
},
{
@@ -1651,7 +1568,7 @@
.n = 1,
.size = sizeof(compat_uint_t),
.align = sizeof(compat_uint_t),
- .get = s390_system_call_get,
+ .regset_get = s390_system_call_get,
.set = s390_system_call_set,
},
{
@@ -1659,7 +1576,7 @@
.n = 1,
.size = sizeof(long),
.align = sizeof(long),
- .get = s390_compat_last_break_get,
+ .regset_get = s390_compat_last_break_get,
.set = s390_compat_last_break_set,
},
{
@@ -1667,7 +1584,7 @@
.n = 1,
.size = 256,
.align = 1,
- .get = s390_tdb_get,
+ .regset_get = s390_tdb_get,
.set = s390_tdb_set,
},
{
@@ -1675,7 +1592,7 @@
.n = __NUM_VXRS_LOW,
.size = sizeof(__u64),
.align = sizeof(__u64),
- .get = s390_vxrs_low_get,
+ .regset_get = s390_vxrs_low_get,
.set = s390_vxrs_low_set,
},
{
@@ -1683,7 +1600,7 @@
.n = __NUM_VXRS_HIGH,
.size = sizeof(__vector128),
.align = sizeof(__vector128),
- .get = s390_vxrs_high_get,
+ .regset_get = s390_vxrs_high_get,
.set = s390_vxrs_high_set,
},
{
@@ -1691,7 +1608,7 @@
.n = sizeof(s390_compat_regs_high) / sizeof(compat_long_t),
.size = sizeof(compat_long_t),
.align = sizeof(compat_long_t),
- .get = s390_compat_regs_high_get,
+ .regset_get = s390_compat_regs_high_get,
.set = s390_compat_regs_high_set,
},
{
@@ -1699,7 +1616,7 @@
.n = sizeof(struct gs_cb) / sizeof(__u64),
.size = sizeof(__u64),
.align = sizeof(__u64),
- .get = s390_gs_cb_get,
+ .regset_get = s390_gs_cb_get,
.set = s390_gs_cb_set,
},
{
@@ -1707,7 +1624,7 @@
.n = sizeof(struct gs_cb) / sizeof(__u64),
.size = sizeof(__u64),
.align = sizeof(__u64),
- .get = s390_gs_bc_get,
+ .regset_get = s390_gs_bc_get,
.set = s390_gs_bc_set,
},
{
@@ -1715,7 +1632,7 @@
.n = sizeof(struct runtime_instr_cb) / sizeof(__u64),
.size = sizeof(__u64),
.align = sizeof(__u64),
- .get = s390_runtime_instr_get,
+ .regset_get = s390_runtime_instr_get,
.set = s390_runtime_instr_set,
},
};
diff --git a/arch/sh/kernel/process_32.c b/arch/sh/kernel/process_32.c
index b0fefd8..cde0a66 100644
--- a/arch/sh/kernel/process_32.c
+++ b/arch/sh/kernel/process_32.c
@@ -103,9 +103,8 @@
fpvalid = !!tsk_used_math(tsk);
if (fpvalid)
- fpvalid = !fpregs_get(tsk, NULL, 0,
- sizeof(struct user_fpu_struct),
- fpu, NULL);
+ fpvalid = !fpregs_get(tsk, NULL,
+ (struct membuf){fpu, sizeof(*fpu)});
#endif
return fpvalid;
diff --git a/arch/sh/kernel/ptrace_32.c b/arch/sh/kernel/ptrace_32.c
index 64bfb71..5c93bdb 100644
--- a/arch/sh/kernel/ptrace_32.c
+++ b/arch/sh/kernel/ptrace_32.c
@@ -134,26 +134,11 @@
static int genregs_get(struct task_struct *target,
const struct user_regset *regset,
- unsigned int pos, unsigned int count,
- void *kbuf, void __user *ubuf)
+ struct membuf to)
{
const struct pt_regs *regs = task_pt_regs(target);
- int ret;
- ret = user_regset_copyout(&pos, &count, &kbuf, &ubuf,
- regs->regs,
- 0, 16 * sizeof(unsigned long));
- if (!ret)
- /* PC, PR, SR, GBR, MACH, MACL, TRA */
- ret = user_regset_copyout(&pos, &count, &kbuf, &ubuf,
- ®s->pc,
- offsetof(struct pt_regs, pc),
- sizeof(struct pt_regs));
- if (!ret)
- ret = user_regset_copyout_zero(&pos, &count, &kbuf, &ubuf,
- sizeof(struct pt_regs), -1);
-
- return ret;
+ return membuf_write(&to, regs, sizeof(struct pt_regs));
}
static int genregs_set(struct task_struct *target,
@@ -182,8 +167,7 @@
#ifdef CONFIG_SH_FPU
int fpregs_get(struct task_struct *target,
const struct user_regset *regset,
- unsigned int pos, unsigned int count,
- void *kbuf, void __user *ubuf)
+ struct membuf to)
{
int ret;
@@ -191,12 +175,8 @@
if (ret)
return ret;
- if ((boot_cpu_data.flags & CPU_HAS_FPU))
- return user_regset_copyout(&pos, &count, &kbuf, &ubuf,
- &target->thread.xstate->hardfpu, 0, -1);
-
- return user_regset_copyout(&pos, &count, &kbuf, &ubuf,
- &target->thread.xstate->softfpu, 0, -1);
+ return membuf_write(&to, target->thread.xstate,
+ sizeof(struct user_fpu_struct));
}
static int fpregs_set(struct task_struct *target,
@@ -230,20 +210,12 @@
#ifdef CONFIG_SH_DSP
static int dspregs_get(struct task_struct *target,
const struct user_regset *regset,
- unsigned int pos, unsigned int count,
- void *kbuf, void __user *ubuf)
+ struct membuf to)
{
const struct pt_dspregs *regs =
(struct pt_dspregs *)&target->thread.dsp_status.dsp_regs;
- int ret;
- ret = user_regset_copyout(&pos, &count, &kbuf, &ubuf, regs,
- 0, sizeof(struct pt_dspregs));
- if (!ret)
- ret = user_regset_copyout_zero(&pos, &count, &kbuf, &ubuf,
- sizeof(struct pt_dspregs), -1);
-
- return ret;
+ return membuf_write(&to, regs, sizeof(struct pt_dspregs));
}
static int dspregs_set(struct task_struct *target,
@@ -324,7 +296,7 @@
.n = ELF_NGREG,
.size = sizeof(long),
.align = sizeof(long),
- .get = genregs_get,
+ .regset_get = genregs_get,
.set = genregs_set,
},
@@ -334,7 +306,7 @@
.n = sizeof(struct user_fpu_struct) / sizeof(long),
.size = sizeof(long),
.align = sizeof(long),
- .get = fpregs_get,
+ .regset_get = fpregs_get,
.set = fpregs_set,
.active = fpregs_active,
},
@@ -345,7 +317,7 @@
.n = sizeof(struct pt_dspregs) / sizeof(long),
.size = sizeof(long),
.align = sizeof(long),
- .get = dspregs_get,
+ .regset_get = dspregs_get,
.set = dspregs_set,
.active = dspregs_active,
},
diff --git a/arch/sh/kernel/setup.c b/arch/sh/kernel/setup.c
index 67f5a3b..4144be6 100644
--- a/arch/sh/kernel/setup.c
+++ b/arch/sh/kernel/setup.c
@@ -290,8 +290,6 @@
#ifdef CONFIG_BLK_DEV_RAM
rd_image_start = RAMDISK_FLAGS & RAMDISK_IMAGE_START_MASK;
- rd_prompt = ((RAMDISK_FLAGS & RAMDISK_PROMPT_FLAG) != 0);
- rd_doload = ((RAMDISK_FLAGS & RAMDISK_LOAD_FLAG) != 0);
#endif
if (!MOUNT_ROOT_RDONLY)
diff --git a/arch/sparc/kernel/ptrace_32.c b/arch/sparc/kernel/ptrace_32.c
index 47eb315..5318174 100644
--- a/arch/sparc/kernel/ptrace_32.c
+++ b/arch/sparc/kernel/ptrace_32.c
@@ -83,41 +83,25 @@
static int genregs32_get(struct task_struct *target,
const struct user_regset *regset,
- unsigned int pos, unsigned int count,
- void *kbuf, void __user *ubuf)
+ struct membuf to)
{
const struct pt_regs *regs = target->thread.kregs;
u32 uregs[16];
- int ret;
if (target == current)
flush_user_windows();
- ret = user_regset_copyout(&pos, &count, &kbuf, &ubuf,
- regs->u_regs,
- 0, 16 * sizeof(u32));
- if (ret || !count)
- return ret;
-
- if (pos < 32 * sizeof(u32)) {
- if (regwindow32_get(target, regs, uregs))
- return -EFAULT;
- ret = user_regset_copyout(&pos, &count, &kbuf, &ubuf,
- uregs,
- 16 * sizeof(u32), 32 * sizeof(u32));
- if (ret || !count)
- return ret;
- }
-
- uregs[0] = regs->psr;
- uregs[1] = regs->pc;
- uregs[2] = regs->npc;
- uregs[3] = regs->y;
- uregs[4] = 0; /* WIM */
- uregs[5] = 0; /* TBR */
- return user_regset_copyout(&pos, &count, &kbuf, &ubuf,
- uregs,
- 32 * sizeof(u32), 38 * sizeof(u32));
+ membuf_write(&to, regs->u_regs, 16 * sizeof(u32));
+ if (!to.left)
+ return 0;
+ if (regwindow32_get(target, regs, uregs))
+ return -EFAULT;
+ membuf_write(&to, uregs, 16 * sizeof(u32));
+ membuf_store(&to, regs->psr);
+ membuf_store(&to, regs->pc);
+ membuf_store(&to, regs->npc);
+ membuf_store(&to, regs->y);
+ return membuf_zero(&to, 2 * sizeof(u32));
}
static int genregs32_set(struct task_struct *target,
@@ -139,19 +123,18 @@
if (ret || !count)
return ret;
- if (pos < 32 * sizeof(u32)) {
- if (regwindow32_get(target, regs, uregs))
- return -EFAULT;
- ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf,
- uregs,
- 16 * sizeof(u32), 32 * sizeof(u32));
- if (ret)
- return ret;
- if (regwindow32_set(target, regs, uregs))
- return -EFAULT;
- if (!count)
- return 0;
- }
+ if (regwindow32_get(target, regs, uregs))
+ return -EFAULT;
+ ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf,
+ uregs,
+ 16 * sizeof(u32), 32 * sizeof(u32));
+ if (ret)
+ return ret;
+ if (regwindow32_set(target, regs, uregs))
+ return -EFAULT;
+ if (!count)
+ return 0;
+
ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf,
&psr,
32 * sizeof(u32), 33 * sizeof(u32));
@@ -182,46 +165,18 @@
static int fpregs32_get(struct task_struct *target,
const struct user_regset *regset,
- unsigned int pos, unsigned int count,
- void *kbuf, void __user *ubuf)
+ struct membuf to)
{
- const unsigned long *fpregs = target->thread.float_regs;
- int ret = 0;
-
#if 0
if (target == current)
save_and_clear_fpu();
#endif
- ret = user_regset_copyout(&pos, &count, &kbuf, &ubuf,
- fpregs,
- 0, 32 * sizeof(u32));
-
- if (!ret)
- ret = user_regset_copyout_zero(&pos, &count, &kbuf, &ubuf,
- 32 * sizeof(u32),
- 33 * sizeof(u32));
- if (!ret)
- ret = user_regset_copyout(&pos, &count, &kbuf, &ubuf,
- &target->thread.fsr,
- 33 * sizeof(u32),
- 34 * sizeof(u32));
-
- if (!ret) {
- unsigned long val;
-
- val = (1 << 8) | (8 << 16);
- ret = user_regset_copyout(&pos, &count, &kbuf, &ubuf,
- &val,
- 34 * sizeof(u32),
- 35 * sizeof(u32));
- }
-
- if (!ret)
- ret = user_regset_copyout_zero(&pos, &count, &kbuf, &ubuf,
- 35 * sizeof(u32), -1);
-
- return ret;
+ membuf_write(&to, target->thread.float_regs, 32 * sizeof(u32));
+ membuf_zero(&to, sizeof(u32));
+ membuf_write(&to, &target->thread.fsr, sizeof(u32));
+ membuf_store(&to, (u32)((1 << 8) | (8 << 16)));
+ return membuf_zero(&to, 64 * sizeof(u32));
}
static int fpregs32_set(struct task_struct *target,
@@ -243,13 +198,11 @@
user_regset_copyin_ignore(&pos, &count, &kbuf, &ubuf,
32 * sizeof(u32),
33 * sizeof(u32));
- if (!ret && count > 0) {
+ if (!ret)
ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf,
&target->thread.fsr,
33 * sizeof(u32),
34 * sizeof(u32));
- }
-
if (!ret)
ret = user_regset_copyin_ignore(&pos, &count, &kbuf, &ubuf,
34 * sizeof(u32), -1);
@@ -268,7 +221,7 @@
.core_note_type = NT_PRSTATUS,
.n = 38,
.size = sizeof(u32), .align = sizeof(u32),
- .get = genregs32_get, .set = genregs32_set
+ .regset_get = genregs32_get, .set = genregs32_set
},
/* Format is:
* F0 --> F31
@@ -284,10 +237,104 @@
.core_note_type = NT_PRFPREG,
.n = 99,
.size = sizeof(u32), .align = sizeof(u32),
- .get = fpregs32_get, .set = fpregs32_set
+ .regset_get = fpregs32_get, .set = fpregs32_set
},
};
+static int getregs_get(struct task_struct *target,
+ const struct user_regset *regset,
+ struct membuf to)
+{
+ const struct pt_regs *regs = target->thread.kregs;
+
+ if (target == current)
+ flush_user_windows();
+
+ membuf_store(&to, regs->psr);
+ membuf_store(&to, regs->pc);
+ membuf_store(&to, regs->npc);
+ membuf_store(&to, regs->y);
+ return membuf_write(&to, regs->u_regs + 1, 15 * sizeof(u32));
+}
+
+static int setregs_set(struct task_struct *target,
+ const struct user_regset *regset,
+ unsigned int pos, unsigned int count,
+ const void *kbuf, const void __user *ubuf)
+{
+ struct pt_regs *regs = target->thread.kregs;
+ u32 v[4];
+ int ret;
+
+ if (target == current)
+ flush_user_windows();
+
+ ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf,
+ v,
+ 0, 4 * sizeof(u32));
+ if (ret)
+ return ret;
+ regs->psr = (regs->psr & ~(PSR_ICC | PSR_SYSCALL)) |
+ (v[0] & (PSR_ICC | PSR_SYSCALL));
+ regs->pc = v[1];
+ regs->npc = v[2];
+ regs->y = v[3];
+ return user_regset_copyin(&pos, &count, &kbuf, &ubuf,
+ regs->u_regs + 1,
+ 4 * sizeof(u32) , 19 * sizeof(u32));
+}
+
+static int getfpregs_get(struct task_struct *target,
+ const struct user_regset *regset,
+ struct membuf to)
+{
+#if 0
+ if (target == current)
+ save_and_clear_fpu();
+#endif
+ membuf_write(&to, &target->thread.float_regs, 32 * sizeof(u32));
+ membuf_write(&to, &target->thread.fsr, sizeof(u32));
+ return membuf_zero(&to, 35 * sizeof(u32));
+}
+
+static int setfpregs_set(struct task_struct *target,
+ const struct user_regset *regset,
+ unsigned int pos, unsigned int count,
+ const void *kbuf, const void __user *ubuf)
+{
+ unsigned long *fpregs = target->thread.float_regs;
+ int ret;
+
+#if 0
+ if (target == current)
+ save_and_clear_fpu();
+#endif
+ ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf,
+ fpregs,
+ 0, 32 * sizeof(u32));
+ if (ret)
+ return ret;
+ return user_regset_copyin(&pos, &count, &kbuf, &ubuf,
+ &target->thread.fsr,
+ 32 * sizeof(u32),
+ 33 * sizeof(u32));
+}
+
+static const struct user_regset ptrace32_regsets[] = {
+ [REGSET_GENERAL] = {
+ .n = 19, .size = sizeof(u32),
+ .regset_get = getregs_get, .set = setregs_set,
+ },
+ [REGSET_FP] = {
+ .n = 68, .size = sizeof(u32),
+ .regset_get = getfpregs_get, .set = setfpregs_set,
+ },
+};
+
+static const struct user_regset_view ptrace32_view = {
+ .regsets = ptrace32_regsets, .n = ARRAY_SIZE(ptrace32_regsets)
+};
+
static const struct user_regset_view user_sparc32_view = {
.name = "sparc", .e_machine = EM_SPARC,
.regsets = sparc32_regsets, .n = ARRAY_SIZE(sparc32_regsets)
@@ -315,74 +362,44 @@
{
unsigned long addr2 = current->thread.kregs->u_regs[UREG_I4];
void __user *addr2p;
- const struct user_regset_view *view;
struct pt_regs __user *pregs;
struct fps __user *fps;
int ret;
- view = task_user_regset_view(current);
addr2p = (void __user *) addr2;
pregs = (struct pt_regs __user *) addr;
fps = (struct fps __user *) addr;
switch(request) {
case PTRACE_GETREGS: {
- ret = copy_regset_to_user(child, view, REGSET_GENERAL,
- 32 * sizeof(u32),
- 4 * sizeof(u32),
- &pregs->psr);
- if (!ret)
- copy_regset_to_user(child, view, REGSET_GENERAL,
- 1 * sizeof(u32),
- 15 * sizeof(u32),
- &pregs->u_regs[0]);
+ ret = copy_regset_to_user(child, &ptrace32_view,
+ REGSET_GENERAL, 0,
+ 19 * sizeof(u32),
+ pregs);
break;
}
case PTRACE_SETREGS: {
- ret = copy_regset_from_user(child, view, REGSET_GENERAL,
- 32 * sizeof(u32),
- 4 * sizeof(u32),
- &pregs->psr);
- if (!ret)
- copy_regset_from_user(child, view, REGSET_GENERAL,
- 1 * sizeof(u32),
- 15 * sizeof(u32),
- &pregs->u_regs[0]);
+ ret = copy_regset_from_user(child, &ptrace32_view,
+ REGSET_GENERAL, 0,
+ 19 * sizeof(u32),
+ pregs);
break;
}
case PTRACE_GETFPREGS: {
- ret = copy_regset_to_user(child, view, REGSET_FP,
- 0 * sizeof(u32),
- 32 * sizeof(u32),
- &fps->regs[0]);
- if (!ret)
- ret = copy_regset_to_user(child, view, REGSET_FP,
- 33 * sizeof(u32),
- 1 * sizeof(u32),
- &fps->fsr);
-
- if (!ret) {
- if (__put_user(0, &fps->fpqd) ||
- __put_user(0, &fps->flags) ||
- __put_user(0, &fps->extra) ||
- clear_user(fps->fpq, sizeof(fps->fpq)))
- ret = -EFAULT;
- }
+ ret = copy_regset_to_user(child, &ptrace32_view,
+ REGSET_FP, 0,
+ 68 * sizeof(u32),
+ fps);
break;
}
case PTRACE_SETFPREGS: {
- ret = copy_regset_from_user(child, view, REGSET_FP,
- 0 * sizeof(u32),
- 32 * sizeof(u32),
- &fps->regs[0]);
- if (!ret)
- ret = copy_regset_from_user(child, view, REGSET_FP,
- 33 * sizeof(u32),
- 1 * sizeof(u32),
- &fps->fsr);
+ ret = copy_regset_from_user(child, &ptrace32_view,
+ REGSET_FP, 0,
+ 33 * sizeof(u32),
+ fps);
break;
}
diff --git a/arch/sparc/kernel/ptrace_64.c b/arch/sparc/kernel/ptrace_64.c
index 7122efb..2b92155d 100644
--- a/arch/sparc/kernel/ptrace_64.c
+++ b/arch/sparc/kernel/ptrace_64.c
@@ -246,52 +246,23 @@
static int genregs64_get(struct task_struct *target,
const struct user_regset *regset,
- unsigned int pos, unsigned int count,
- void *kbuf, void __user *ubuf)
+ struct membuf to)
{
const struct pt_regs *regs = task_pt_regs(target);
- int ret;
+ struct reg_window window;
if (target == current)
flushw_user();
- ret = user_regset_copyout(&pos, &count, &kbuf, &ubuf,
- regs->u_regs,
- 0, 16 * sizeof(u64));
- if (!ret && count && pos < (32 * sizeof(u64))) {
- struct reg_window window;
-
- if (regwindow64_get(target, regs, &window))
- return -EFAULT;
- ret = user_regset_copyout(&pos, &count, &kbuf, &ubuf,
- &window,
- 16 * sizeof(u64),
- 32 * sizeof(u64));
- }
-
- if (!ret) {
- /* TSTATE, TPC, TNPC */
- ret = user_regset_copyout(&pos, &count, &kbuf, &ubuf,
- ®s->tstate,
- 32 * sizeof(u64),
- 35 * sizeof(u64));
- }
-
- if (!ret) {
- unsigned long y = regs->y;
-
- ret = user_regset_copyout(&pos, &count, &kbuf, &ubuf,
- &y,
- 35 * sizeof(u64),
- 36 * sizeof(u64));
- }
-
- if (!ret) {
- ret = user_regset_copyout_zero(&pos, &count, &kbuf, &ubuf,
- 36 * sizeof(u64), -1);
-
- }
- return ret;
+ membuf_write(&to, regs->u_regs, 16 * sizeof(u64));
+ if (!to.left)
+ return 0;
+ if (regwindow64_get(target, regs, &window))
+ return -EFAULT;
+ membuf_write(&to, &window, 16 * sizeof(u64));
+ /* TSTATE, TPC, TNPC */
+ membuf_write(&to, ®s->tstate, 3 * sizeof(u64));
+ return membuf_store(&to, (u64)regs->y);
}
static int genregs64_set(struct task_struct *target,
@@ -370,69 +341,32 @@
static int fpregs64_get(struct task_struct *target,
const struct user_regset *regset,
- unsigned int pos, unsigned int count,
- void *kbuf, void __user *ubuf)
+ struct membuf to)
{
- const unsigned long *fpregs = task_thread_info(target)->fpregs;
- unsigned long fprs, fsr, gsr;
- int ret;
+ struct thread_info *t = task_thread_info(target);
+ unsigned long fprs;
if (target == current)
save_and_clear_fpu();
- fprs = task_thread_info(target)->fpsaved[0];
+ fprs = t->fpsaved[0];
if (fprs & FPRS_DL)
- ret = user_regset_copyout(&pos, &count, &kbuf, &ubuf,
- fpregs,
- 0, 16 * sizeof(u64));
+ membuf_write(&to, t->fpregs, 16 * sizeof(u64));
else
- ret = user_regset_copyout_zero(&pos, &count, &kbuf, &ubuf,
- 0,
- 16 * sizeof(u64));
+ membuf_zero(&to, 16 * sizeof(u64));
- if (!ret) {
- if (fprs & FPRS_DU)
- ret = user_regset_copyout(&pos, &count,
- &kbuf, &ubuf,
- fpregs + 16,
- 16 * sizeof(u64),
- 32 * sizeof(u64));
- else
- ret = user_regset_copyout_zero(&pos, &count,
- &kbuf, &ubuf,
- 16 * sizeof(u64),
- 32 * sizeof(u64));
- }
-
+ if (fprs & FPRS_DU)
+ membuf_write(&to, t->fpregs + 16, 16 * sizeof(u64));
+ else
+ membuf_zero(&to, 16 * sizeof(u64));
if (fprs & FPRS_FEF) {
- fsr = task_thread_info(target)->xfsr[0];
- gsr = task_thread_info(target)->gsr[0];
+ membuf_store(&to, t->xfsr[0]);
+ membuf_store(&to, t->gsr[0]);
} else {
- fsr = gsr = 0;
+ membuf_zero(&to, 2 * sizeof(u64));
}
-
- if (!ret)
- ret = user_regset_copyout(&pos, &count, &kbuf, &ubuf,
- &fsr,
- 32 * sizeof(u64),
- 33 * sizeof(u64));
- if (!ret)
- ret = user_regset_copyout(&pos, &count, &kbuf, &ubuf,
- &gsr,
- 33 * sizeof(u64),
- 34 * sizeof(u64));
- if (!ret)
- ret = user_regset_copyout(&pos, &count, &kbuf, &ubuf,
- &fprs,
- 34 * sizeof(u64),
- 35 * sizeof(u64));
-
- if (!ret)
- ret = user_regset_copyout_zero(&pos, &count, &kbuf, &ubuf,
- 35 * sizeof(u64), -1);
-
- return ret;
+ return membuf_store(&to, fprs);
}
static int fpregs64_set(struct task_struct *target,
@@ -490,7 +424,7 @@
.core_note_type = NT_PRSTATUS,
.n = 36,
.size = sizeof(u64), .align = sizeof(u64),
- .get = genregs64_get, .set = genregs64_set
+ .regset_get = genregs64_get, .set = genregs64_set
},
/* Format is:
* F0 --> F63
@@ -502,10 +436,96 @@
.core_note_type = NT_PRFPREG,
.n = 35,
.size = sizeof(u64), .align = sizeof(u64),
- .get = fpregs64_get, .set = fpregs64_set
+ .regset_get = fpregs64_get, .set = fpregs64_set
},
};
+static int getregs64_get(struct task_struct *target,
+ const struct user_regset *regset,
+ struct membuf to)
+{
+ const struct pt_regs *regs = task_pt_regs(target);
+
+ if (target == current)
+ flushw_user();
+
+ membuf_write(&to, regs->u_regs + 1, 15 * sizeof(u64));
+ membuf_store(&to, (u64)0);
+ membuf_write(&to, ®s->tstate, 3 * sizeof(u64));
+ return membuf_store(&to, (u64)regs->y);
+}
+
+static int setregs64_set(struct task_struct *target,
+ const struct user_regset *regset,
+ unsigned int pos, unsigned int count,
+ const void *kbuf, const void __user *ubuf)
+{
+ struct pt_regs *regs = task_pt_regs(target);
+ unsigned long y = regs->y;
+ unsigned long tstate;
+ int ret;
+
+ if (target == current)
+ flushw_user();
+
+ ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf,
+ regs->u_regs + 1,
+ 0 * sizeof(u64),
+ 15 * sizeof(u64));
+ if (ret)
+ return ret;
+ ret =user_regset_copyin_ignore(&pos, &count, &kbuf, &ubuf,
+ 15 * sizeof(u64), 16 * sizeof(u64));
+ if (ret)
+ return ret;
+ /* TSTATE */
+ ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf,
+ &tstate,
+ 16 * sizeof(u64),
+ 17 * sizeof(u64));
+ if (ret)
+ return ret;
+ /* Only the condition codes and the "in syscall"
+ * state can be modified in the %tstate register.
+ */
+ tstate &= (TSTATE_ICC | TSTATE_XCC | TSTATE_SYSCALL);
+ regs->tstate &= ~(TSTATE_ICC | TSTATE_XCC | TSTATE_SYSCALL);
+ regs->tstate |= tstate;
+
+ /* TPC, TNPC */
+ ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf,
+ ®s->tpc,
+ 17 * sizeof(u64),
+ 19 * sizeof(u64));
+ if (ret)
+ return ret;
+ /* Y */
+ ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf,
+ &y,
+ 19 * sizeof(u64),
+ 20 * sizeof(u64));
+ if (!ret)
+ regs->y = y;
+ return ret;
+}
+
+static const struct user_regset ptrace64_regsets[] = {
+ /* Format is:
+ * G1 --> G7
+ * O0 --> O7
+ * 0
+ * TSTATE, TPC, TNPC, Y
+ */
+ [REGSET_GENERAL] = {
+ .n = 20, .size = sizeof(u64),
+ .regset_get = getregs64_get, .set = setregs64_set,
+ },
+};
+
+static const struct user_regset_view ptrace64_view = {
+ .regsets = ptrace64_regsets, .n = ARRAY_SIZE(ptrace64_regsets)
+};
+
static const struct user_regset_view user_sparc64_view = {
.name = "sparc64", .e_machine = EM_SPARCV9,
.regsets = sparc64_regsets, .n = ARRAY_SIZE(sparc64_regsets)
@@ -514,108 +534,28 @@
#ifdef CONFIG_COMPAT
static int genregs32_get(struct task_struct *target,
const struct user_regset *regset,
- unsigned int pos, unsigned int count,
- void *kbuf, void __user *ubuf)
+ struct membuf to)
{
const struct pt_regs *regs = task_pt_regs(target);
- compat_ulong_t __user *reg_window;
- compat_ulong_t *k = kbuf;
- compat_ulong_t __user *u = ubuf;
- compat_ulong_t reg;
+ u32 uregs[16];
+ int i;
if (target == current)
flushw_user();
- pos /= sizeof(reg);
- count /= sizeof(reg);
-
- if (kbuf) {
- for (; count > 0 && pos < 16; count--)
- *k++ = regs->u_regs[pos++];
-
- reg_window = (compat_ulong_t __user *) regs->u_regs[UREG_I6];
- reg_window -= 16;
- if (target == current) {
- for (; count > 0 && pos < 32; count--) {
- if (get_user(*k++, ®_window[pos++]))
- return -EFAULT;
- }
- } else {
- for (; count > 0 && pos < 32; count--) {
- if (access_process_vm(target,
- (unsigned long)
- ®_window[pos],
- k, sizeof(*k),
- FOLL_FORCE)
- != sizeof(*k))
- return -EFAULT;
- k++;
- pos++;
- }
- }
- } else {
- for (; count > 0 && pos < 16; count--) {
- if (put_user((compat_ulong_t) regs->u_regs[pos++], u++))
- return -EFAULT;
- }
-
- reg_window = (compat_ulong_t __user *) regs->u_regs[UREG_I6];
- reg_window -= 16;
- if (target == current) {
- for (; count > 0 && pos < 32; count--) {
- if (get_user(reg, ®_window[pos++]) ||
- put_user(reg, u++))
- return -EFAULT;
- }
- } else {
- for (; count > 0 && pos < 32; count--) {
- if (access_process_vm(target,
- (unsigned long)
- ®_window[pos++],
- ®, sizeof(reg),
- FOLL_FORCE)
- != sizeof(reg))
- return -EFAULT;
- if (put_user(reg, u++))
- return -EFAULT;
- }
- }
- }
- while (count > 0) {
- switch (pos) {
- case 32: /* PSR */
- reg = tstate_to_psr(regs->tstate);
- break;
- case 33: /* PC */
- reg = regs->tpc;
- break;
- case 34: /* NPC */
- reg = regs->tnpc;
- break;
- case 35: /* Y */
- reg = regs->y;
- break;
- case 36: /* WIM */
- case 37: /* TBR */
- reg = 0;
- break;
- default:
- goto finish;
- }
-
- if (kbuf)
- *k++ = reg;
- else if (put_user(reg, u++))
- return -EFAULT;
- pos++;
- count--;
- }
-finish:
- pos *= sizeof(reg);
- count *= sizeof(reg);
-
- return user_regset_copyout_zero(&pos, &count, &kbuf, &ubuf,
- 38 * sizeof(reg), -1);
+ for (i = 0; i < 16; i++)
+ membuf_store(&to, (u32)regs->u_regs[i]);
+ if (!to.left)
+ return 0;
+ if (get_from_target(target, regs->u_regs[UREG_I6],
+ uregs, sizeof(uregs)))
+ return -EFAULT;
+ membuf_write(&to, uregs, 16 * sizeof(u32));
+ membuf_store(&to, (u32)tstate_to_psr(regs->tstate));
+ membuf_store(&to, (u32)(regs->tpc));
+ membuf_store(&to, (u32)(regs->tnpc));
+ membuf_store(&to, (u32)(regs->y));
+ return membuf_zero(&to, 2 * sizeof(u32));
}
static int genregs32_set(struct task_struct *target,
@@ -737,56 +677,24 @@
static int fpregs32_get(struct task_struct *target,
const struct user_regset *regset,
- unsigned int pos, unsigned int count,
- void *kbuf, void __user *ubuf)
+ struct membuf to)
{
- const unsigned long *fpregs = task_thread_info(target)->fpregs;
- compat_ulong_t enabled;
- unsigned long fprs;
- compat_ulong_t fsr;
- int ret = 0;
+ struct thread_info *t = task_thread_info(target);
+ bool enabled;
if (target == current)
save_and_clear_fpu();
- fprs = task_thread_info(target)->fpsaved[0];
- if (fprs & FPRS_FEF) {
- fsr = task_thread_info(target)->xfsr[0];
- enabled = 1;
- } else {
- fsr = 0;
- enabled = 0;
- }
+ enabled = t->fpsaved[0] & FPRS_FEF;
- ret = user_regset_copyout(&pos, &count, &kbuf, &ubuf,
- fpregs,
- 0, 32 * sizeof(u32));
-
- if (!ret)
- ret = user_regset_copyout_zero(&pos, &count, &kbuf, &ubuf,
- 32 * sizeof(u32),
- 33 * sizeof(u32));
- if (!ret)
- ret = user_regset_copyout(&pos, &count, &kbuf, &ubuf,
- &fsr,
- 33 * sizeof(u32),
- 34 * sizeof(u32));
-
- if (!ret) {
- compat_ulong_t val;
-
- val = (enabled << 8) | (8 << 16);
- ret = user_regset_copyout(&pos, &count, &kbuf, &ubuf,
- &val,
- 34 * sizeof(u32),
- 35 * sizeof(u32));
- }
-
- if (!ret)
- ret = user_regset_copyout_zero(&pos, &count, &kbuf, &ubuf,
- 35 * sizeof(u32), -1);
-
- return ret;
+ membuf_write(&to, t->fpregs, 32 * sizeof(u32));
+ membuf_zero(&to, sizeof(u32));
+ if (enabled)
+ membuf_store(&to, (u32)t->xfsr[0]);
+ else
+ membuf_zero(&to, sizeof(u32));
+ membuf_store(&to, (u32)((enabled << 8) | (8 << 16)));
+ return membuf_zero(&to, 64 * sizeof(u32));
}
static int fpregs32_set(struct task_struct *target,
@@ -847,7 +755,7 @@
.core_note_type = NT_PRSTATUS,
.n = 38,
.size = sizeof(u32), .align = sizeof(u32),
- .get = genregs32_get, .set = genregs32_set
+ .regset_get = genregs32_get, .set = genregs32_set
},
/* Format is:
* F0 --> F31
@@ -863,10 +771,133 @@
.core_note_type = NT_PRFPREG,
.n = 99,
.size = sizeof(u32), .align = sizeof(u32),
- .get = fpregs32_get, .set = fpregs32_set
+ .regset_get = fpregs32_get, .set = fpregs32_set
},
};
+static int getregs_get(struct task_struct *target,
+ const struct user_regset *regset,
+ struct membuf to)
+{
+ const struct pt_regs *regs = task_pt_regs(target);
+ int i;
+
+ if (target == current)
+ flushw_user();
+
+ membuf_store(&to, (u32)tstate_to_psr(regs->tstate));
+ membuf_store(&to, (u32)(regs->tpc));
+ membuf_store(&to, (u32)(regs->tnpc));
+ membuf_store(&to, (u32)(regs->y));
+ for (i = 1; i < 16; i++)
+ membuf_store(&to, (u32)regs->u_regs[i]);
+ return to.left;
+}
+
+static int setregs_set(struct task_struct *target,
+ const struct user_regset *regset,
+ unsigned int pos, unsigned int count,
+ const void *kbuf, const void __user *ubuf)
+{
+ struct pt_regs *regs = task_pt_regs(target);
+ unsigned long tstate;
+ u32 uregs[19];
+ int i, ret;
+
+ if (target == current)
+ flushw_user();
+
+ ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf,
+ uregs,
+ 0, 19 * sizeof(u32));
+ if (ret)
+ return ret;
+
+ tstate = regs->tstate;
+ tstate &= ~(TSTATE_ICC | TSTATE_XCC | TSTATE_SYSCALL);
+ tstate |= psr_to_tstate_icc(uregs[0]);
+ if (uregs[0] & PSR_SYSCALL)
+ tstate |= TSTATE_SYSCALL;
+ regs->tstate = tstate;
+ regs->tpc = uregs[1];
+ regs->tnpc = uregs[2];
+ regs->y = uregs[3];
+
+ for (i = 1; i < 15; i++)
+ regs->u_regs[i] = uregs[3 + i];
+ return 0;
+}
+
+static int getfpregs_get(struct task_struct *target,
+ const struct user_regset *regset,
+ struct membuf to)
+{
+ struct thread_info *t = task_thread_info(target);
+
+ if (target == current)
+ save_and_clear_fpu();
+
+ membuf_write(&to, t->fpregs, 32 * sizeof(u32));
+ if (t->fpsaved[0] & FPRS_FEF)
+ membuf_store(&to, (u32)t->xfsr[0]);
+ else
+ membuf_zero(&to, sizeof(u32));
+ return membuf_zero(&to, 35 * sizeof(u32));
+}
+
+static int setfpregs_set(struct task_struct *target,
+ const struct user_regset *regset,
+ unsigned int pos, unsigned int count,
+ const void *kbuf, const void __user *ubuf)
+{
+ unsigned long *fpregs = task_thread_info(target)->fpregs;
+ unsigned long fprs;
+ int ret;
+
+ if (target == current)
+ save_and_clear_fpu();
+
+ fprs = task_thread_info(target)->fpsaved[0];
+
+ ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf,
+ fpregs,
+ 0, 32 * sizeof(u32));
+ if (!ret) {
+ compat_ulong_t fsr;
+ unsigned long val;
+
+ ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf,
+ &fsr,
+ 32 * sizeof(u32),
+ 33 * sizeof(u32));
+ if (!ret) {
+ val = task_thread_info(target)->xfsr[0];
+ val &= 0xffffffff00000000UL;
+ val |= fsr;
+ task_thread_info(target)->xfsr[0] = val;
+ }
+ }
+
+ fprs |= (FPRS_FEF | FPRS_DL);
+ task_thread_info(target)->fpsaved[0] = fprs;
+ return ret;
+}
+
+static const struct user_regset ptrace32_regsets[] = {
+ [REGSET_GENERAL] = {
+ .n = 19, .size = sizeof(u32),
+ .regset_get = getregs_get, .set = setregs_set,
+ },
+ [REGSET_FP] = {
+ .n = 68, .size = sizeof(u32),
+ .regset_get = getfpregs_get, .set = setfpregs_set,
+ },
+};
+
+static const struct user_regset_view ptrace32_view = {
+ .regsets = ptrace32_regsets, .n = ARRAY_SIZE(ptrace32_regsets)
+};
+
static const struct user_regset_view user_sparc32_view = {
.name = "sparc", .e_machine = EM_SPARC,
.regsets = sparc32_regsets, .n = ARRAY_SIZE(sparc32_regsets)
@@ -898,7 +929,6 @@
long compat_arch_ptrace(struct task_struct *child, compat_long_t request,
compat_ulong_t caddr, compat_ulong_t cdata)
{
- const struct user_regset_view *view = task_user_regset_view(current);
compat_ulong_t caddr2 = task_pt_regs(current)->u_regs[UREG_I4];
struct pt_regs32 __user *pregs;
struct compat_fps __user *fps;
@@ -916,58 +946,31 @@
break;
case PTRACE_GETREGS:
- ret = copy_regset_to_user(child, view, REGSET_GENERAL,
- 32 * sizeof(u32),
- 4 * sizeof(u32),
- &pregs->psr);
- if (!ret)
- ret = copy_regset_to_user(child, view, REGSET_GENERAL,
- 1 * sizeof(u32),
- 15 * sizeof(u32),
- &pregs->u_regs[0]);
+ ret = copy_regset_to_user(child, &ptrace32_view,
+ REGSET_GENERAL, 0,
+ 19 * sizeof(u32),
+ pregs);
break;
case PTRACE_SETREGS:
- ret = copy_regset_from_user(child, view, REGSET_GENERAL,
- 32 * sizeof(u32),
- 4 * sizeof(u32),
- &pregs->psr);
- if (!ret)
- ret = copy_regset_from_user(child, view, REGSET_GENERAL,
- 1 * sizeof(u32),
- 15 * sizeof(u32),
- &pregs->u_regs[0]);
+ ret = copy_regset_from_user(child, &ptrace32_view,
+ REGSET_GENERAL, 0,
+ 19 * sizeof(u32),
+ pregs);
break;
case PTRACE_GETFPREGS:
- ret = copy_regset_to_user(child, view, REGSET_FP,
- 0 * sizeof(u32),
- 32 * sizeof(u32),
- &fps->regs[0]);
- if (!ret)
- ret = copy_regset_to_user(child, view, REGSET_FP,
- 33 * sizeof(u32),
- 1 * sizeof(u32),
- &fps->fsr);
- if (!ret) {
- if (__put_user(0, &fps->flags) ||
- __put_user(0, &fps->extra) ||
- __put_user(0, &fps->fpqd) ||
- clear_user(&fps->fpq[0], 32 * sizeof(unsigned int)))
- ret = -EFAULT;
- }
+ ret = copy_regset_to_user(child, &ptrace32_view,
+ REGSET_FP, 0,
+ 68 * sizeof(u32),
+ fps);
break;
case PTRACE_SETFPREGS:
- ret = copy_regset_from_user(child, view, REGSET_FP,
- 0 * sizeof(u32),
- 32 * sizeof(u32),
- &fps->regs[0]);
- if (!ret)
- ret = copy_regset_from_user(child, view, REGSET_FP,
- 33 * sizeof(u32),
- 1 * sizeof(u32),
- &fps->fsr);
+ ret = copy_regset_from_user(child, &ptrace32_view,
+ REGSET_FP, 0,
+ 33 * sizeof(u32),
+ fps);
break;
case PTRACE_READTEXT:
@@ -1026,31 +1029,17 @@
break;
case PTRACE_GETREGS64:
- ret = copy_regset_to_user(child, view, REGSET_GENERAL,
- 1 * sizeof(u64),
- 15 * sizeof(u64),
- &pregs->u_regs[0]);
- if (!ret) {
- /* XXX doesn't handle 'y' register correctly XXX */
- ret = copy_regset_to_user(child, view, REGSET_GENERAL,
- 32 * sizeof(u64),
- 4 * sizeof(u64),
- &pregs->tstate);
- }
+ ret = copy_regset_to_user(child, &ptrace64_view,
+ REGSET_GENERAL, 0,
+ 19 * sizeof(u64),
+ pregs);
break;
case PTRACE_SETREGS64:
- ret = copy_regset_from_user(child, view, REGSET_GENERAL,
- 1 * sizeof(u64),
- 15 * sizeof(u64),
- &pregs->u_regs[0]);
- if (!ret) {
- /* XXX doesn't handle 'y' register correctly XXX */
- ret = copy_regset_from_user(child, view, REGSET_GENERAL,
- 32 * sizeof(u64),
- 4 * sizeof(u64),
- &pregs->tstate);
- }
+ ret = copy_regset_from_user(child, &ptrace64_view,
+ REGSET_GENERAL, 0,
+ 19 * sizeof(u64),
+ pregs);
break;
case PTRACE_GETFPREGS64:
diff --git a/arch/sparc/kernel/setup_32.c b/arch/sparc/kernel/setup_32.c
index 6d07b85..eea43a1 100644
--- a/arch/sparc/kernel/setup_32.c
+++ b/arch/sparc/kernel/setup_32.c
@@ -353,8 +353,6 @@
ROOT_DEV = old_decode_dev(root_dev);
#ifdef CONFIG_BLK_DEV_RAM
rd_image_start = ram_flags & RAMDISK_IMAGE_START_MASK;
- rd_prompt = ((ram_flags & RAMDISK_PROMPT_FLAG) != 0);
- rd_doload = ((ram_flags & RAMDISK_LOAD_FLAG) != 0);
#endif
prom_setsync(prom_sync_me);
diff --git a/arch/sparc/kernel/setup_64.c b/arch/sparc/kernel/setup_64.c
index f765fda..d872441 100644
--- a/arch/sparc/kernel/setup_64.c
+++ b/arch/sparc/kernel/setup_64.c
@@ -659,8 +659,6 @@
ROOT_DEV = old_decode_dev(root_dev);
#ifdef CONFIG_BLK_DEV_RAM
rd_image_start = ram_flags & RAMDISK_IMAGE_START_MASK;
- rd_prompt = ((ram_flags & RAMDISK_PROMPT_FLAG) != 0);
- rd_doload = ((ram_flags & RAMDISK_LOAD_FLAG) != 0);
#endif
task_thread_info(&init_task)->kregs = &fake_swapper_regs;
diff --git a/arch/x86/include/asm/fpu/internal.h b/arch/x86/include/asm/fpu/internal.h
index 6b10cda..0a460f2 100644
--- a/arch/x86/include/asm/fpu/internal.h
+++ b/arch/x86/include/asm/fpu/internal.h
@@ -34,7 +34,6 @@
extern void fpu__clear_user_states(struct fpu *fpu);
extern void fpu__clear_all(struct fpu *fpu);
extern int fpu__exception_code(struct fpu *fpu, int trap_nr);
-extern int dump_fpu(struct pt_regs *ptregs, struct user_i387_struct *fpstate);
/*
* Boot time FPU initialization functions:
diff --git a/arch/x86/include/asm/fpu/regset.h b/arch/x86/include/asm/fpu/regset.h
index d5bdffb..4f928d6 100644
--- a/arch/x86/include/asm/fpu/regset.h
+++ b/arch/x86/include/asm/fpu/regset.h
@@ -8,8 +8,8 @@
#include <linux/regset.h>
extern user_regset_active_fn regset_fpregs_active, regset_xregset_fpregs_active;
-extern user_regset_get_fn fpregs_get, xfpregs_get, fpregs_soft_get,
- xstateregs_get;
+extern user_regset_get2_fn fpregs_get, xfpregs_get, fpregs_soft_get,
+ xstateregs_get;
extern user_regset_set_fn fpregs_set, xfpregs_set, fpregs_soft_set,
xstateregs_set;
diff --git a/arch/x86/include/asm/fpu/xstate.h b/arch/x86/include/asm/fpu/xstate.h
index 1559554..14ab815 100644
--- a/arch/x86/include/asm/fpu/xstate.h
+++ b/arch/x86/include/asm/fpu/xstate.h
@@ -104,8 +104,8 @@
const void *get_xsave_field_ptr(int xfeature_nr);
int using_compacted_format(void);
int xfeature_size(int xfeature_nr);
-int copy_xstate_to_kernel(void *kbuf, struct xregs_state *xsave, unsigned int offset, unsigned int size);
-int copy_xstate_to_user(void __user *ubuf, struct xregs_state *xsave, unsigned int offset, unsigned int size);
+struct membuf;
+void copy_xstate_to_kernel(struct membuf to, struct xregs_state *xsave);
int copy_kernel_to_xstate(struct xregs_state *xsave, const void *kbuf);
int copy_user_to_xstate(struct xregs_state *xsave, const void __user *ubuf);
void copy_supervisor_to_kernel(struct xregs_state *xsave);
diff --git a/arch/x86/include/asm/xen/hypercall.h b/arch/x86/include/asm/xen/hypercall.h
index ba4c1b1..454b208 100644
--- a/arch/x86/include/asm/xen/hypercall.h
+++ b/arch/x86/include/asm/xen/hypercall.h
@@ -82,7 +82,7 @@
* - clobber the rest
*
* The result certainly isn't pretty, and it really shows up cpp's
- * weakness as as macro language. Sorry. (But let's just give thanks
+ * weakness as a macro language. Sorry. (But let's just give thanks
* there aren't more than 5 arguments...)
*/
diff --git a/arch/x86/kernel/fpu/regset.c b/arch/x86/kernel/fpu/regset.c
index bd1d064..c413756 100644
--- a/arch/x86/kernel/fpu/regset.c
+++ b/arch/x86/kernel/fpu/regset.c
@@ -27,8 +27,7 @@
}
int xfpregs_get(struct task_struct *target, const struct user_regset *regset,
- unsigned int pos, unsigned int count,
- void *kbuf, void __user *ubuf)
+ struct membuf to)
{
struct fpu *fpu = &target->thread.fpu;
@@ -38,8 +37,7 @@
fpu__prepare_read(fpu);
fpstate_sanitize_xstate(fpu);
- return user_regset_copyout(&pos, &count, &kbuf, &ubuf,
- &fpu->state.fxsave, 0, -1);
+ return membuf_write(&to, &fpu->state.fxsave, sizeof(struct fxregs_state));
}
int xfpregs_set(struct task_struct *target, const struct user_regset *regset,
@@ -74,12 +72,10 @@
}
int xstateregs_get(struct task_struct *target, const struct user_regset *regset,
- unsigned int pos, unsigned int count,
- void *kbuf, void __user *ubuf)
+ struct membuf to)
{
struct fpu *fpu = &target->thread.fpu;
struct xregs_state *xsave;
- int ret;
if (!boot_cpu_has(X86_FEATURE_XSAVE))
return -ENODEV;
@@ -89,10 +85,8 @@
fpu__prepare_read(fpu);
if (using_compacted_format()) {
- if (kbuf)
- ret = copy_xstate_to_kernel(kbuf, xsave, pos, count);
- else
- ret = copy_xstate_to_user(ubuf, xsave, pos, count);
+ copy_xstate_to_kernel(to, xsave);
+ return 0;
} else {
fpstate_sanitize_xstate(fpu);
/*
@@ -105,9 +99,8 @@
/*
* Copy the xstate memory layout.
*/
- ret = user_regset_copyout(&pos, &count, &kbuf, &ubuf, xsave, 0, -1);
+ return membuf_write(&to, xsave, fpu_user_xstate_size);
}
- return ret;
}
int xstateregs_set(struct task_struct *target, const struct user_regset *regset,
@@ -293,8 +286,7 @@
}
int fpregs_get(struct task_struct *target, const struct user_regset *regset,
- unsigned int pos, unsigned int count,
- void *kbuf, void __user *ubuf)
+ struct membuf to)
{
struct fpu *fpu = &target->thread.fpu;
struct user_i387_ia32_struct env;
@@ -302,23 +294,22 @@
fpu__prepare_read(fpu);
if (!boot_cpu_has(X86_FEATURE_FPU))
- return fpregs_soft_get(target, regset, pos, count, kbuf, ubuf);
+ return fpregs_soft_get(target, regset, to);
- if (!boot_cpu_has(X86_FEATURE_FXSR))
- return user_regset_copyout(&pos, &count, &kbuf, &ubuf,
- &fpu->state.fsave, 0,
- -1);
+ if (!boot_cpu_has(X86_FEATURE_FXSR)) {
+ return membuf_write(&to, &fpu->state.fsave,
+ sizeof(struct fregs_state));
+ }
fpstate_sanitize_xstate(fpu);
- if (kbuf && pos == 0 && count == sizeof(env)) {
- convert_from_fxsr(kbuf, target);
+ if (to.left == sizeof(env)) {
+ convert_from_fxsr(to.p, target);
return 0;
}
convert_from_fxsr(&env, target);
-
- return user_regset_copyout(&pos, &count, &kbuf, &ubuf, &env, 0, -1);
+ return membuf_write(&to, &env, sizeof(env));
}
int fpregs_set(struct task_struct *target, const struct user_regset *regset,
@@ -356,20 +347,4 @@
return ret;
}
-/*
- * FPU state for core dumps.
- * This is only used for a.out dumps now.
- * It is declared generically using elf_fpregset_t (which is
- * struct user_i387_struct) but is in fact only used for 32-bit
- * dumps, so on 64-bit it is really struct user_i387_ia32_struct.
- */
-int dump_fpu(struct pt_regs *regs, struct user_i387_struct *ufpu)
-{
- struct task_struct *tsk = current;
-
- return !fpregs_get(tsk, NULL, 0, sizeof(struct user_i387_ia32_struct),
- ufpu, NULL);
-}
-EXPORT_SYMBOL(dump_fpu);
-
#endif /* CONFIG_X86_32 || CONFIG_IA32_EMULATION */
diff --git a/arch/x86/kernel/fpu/signal.c b/arch/x86/kernel/fpu/signal.c
index 9393a44..a4ec653 100644
--- a/arch/x86/kernel/fpu/signal.c
+++ b/arch/x86/kernel/fpu/signal.c
@@ -170,14 +170,15 @@
ia32_fxstate &= (IS_ENABLED(CONFIG_X86_32) ||
IS_ENABLED(CONFIG_IA32_EMULATION));
+ if (!static_cpu_has(X86_FEATURE_FPU)) {
+ struct user_i387_ia32_struct fp;
+ fpregs_soft_get(current, NULL, (struct membuf){.p = &fp,
+ .left = sizeof(fp)});
+ return copy_to_user(buf, &fp, sizeof(fp)) ? -EFAULT : 0;
+ }
+
if (!access_ok(buf, size))
return -EACCES;
-
- if (!static_cpu_has(X86_FEATURE_FPU))
- return fpregs_soft_get(current, NULL, 0,
- sizeof(struct user_i387_ia32_struct), NULL,
- (struct _fpstate_32 __user *) buf) ? -1 : 1;
-
retry:
/*
* Load the FPU registers if they are not valid for the current task.
diff --git a/arch/x86/kernel/fpu/xstate.c b/arch/x86/kernel/fpu/xstate.c
index be2a68a..7a2bf88 100644
--- a/arch/x86/kernel/fpu/xstate.c
+++ b/arch/x86/kernel/fpu/xstate.c
@@ -1014,32 +1014,20 @@
return true;
}
-static void fill_gap(unsigned to, void **kbuf, unsigned *pos, unsigned *count)
+static void fill_gap(struct membuf *to, unsigned *last, unsigned offset)
{
- if (*pos < to) {
- unsigned size = to - *pos;
-
- if (size > *count)
- size = *count;
- memcpy(*kbuf, (void *)&init_fpstate.xsave + *pos, size);
- *kbuf += size;
- *pos += size;
- *count -= size;
- }
+ if (*last >= offset)
+ return;
+ membuf_write(to, (void *)&init_fpstate.xsave + *last, offset - *last);
+ *last = offset;
}
-static void copy_part(unsigned offset, unsigned size, void *from,
- void **kbuf, unsigned *pos, unsigned *count)
+static void copy_part(struct membuf *to, unsigned *last, unsigned offset,
+ unsigned size, void *from)
{
- fill_gap(offset, kbuf, pos, count);
- if (size > *count)
- size = *count;
- if (size) {
- memcpy(*kbuf, from, size);
- *kbuf += size;
- *pos += size;
- *count -= size;
- }
+ fill_gap(to, last, offset);
+ membuf_write(to, from, size);
+ *last = offset + size;
}
/*
@@ -1049,20 +1037,15 @@
* It supports partial copy but pos always starts from zero. This is called
* from xstateregs_get() and there we check the CPU has XSAVES.
*/
-int copy_xstate_to_kernel(void *kbuf, struct xregs_state *xsave, unsigned int offset_start, unsigned int size_total)
+void copy_xstate_to_kernel(struct membuf to, struct xregs_state *xsave)
{
struct xstate_header header;
const unsigned off_mxcsr = offsetof(struct fxregs_state, mxcsr);
- unsigned count = size_total;
+ unsigned size = to.left;
+ unsigned last = 0;
int i;
/*
- * Currently copy_regset_to_user() starts from pos 0:
- */
- if (unlikely(offset_start != 0))
- return -EFAULT;
-
- /*
* The destination is a ptrace buffer; we put in only user xstates:
*/
memset(&header, 0, sizeof(header));
@@ -1070,27 +1053,26 @@
header.xfeatures &= xfeatures_mask_user();
if (header.xfeatures & XFEATURE_MASK_FP)
- copy_part(0, off_mxcsr,
- &xsave->i387, &kbuf, &offset_start, &count);
+ copy_part(&to, &last, 0, off_mxcsr, &xsave->i387);
if (header.xfeatures & (XFEATURE_MASK_SSE | XFEATURE_MASK_YMM))
- copy_part(off_mxcsr, MXCSR_AND_FLAGS_SIZE,
- &xsave->i387.mxcsr, &kbuf, &offset_start, &count);
+ copy_part(&to, &last, off_mxcsr,
+ MXCSR_AND_FLAGS_SIZE, &xsave->i387.mxcsr);
if (header.xfeatures & XFEATURE_MASK_FP)
- copy_part(offsetof(struct fxregs_state, st_space), 128,
- &xsave->i387.st_space, &kbuf, &offset_start, &count);
+ copy_part(&to, &last, offsetof(struct fxregs_state, st_space),
+ 128, &xsave->i387.st_space);
if (header.xfeatures & XFEATURE_MASK_SSE)
- copy_part(xstate_offsets[XFEATURE_SSE], 256,
- &xsave->i387.xmm_space, &kbuf, &offset_start, &count);
+ copy_part(&to, &last, xstate_offsets[XFEATURE_SSE],
+ 256, &xsave->i387.xmm_space);
/*
* Fill xsave->i387.sw_reserved value for ptrace frame:
*/
- copy_part(offsetof(struct fxregs_state, sw_reserved), 48,
- xstate_fx_sw_bytes, &kbuf, &offset_start, &count);
+ copy_part(&to, &last, offsetof(struct fxregs_state, sw_reserved),
+ 48, xstate_fx_sw_bytes);
/*
* Copy xregs_state->header:
*/
- copy_part(offsetof(struct xregs_state, header), sizeof(header),
- &header, &kbuf, &offset_start, &count);
+ copy_part(&to, &last, offsetof(struct xregs_state, header),
+ sizeof(header), &header);
for (i = FIRST_EXTENDED_XFEATURE; i < XFEATURE_MAX; i++) {
/*
@@ -1099,104 +1081,12 @@
if ((header.xfeatures >> i) & 1) {
void *src = __raw_xsave_addr(xsave, i);
- copy_part(xstate_offsets[i], xstate_sizes[i],
- src, &kbuf, &offset_start, &count);
+ copy_part(&to, &last, xstate_offsets[i],
+ xstate_sizes[i], src);
}
}
- fill_gap(size_total, &kbuf, &offset_start, &count);
-
- return 0;
-}
-
-static inline int
-__copy_xstate_to_user(void __user *ubuf, const void *data, unsigned int offset, unsigned int size, unsigned int size_total)
-{
- if (!size)
- return 0;
-
- if (offset < size_total) {
- unsigned int copy = min(size, size_total - offset);
-
- if (__copy_to_user(ubuf + offset, data, copy))
- return -EFAULT;
- }
- return 0;
-}
-
-/*
- * Convert from kernel XSAVES compacted format to standard format and copy
- * to a user-space buffer. It supports partial copy but pos always starts from
- * zero. This is called from xstateregs_get() and there we check the CPU
- * has XSAVES.
- */
-int copy_xstate_to_user(void __user *ubuf, struct xregs_state *xsave, unsigned int offset_start, unsigned int size_total)
-{
- unsigned int offset, size;
- int ret, i;
- struct xstate_header header;
-
- /*
- * Currently copy_regset_to_user() starts from pos 0:
- */
- if (unlikely(offset_start != 0))
- return -EFAULT;
-
- /*
- * The destination is a ptrace buffer; we put in only user xstates:
- */
- memset(&header, 0, sizeof(header));
- header.xfeatures = xsave->header.xfeatures;
- header.xfeatures &= xfeatures_mask_user();
-
- /*
- * Copy xregs_state->header:
- */
- offset = offsetof(struct xregs_state, header);
- size = sizeof(header);
-
- ret = __copy_xstate_to_user(ubuf, &header, offset, size, size_total);
- if (ret)
- return ret;
-
- for (i = 0; i < XFEATURE_MAX; i++) {
- /*
- * Copy only in-use xstates:
- */
- if ((header.xfeatures >> i) & 1) {
- void *src = __raw_xsave_addr(xsave, i);
-
- offset = xstate_offsets[i];
- size = xstate_sizes[i];
-
- /* The next component has to fit fully into the output buffer: */
- if (offset + size > size_total)
- break;
-
- ret = __copy_xstate_to_user(ubuf, src, offset, size, size_total);
- if (ret)
- return ret;
- }
-
- }
-
- if (xfeatures_mxcsr_quirk(header.xfeatures)) {
- offset = offsetof(struct fxregs_state, mxcsr);
- size = MXCSR_AND_FLAGS_SIZE;
- __copy_xstate_to_user(ubuf, &xsave->i387.mxcsr, offset, size, size_total);
- }
-
- /*
- * Fill xsave->i387.sw_reserved value for ptrace frame:
- */
- offset = offsetof(struct fxregs_state, sw_reserved);
- size = sizeof(xstate_fx_sw_bytes);
-
- ret = __copy_xstate_to_user(ubuf, xstate_fx_sw_bytes, offset, size, size_total);
- if (ret)
- return ret;
-
- return 0;
+ fill_gap(&to, &last, size);
}
/*
diff --git a/arch/x86/kernel/ptrace.c b/arch/x86/kernel/ptrace.c
index 3f00648..5679aa3 100644
--- a/arch/x86/kernel/ptrace.c
+++ b/arch/x86/kernel/ptrace.c
@@ -412,26 +412,12 @@
static int genregs_get(struct task_struct *target,
const struct user_regset *regset,
- unsigned int pos, unsigned int count,
- void *kbuf, void __user *ubuf)
+ struct membuf to)
{
- if (kbuf) {
- unsigned long *k = kbuf;
- while (count >= sizeof(*k)) {
- *k++ = getreg(target, pos);
- count -= sizeof(*k);
- pos += sizeof(*k);
- }
- } else {
- unsigned long __user *u = ubuf;
- while (count >= sizeof(*u)) {
- if (__put_user(getreg(target, pos), u++))
- return -EFAULT;
- count -= sizeof(*u);
- pos += sizeof(*u);
- }
- }
+ int reg;
+ for (reg = 0; to.left; reg++)
+ membuf_store(&to, getreg(target, reg * sizeof(unsigned long)));
return 0;
}
@@ -695,16 +681,14 @@
static int ioperm_get(struct task_struct *target,
const struct user_regset *regset,
- unsigned int pos, unsigned int count,
- void *kbuf, void __user *ubuf)
+ struct membuf to)
{
struct io_bitmap *iobm = target->thread.io_bitmap;
if (!iobm)
return -ENXIO;
- return user_regset_copyout(&pos, &count, &kbuf, &ubuf,
- iobm->bitmap, 0, IO_BITMAP_BYTES);
+ return membuf_write(&to, iobm->bitmap, IO_BITMAP_BYTES);
}
/*
@@ -1007,28 +991,15 @@
static int genregs32_get(struct task_struct *target,
const struct user_regset *regset,
- unsigned int pos, unsigned int count,
- void *kbuf, void __user *ubuf)
+ struct membuf to)
{
- if (kbuf) {
- compat_ulong_t *k = kbuf;
- while (count >= sizeof(*k)) {
- getreg32(target, pos, k++);
- count -= sizeof(*k);
- pos += sizeof(*k);
- }
- } else {
- compat_ulong_t __user *u = ubuf;
- while (count >= sizeof(*u)) {
- compat_ulong_t word;
- getreg32(target, pos, &word);
- if (__put_user(word, u++))
- return -EFAULT;
- count -= sizeof(*u);
- pos += sizeof(*u);
- }
- }
+ int reg;
+ for (reg = 0; to.left; reg++) {
+ u32 val;
+ getreg32(target, reg * 4, &val);
+ membuf_store(&to, val);
+ }
return 0;
}
@@ -1238,25 +1209,25 @@
.core_note_type = NT_PRSTATUS,
.n = sizeof(struct user_regs_struct) / sizeof(long),
.size = sizeof(long), .align = sizeof(long),
- .get = genregs_get, .set = genregs_set
+ .regset_get = genregs_get, .set = genregs_set
},
[REGSET_FP] = {
.core_note_type = NT_PRFPREG,
.n = sizeof(struct user_i387_struct) / sizeof(long),
.size = sizeof(long), .align = sizeof(long),
- .active = regset_xregset_fpregs_active, .get = xfpregs_get, .set = xfpregs_set
+ .active = regset_xregset_fpregs_active, .regset_get = xfpregs_get, .set = xfpregs_set
},
[REGSET_XSTATE] = {
.core_note_type = NT_X86_XSTATE,
.size = sizeof(u64), .align = sizeof(u64),
- .active = xstateregs_active, .get = xstateregs_get,
+ .active = xstateregs_active, .regset_get = xstateregs_get,
.set = xstateregs_set
},
[REGSET_IOPERM64] = {
.core_note_type = NT_386_IOPERM,
.n = IO_BITMAP_LONGS,
.size = sizeof(long), .align = sizeof(long),
- .active = ioperm_active, .get = ioperm_get
+ .active = ioperm_active, .regset_get = ioperm_get
},
};
@@ -1279,24 +1250,24 @@
.core_note_type = NT_PRSTATUS,
.n = sizeof(struct user_regs_struct32) / sizeof(u32),
.size = sizeof(u32), .align = sizeof(u32),
- .get = genregs32_get, .set = genregs32_set
+ .regset_get = genregs32_get, .set = genregs32_set
},
[REGSET_FP] = {
.core_note_type = NT_PRFPREG,
.n = sizeof(struct user_i387_ia32_struct) / sizeof(u32),
.size = sizeof(u32), .align = sizeof(u32),
- .active = regset_fpregs_active, .get = fpregs_get, .set = fpregs_set
+ .active = regset_fpregs_active, .regset_get = fpregs_get, .set = fpregs_set
},
[REGSET_XFP] = {
.core_note_type = NT_PRXFPREG,
.n = sizeof(struct user32_fxsr_struct) / sizeof(u32),
.size = sizeof(u32), .align = sizeof(u32),
- .active = regset_xregset_fpregs_active, .get = xfpregs_get, .set = xfpregs_set
+ .active = regset_xregset_fpregs_active, .regset_get = xfpregs_get, .set = xfpregs_set
},
[REGSET_XSTATE] = {
.core_note_type = NT_X86_XSTATE,
.size = sizeof(u64), .align = sizeof(u64),
- .active = xstateregs_active, .get = xstateregs_get,
+ .active = xstateregs_active, .regset_get = xstateregs_get,
.set = xstateregs_set
},
[REGSET_TLS] = {
@@ -1305,13 +1276,13 @@
.size = sizeof(struct user_desc),
.align = sizeof(struct user_desc),
.active = regset_tls_active,
- .get = regset_tls_get, .set = regset_tls_set
+ .regset_get = regset_tls_get, .set = regset_tls_set
},
[REGSET_IOPERM32] = {
.core_note_type = NT_386_IOPERM,
.n = IO_BITMAP_BYTES / sizeof(u32),
.size = sizeof(u32), .align = sizeof(u32),
- .active = ioperm_active, .get = ioperm_get
+ .active = ioperm_active, .regset_get = ioperm_get
},
};
diff --git a/arch/x86/kernel/setup.c b/arch/x86/kernel/setup.c
index a3767e7..b9a68d8 100644
--- a/arch/x86/kernel/setup.c
+++ b/arch/x86/kernel/setup.c
@@ -870,8 +870,6 @@
#ifdef CONFIG_BLK_DEV_RAM
rd_image_start = boot_params.hdr.ram_size & RAMDISK_IMAGE_START_MASK;
- rd_prompt = ((boot_params.hdr.ram_size & RAMDISK_PROMPT_FLAG) != 0);
- rd_doload = ((boot_params.hdr.ram_size & RAMDISK_LOAD_FLAG) != 0);
#endif
#ifdef CONFIG_EFI
if (!strncmp((char *)&boot_params.efi_info.efi_loader_signature,
diff --git a/arch/x86/kernel/tls.c b/arch/x86/kernel/tls.c
index 71d3fef..64a496a 100644
--- a/arch/x86/kernel/tls.c
+++ b/arch/x86/kernel/tls.c
@@ -256,36 +256,16 @@
}
int regset_tls_get(struct task_struct *target, const struct user_regset *regset,
- unsigned int pos, unsigned int count,
- void *kbuf, void __user *ubuf)
+ struct membuf to)
{
const struct desc_struct *tls;
+ struct user_desc v;
+ int pos;
- if (pos >= GDT_ENTRY_TLS_ENTRIES * sizeof(struct user_desc) ||
- (pos % sizeof(struct user_desc)) != 0 ||
- (count % sizeof(struct user_desc)) != 0)
- return -EINVAL;
-
- pos /= sizeof(struct user_desc);
- count /= sizeof(struct user_desc);
-
- tls = &target->thread.tls_array[pos];
-
- if (kbuf) {
- struct user_desc *info = kbuf;
- while (count-- > 0)
- fill_user_desc(info++, GDT_ENTRY_TLS_MIN + pos++,
- tls++);
- } else {
- struct user_desc __user *u_info = ubuf;
- while (count-- > 0) {
- struct user_desc info;
- fill_user_desc(&info, GDT_ENTRY_TLS_MIN + pos++, tls++);
- if (__copy_to_user(u_info++, &info, sizeof(info)))
- return -EFAULT;
- }
+ for (pos = 0, tls = target->thread.tls_array; to.left; pos++, tls++) {
+ fill_user_desc(&v, GDT_ENTRY_TLS_MIN + pos, tls);
+ membuf_write(&to, &v, sizeof(v));
}
-
return 0;
}
diff --git a/arch/x86/kernel/tls.h b/arch/x86/kernel/tls.h
index 3a76e1d..fc39447 100644
--- a/arch/x86/kernel/tls.h
+++ b/arch/x86/kernel/tls.h
@@ -12,7 +12,7 @@
#include <linux/regset.h>
extern user_regset_active_fn regset_tls_active;
-extern user_regset_get_fn regset_tls_get;
+extern user_regset_get2_fn regset_tls_get;
extern user_regset_set_fn regset_tls_set;
#endif /* _ARCH_X86_KERNEL_TLS_H */
diff --git a/arch/x86/math-emu/fpu_entry.c b/arch/x86/math-emu/fpu_entry.c
index a873da6..8679a9d 100644
--- a/arch/x86/math-emu/fpu_entry.c
+++ b/arch/x86/math-emu/fpu_entry.c
@@ -689,12 +689,10 @@
int fpregs_soft_get(struct task_struct *target,
const struct user_regset *regset,
- unsigned int pos, unsigned int count,
- void *kbuf, void __user *ubuf)
+ struct membuf to)
{
struct swregs_state *s387 = &target->thread.fpu.state.soft;
const void *space = s387->st_space;
- int ret;
int offset = (S387->ftop & 7) * 10, other = 80 - offset;
RE_ENTRANT_CHECK_OFF;
@@ -709,18 +707,11 @@
S387->fos |= 0xffff0000;
#endif /* PECULIAR_486 */
- ret = user_regset_copyout(&pos, &count, &kbuf, &ubuf, s387, 0,
- offsetof(struct swregs_state, st_space));
-
- /* Copy all registers in stack order. */
- if (!ret)
- ret = user_regset_copyout(&pos, &count, &kbuf, &ubuf,
- space + offset, 0, other);
- if (!ret)
- ret = user_regset_copyout(&pos, &count, &kbuf, &ubuf,
- space, 0, offset);
+ membuf_write(&to, s387, offsetof(struct swregs_state, st_space));
+ membuf_write(&to, space + offset, other);
+ membuf_write(&to, space, offset);
RE_ENTRANT_CHECK_ON;
- return ret;
+ return 0;
}
diff --git a/arch/x86/mm/init_64.c b/arch/x86/mm/init_64.c
index 1c1209d..3b246ae 100644
--- a/arch/x86/mm/init_64.c
+++ b/arch/x86/mm/init_64.c
@@ -1252,28 +1252,23 @@
p4d_t *p4d;
pud_t *pud;
- p4d = p4d_offset(pgd, addr);
- if (p4d_none(*p4d)) {
- /* Can only happen with 5-level paging */
- p4d = p4d_alloc(&init_mm, pgd, addr);
- if (!p4d) {
- lvl = "p4d";
- goto failed;
- }
- }
+ lvl = "p4d";
+ p4d = p4d_alloc(&init_mm, pgd, addr);
+ if (!p4d)
+ goto failed;
+ /*
+ * With 5-level paging the P4D level is not folded. So the PGDs
+ * are now populated and there is no need to walk down to the
+ * PUD level.
+ */
if (pgtable_l5_enabled())
continue;
- pud = pud_offset(p4d, addr);
- if (pud_none(*pud)) {
- /* Ends up here only with 4-level paging */
- pud = pud_alloc(&init_mm, p4d, addr);
- if (!pud) {
- lvl = "pud";
- goto failed;
- }
- }
+ lvl = "pud";
+ pud = pud_alloc(&init_mm, p4d, addr);
+ if (!pud)
+ goto failed;
}
return;
diff --git a/arch/xtensa/kernel/ptrace.c b/arch/xtensa/kernel/ptrace.c
index ce4a32b..bb3f479 100644
--- a/arch/xtensa/kernel/ptrace.c
+++ b/arch/xtensa/kernel/ptrace.c
@@ -39,8 +39,7 @@
static int gpr_get(struct task_struct *target,
const struct user_regset *regset,
- unsigned int pos, unsigned int count,
- void *kbuf, void __user *ubuf)
+ struct membuf to)
{
struct pt_regs *regs = task_pt_regs(target);
struct user_pt_regs newregs = {
@@ -63,8 +62,7 @@
regs->areg,
(WSBITS - regs->windowbase) * 16);
- return user_regset_copyout(&pos, &count, &kbuf, &ubuf,
- &newregs, 0, -1);
+ return membuf_write(&to, &newregs, sizeof(newregs));
}
static int gpr_set(struct task_struct *target,
@@ -121,8 +119,7 @@
static int tie_get(struct task_struct *target,
const struct user_regset *regset,
- unsigned int pos, unsigned int count,
- void *kbuf, void __user *ubuf)
+ struct membuf to)
{
int ret;
struct pt_regs *regs = task_pt_regs(target);
@@ -147,8 +144,7 @@
newregs->cp6 = ti->xtregs_cp.cp6;
newregs->cp7 = ti->xtregs_cp.cp7;
#endif
- ret = user_regset_copyout(&pos, &count, &kbuf, &ubuf,
- newregs, 0, -1);
+ ret = membuf_write(&to, newregs, sizeof(*newregs));
kfree(newregs);
return ret;
}
@@ -203,7 +199,7 @@
.n = sizeof(struct user_pt_regs) / sizeof(u32),
.size = sizeof(u32),
.align = sizeof(u32),
- .get = gpr_get,
+ .regset_get = gpr_get,
.set = gpr_set,
},
[REGSET_TIE] = {
@@ -211,7 +207,7 @@
.n = sizeof(elf_xtregs_t) / sizeof(u32),
.size = sizeof(u32),
.align = sizeof(u32),
- .get = tie_get,
+ .regset_get = tie_get,
.set = tie_set,
},
};
diff --git a/block/blk-pm.c b/block/blk-pm.c
index 1adc1cd..b85234d 100644
--- a/block/blk-pm.c
+++ b/block/blk-pm.c
@@ -164,9 +164,8 @@
*
* Description:
* Update the queue's runtime status according to the return value of the
- * device's runtime_resume function. If it is successfully resumed, process
- * the requests that are queued into the device's queue when it is resuming
- * and then mark last busy and initiate autosuspend for it.
+ * device's runtime_resume function. If the resume was successful, call
+ * blk_set_runtime_active() to do the real work of restarting the queue.
*
* This function should be called near the end of the device's
* runtime_resume callback.
@@ -175,19 +174,13 @@
{
if (!q->dev)
return;
-
- spin_lock_irq(&q->queue_lock);
if (!err) {
- q->rpm_status = RPM_ACTIVE;
- pm_runtime_mark_last_busy(q->dev);
- pm_request_autosuspend(q->dev);
+ blk_set_runtime_active(q);
} else {
+ spin_lock_irq(&q->queue_lock);
q->rpm_status = RPM_SUSPENDED;
+ spin_unlock_irq(&q->queue_lock);
}
- spin_unlock_irq(&q->queue_lock);
-
- if (!err)
- blk_clear_pm_only(q);
}
EXPORT_SYMBOL(blk_post_runtime_resume);
@@ -204,15 +197,25 @@
* This function can be used in driver's resume hook to correct queue
* runtime PM status and re-enable peeking requests from the queue. It
* should be called before first request is added to the queue.
+ *
+ * This function is also called by blk_post_runtime_resume() for successful
+ * runtime resumes. It does everything necessary to restart the queue.
*/
void blk_set_runtime_active(struct request_queue *q)
{
- if (q->dev) {
- spin_lock_irq(&q->queue_lock);
- q->rpm_status = RPM_ACTIVE;
- pm_runtime_mark_last_busy(q->dev);
- pm_request_autosuspend(q->dev);
- spin_unlock_irq(&q->queue_lock);
- }
+ int old_status;
+
+ if (!q->dev)
+ return;
+
+ spin_lock_irq(&q->queue_lock);
+ old_status = q->rpm_status;
+ q->rpm_status = RPM_ACTIVE;
+ pm_runtime_mark_last_busy(q->dev);
+ pm_request_autosuspend(q->dev);
+ spin_unlock_irq(&q->queue_lock);
+
+ if (old_status != RPM_ACTIVE)
+ blk_clear_pm_only(q);
}
EXPORT_SYMBOL(blk_set_runtime_active);
diff --git a/drivers/acpi/thermal.c b/drivers/acpi/thermal.c
index 19067a5..12c0ece 100644
--- a/drivers/acpi/thermal.c
+++ b/drivers/acpi/thermal.c
@@ -172,7 +172,6 @@
struct acpi_thermal_trips trips;
struct acpi_handle_list devices;
struct thermal_zone_device *thermal_zone;
- int tz_enabled;
int kelvin_offset; /* in millidegrees */
struct work_struct thermal_check_work;
};
@@ -500,9 +499,6 @@
{
struct acpi_thermal *tz = data;
- if (!tz->tz_enabled)
- return;
-
thermal_zone_device_update(tz->thermal_zone,
THERMAL_EVENT_UNSPECIFIED);
}
@@ -526,50 +522,6 @@
return 0;
}
-static int thermal_get_mode(struct thermal_zone_device *thermal,
- enum thermal_device_mode *mode)
-{
- struct acpi_thermal *tz = thermal->devdata;
-
- if (!tz)
- return -EINVAL;
-
- *mode = tz->tz_enabled ? THERMAL_DEVICE_ENABLED :
- THERMAL_DEVICE_DISABLED;
-
- return 0;
-}
-
-static int thermal_set_mode(struct thermal_zone_device *thermal,
- enum thermal_device_mode mode)
-{
- struct acpi_thermal *tz = thermal->devdata;
- int enable;
-
- if (!tz)
- return -EINVAL;
-
- /*
- * enable/disable thermal management from ACPI thermal driver
- */
- if (mode == THERMAL_DEVICE_ENABLED)
- enable = 1;
- else if (mode == THERMAL_DEVICE_DISABLED) {
- enable = 0;
- pr_warn("thermal zone will be disabled\n");
- } else
- return -EINVAL;
-
- if (enable != tz->tz_enabled) {
- tz->tz_enabled = enable;
- ACPI_DEBUG_PRINT((ACPI_DB_INFO,
- "%s kernel ACPI thermal control\n",
- tz->tz_enabled ? "Enable" : "Disable"));
- acpi_thermal_check(tz);
- }
- return 0;
-}
-
static int thermal_get_trip_type(struct thermal_zone_device *thermal,
int trip, enum thermal_trip_type *type)
{
@@ -856,8 +808,6 @@
.bind = acpi_thermal_bind_cooling_device,
.unbind = acpi_thermal_unbind_cooling_device,
.get_temp = thermal_get_temp,
- .get_mode = thermal_get_mode,
- .set_mode = thermal_set_mode,
.get_trip_type = thermal_get_trip_type,
.get_trip_temp = thermal_get_trip_temp,
.get_crit_temp = thermal_get_crit_temp,
@@ -901,23 +851,39 @@
result = sysfs_create_link(&tz->device->dev.kobj,
&tz->thermal_zone->device.kobj, "thermal_zone");
if (result)
- return result;
+ goto unregister_tzd;
result = sysfs_create_link(&tz->thermal_zone->device.kobj,
&tz->device->dev.kobj, "device");
if (result)
- return result;
+ goto remove_tz_link;
status = acpi_bus_attach_private_data(tz->device->handle,
tz->thermal_zone);
- if (ACPI_FAILURE(status))
- return -ENODEV;
+ if (ACPI_FAILURE(status)) {
+ result = -ENODEV;
+ goto remove_dev_link;
+ }
- tz->tz_enabled = 1;
+ result = thermal_zone_device_enable(tz->thermal_zone);
+ if (result)
+ goto acpi_bus_detach;
dev_info(&tz->device->dev, "registered as thermal_zone%d\n",
tz->thermal_zone->id);
+
return 0;
+
+acpi_bus_detach:
+ acpi_bus_detach_private_data(tz->device->handle);
+remove_dev_link:
+ sysfs_remove_link(&tz->thermal_zone->device.kobj, "device");
+remove_tz_link:
+ sysfs_remove_link(&tz->device->dev.kobj, "thermal_zone");
+unregister_tzd:
+ thermal_zone_device_unregister(tz->thermal_zone);
+
+ return result;
}
static void acpi_thermal_unregister_thermal_zone(struct acpi_thermal *tz)
diff --git a/drivers/auxdisplay/charlcd.c b/drivers/auxdisplay/charlcd.c
index d58278a..5aee0f54 100644
--- a/drivers/auxdisplay/charlcd.c
+++ b/drivers/auxdisplay/charlcd.c
@@ -485,24 +485,19 @@
shift = 0;
value = 0;
while (*esc && cgoffset < 8) {
- shift ^= 4;
- if (*esc >= '0' && *esc <= '9') {
- value |= (*esc - '0') << shift;
- } else if (*esc >= 'A' && *esc <= 'F') {
- value |= (*esc - 'A' + 10) << shift;
- } else if (*esc >= 'a' && *esc <= 'f') {
- value |= (*esc - 'a' + 10) << shift;
- } else {
- esc++;
- continue;
- }
+ int half;
+ shift ^= 4;
+
+ half = hex_to_bin(*esc++);
+ if (half < 0)
+ continue;
+
+ value |= half << shift;
if (shift == 0) {
cgbytes[cgoffset++] = value;
value = 0;
}
-
- esc++;
}
lcd->ops->write_cmd(lcd, LCD_CMD_SET_CGRAM_ADDR | (cgaddr * 8));
diff --git a/drivers/base/devtmpfs.c b/drivers/base/devtmpfs.c
index c9017e0..eac184e 100644
--- a/drivers/base/devtmpfs.c
+++ b/drivers/base/devtmpfs.c
@@ -25,6 +25,7 @@
#include <linux/sched.h>
#include <linux/slab.h>
#include <linux/kthread.h>
+#include <linux/init_syscalls.h>
#include <uapi/linux/mount.h>
#include "base.h"
@@ -359,7 +360,7 @@
if (!thread)
return 0;
- err = do_mount("devtmpfs", "dev", "devtmpfs", MS_SILENT, NULL);
+ err = init_mount("devtmpfs", "dev", "devtmpfs", MS_SILENT, NULL);
if (err)
printk(KERN_INFO "devtmpfs: error mounting %i\n", err);
else
@@ -378,30 +379,8 @@
return handle_remove(name, dev);
}
-static int devtmpfs_setup(void *p)
+static void __noreturn devtmpfs_work_loop(void)
{
- int err;
-
- err = ksys_unshare(CLONE_NEWNS);
- if (err)
- goto out;
- err = do_mount("devtmpfs", "/", "devtmpfs", MS_SILENT, NULL);
- if (err)
- goto out;
- ksys_chdir("/.."); /* will traverse into overmounted root */
- ksys_chroot(".");
-out:
- *(int *)p = err;
- complete(&setup_done);
- return err;
-}
-
-static int devtmpfsd(void *p)
-{
- int err = devtmpfs_setup(p);
-
- if (err)
- return err;
while (1) {
spin_lock(&req_lock);
while (requests) {
@@ -421,6 +400,38 @@
spin_unlock(&req_lock);
schedule();
}
+}
+
+static int __init devtmpfs_setup(void *p)
+{
+ int err;
+
+ err = ksys_unshare(CLONE_NEWNS);
+ if (err)
+ goto out;
+ err = init_mount("devtmpfs", "/", "devtmpfs", MS_SILENT, NULL);
+ if (err)
+ goto out;
+ init_chdir("/.."); /* will traverse into overmounted root */
+ init_chroot(".");
+out:
+ *(int *)p = err;
+ complete(&setup_done);
+ return err;
+}
+
+/*
+ * The __ref is because devtmpfs_setup needs to be __init for the routines it
+ * calls. That call is done while devtmpfs_init, which is marked __init,
+ * synchronously waits for it to complete.
+ */
+static int __ref devtmpfsd(void *p)
+{
+ int err = devtmpfs_setup(p);
+
+ if (err)
+ return err;
+ devtmpfs_work_loop();
return 0;
}
diff --git a/drivers/cpuidle/cpuidle-powernv.c b/drivers/cpuidle/cpuidle-powernv.c
index 1b299e80..addaa6e 100644
--- a/drivers/cpuidle/cpuidle-powernv.c
+++ b/drivers/cpuidle/cpuidle-powernv.c
@@ -244,20 +244,6 @@
stop_psscr_table[index].mask = psscr_mask;
}
-/*
- * Returns 0 if prop1_len == prop2_len. Else returns -1
- */
-static inline int validate_dt_prop_sizes(const char *prop1, int prop1_len,
- const char *prop2, int prop2_len)
-{
- if (prop1_len == prop2_len)
- return 0;
-
- pr_warn("cpuidle-powernv: array sizes don't match for %s and %s\n",
- prop1, prop2);
- return -1;
-}
-
extern u32 pnv_get_supported_cpuidle_states(void);
static int powernv_add_idle_states(void)
{
diff --git a/drivers/cpuidle/cpuidle-pseries.c b/drivers/cpuidle/cpuidle-pseries.c
index 6513ef2..ff6d99e 100644
--- a/drivers/cpuidle/cpuidle-pseries.c
+++ b/drivers/cpuidle/cpuidle-pseries.c
@@ -21,8 +21,9 @@
#include <asm/runlatch.h>
#include <asm/idle.h>
#include <asm/plpar_wrappers.h>
+#include <asm/rtas.h>
-struct cpuidle_driver pseries_idle_driver = {
+static struct cpuidle_driver pseries_idle_driver = {
.name = "pseries_idle",
.owner = THIS_MODULE,
};
@@ -86,19 +87,150 @@
}
}
+/*
+ * XCEDE: Extended CEDE states discovered through the
+ * "ibm,get-systems-parameter" RTAS call with the token
+ * CEDE_LATENCY_TOKEN
+ */
+
+/*
+ * Section 7.3.16 System Parameters Option of PAPR version 2.8.1 has a
+ * table with all the parameters to ibm,get-system-parameters.
+ * CEDE_LATENCY_TOKEN corresponds to the token value for Cede Latency
+ * Settings Information.
+ */
+#define CEDE_LATENCY_TOKEN 45
+
+/*
+ * If the platform supports the cede latency settings information system
+ * parameter it must provide the following information in the NULL terminated
+ * parameter string:
+ *
+ * a. The first byte is the length “N” of each cede latency setting record minus
+ * one (zero indicates a length of 1 byte).
+ *
+ * b. For each supported cede latency setting a cede latency setting record
+ * consisting of the first “N” bytes as per the following table.
+ *
+ * -----------------------------
+ * | Field | Field |
+ * | Name | Length |
+ * -----------------------------
+ * | Cede Latency | 1 Byte |
+ * | Specifier Value | |
+ * -----------------------------
+ * | Maximum wakeup | |
+ * | latency in | 8 Bytes |
+ * | tb-ticks | |
+ * -----------------------------
+ * | Responsive to | |
+ * | external | 1 Byte |
+ * | interrupts | |
+ * -----------------------------
+ *
+ * This version has cede latency record size = 10.
+ *
+ * The structure xcede_latency_payload represents a) and b) with
+ * xcede_latency_record representing the table in b).
+ *
+ * xcede_latency_parameter is what gets returned by
+ * ibm,get-systems-parameter RTAS call when made with
+ * CEDE_LATENCY_TOKEN.
+ *
+ * These structures are only used to represent the data obtained by the RTAS
+ * call. The data is in big-endian.
+ */
+struct xcede_latency_record {
+ u8 hint;
+ __be64 latency_ticks;
+ u8 wake_on_irqs;
+} __packed;
+
+// Make space for 16 records, which "should be enough".
+struct xcede_latency_payload {
+ u8 record_size;
+ struct xcede_latency_record records[16];
+} __packed;
+
+struct xcede_latency_parameter {
+ __be16 payload_size;
+ struct xcede_latency_payload payload;
+ u8 null_char;
+} __packed;
+
+static unsigned int nr_xcede_records;
+static struct xcede_latency_parameter xcede_latency_parameter __initdata;
+
+static int __init parse_cede_parameters(void)
+{
+ struct xcede_latency_payload *payload;
+ u32 total_xcede_records_size;
+ u8 xcede_record_size;
+ u16 payload_size;
+ int ret, i;
+
+ ret = rtas_call(rtas_token("ibm,get-system-parameter"), 3, 1,
+ NULL, CEDE_LATENCY_TOKEN, __pa(&xcede_latency_parameter),
+ sizeof(xcede_latency_parameter));
+ if (ret) {
+ pr_err("xcede: Error parsing CEDE_LATENCY_TOKEN\n");
+ return ret;
+ }
+
+ payload_size = be16_to_cpu(xcede_latency_parameter.payload_size);
+ payload = &xcede_latency_parameter.payload;
+
+ xcede_record_size = payload->record_size + 1;
+
+ if (xcede_record_size != sizeof(struct xcede_latency_record)) {
+ pr_err("xcede: Expected record-size %lu. Observed size %u.\n",
+ sizeof(struct xcede_latency_record), xcede_record_size);
+ return -EINVAL;
+ }
+
+ pr_info("xcede: xcede_record_size = %d\n", xcede_record_size);
+
+ /*
+ * Since the payload_size includes the last NULL byte and the
+ * xcede_record_size, the remaining bytes correspond to array of all
+ * cede_latency settings.
+ */
+ total_xcede_records_size = payload_size - 2;
+ nr_xcede_records = total_xcede_records_size / xcede_record_size;
+
+ for (i = 0; i < nr_xcede_records; i++) {
+ struct xcede_latency_record *record = &payload->records[i];
+ u64 latency_ticks = be64_to_cpu(record->latency_ticks);
+ u8 wake_on_irqs = record->wake_on_irqs;
+ u8 hint = record->hint;
+
+ pr_info("xcede: Record %d : hint = %u, latency = 0x%llx tb ticks, Wake-on-irq = %u\n",
+ i, hint, latency_ticks, wake_on_irqs);
+ }
+
+ return 0;
+}
+
+#define NR_DEDICATED_STATES 2 /* snooze, CEDE */
+static u8 cede_latency_hint[NR_DEDICATED_STATES];
+
static int dedicated_cede_loop(struct cpuidle_device *dev,
struct cpuidle_driver *drv,
int index)
{
+ u8 old_latency_hint;
pseries_idle_prolog();
get_lppaca()->donate_dedicated_cpu = 1;
+ old_latency_hint = get_lppaca()->cede_latency_hint;
+ get_lppaca()->cede_latency_hint = cede_latency_hint[index];
HMT_medium();
check_and_cede_processor();
local_irq_disable();
get_lppaca()->donate_dedicated_cpu = 0;
+ get_lppaca()->cede_latency_hint = old_latency_hint;
pseries_idle_epilog();
@@ -130,7 +262,7 @@
/*
* States for dedicated partition case.
*/
-static struct cpuidle_state dedicated_states[] = {
+static struct cpuidle_state dedicated_states[NR_DEDICATED_STATES] = {
{ /* Snooze */
.name = "snooze",
.desc = "snooze",
@@ -211,6 +343,54 @@
return 0;
}
+static void __init fixup_cede0_latency(void)
+{
+ struct xcede_latency_payload *payload;
+ u64 min_latency_us;
+ int i;
+
+ min_latency_us = dedicated_states[1].exit_latency; // CEDE latency
+
+ if (parse_cede_parameters())
+ return;
+
+ pr_info("cpuidle: Skipping the %d Extended CEDE idle states\n",
+ nr_xcede_records);
+
+ payload = &xcede_latency_parameter.payload;
+ for (i = 0; i < nr_xcede_records; i++) {
+ struct xcede_latency_record *record = &payload->records[i];
+ u64 latency_tb = be64_to_cpu(record->latency_ticks);
+ u64 latency_us = tb_to_ns(latency_tb) / NSEC_PER_USEC;
+
+ if (latency_us < min_latency_us)
+ min_latency_us = latency_us;
+ }
+
+ /*
+ * By default, we assume that CEDE(0) has exit latency 10us,
+ * since there is no way for us to query from the platform.
+ *
+ * However, if the wakeup latency of an Extended CEDE state is
+ * smaller than 10us, then we can be sure that CEDE(0)
+ * requires no more than that.
+ *
+ * Perform the fix-up.
+ */
+ if (min_latency_us < dedicated_states[1].exit_latency) {
+ u64 cede0_latency = min_latency_us - 1;
+
+ if (cede0_latency <= 0)
+ cede0_latency = min_latency_us;
+
+ dedicated_states[1].exit_latency = cede0_latency;
+ dedicated_states[1].target_residency = 10 * (cede0_latency);
+ pr_info("cpuidle: Fixed up CEDE exit latency to %llu us\n",
+ cede0_latency);
+ }
+
+}
+
/*
* pseries_idle_probe()
* Choose state table for shared versus dedicated partition
@@ -232,8 +412,9 @@
cpuidle_state_table = shared_states;
max_idle_state = ARRAY_SIZE(shared_states);
} else {
+ fixup_cede0_latency();
cpuidle_state_table = dedicated_states;
- max_idle_state = ARRAY_SIZE(dedicated_states);
+ max_idle_state = NR_DEDICATED_STATES;
}
} else
return -ENODEV;
diff --git a/drivers/crypto/vmx/aesp8-ppc.pl b/drivers/crypto/vmx/aesp8-ppc.pl
index db87436..50a0a18 100644
--- a/drivers/crypto/vmx/aesp8-ppc.pl
+++ b/drivers/crypto/vmx/aesp8-ppc.pl
@@ -50,7 +50,7 @@
# Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
# project. The module is, however, dual licensed under OpenSSL and
# CRYPTOGAMS licenses depending on where you obtain it. For further
-# details see http://www.openssl.org/~appro/cryptogams/.
+# details see https://www.openssl.org/~appro/cryptogams/.
# ====================================================================
#
# This module implements support for AES instructions as per PowerISA
diff --git a/drivers/crypto/vmx/ghashp8-ppc.pl b/drivers/crypto/vmx/ghashp8-ppc.pl
index 38b0650..09bba18 100644
--- a/drivers/crypto/vmx/ghashp8-ppc.pl
+++ b/drivers/crypto/vmx/ghashp8-ppc.pl
@@ -13,7 +13,7 @@
# Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
# project. The module is, however, dual licensed under OpenSSL and
# CRYPTOGAMS licenses depending on where you obtain it. For further
-# details see http://www.openssl.org/~appro/cryptogams/.
+# details see https://www.openssl.org/~appro/cryptogams/.
# ====================================================================
#
# GHASH for for PowerISA v2.07.
diff --git a/drivers/firmware/qcom_scm.c b/drivers/firmware/qcom_scm.c
index 8393bb3..e8bbf2d 100644
--- a/drivers/firmware/qcom_scm.c
+++ b/drivers/firmware/qcom_scm.c
@@ -923,6 +923,107 @@
EXPORT_SYMBOL(qcom_scm_ocmem_unlock);
/**
+ * qcom_scm_ice_available() - Is the ICE key programming interface available?
+ *
+ * Return: true iff the SCM calls wrapped by qcom_scm_ice_invalidate_key() and
+ * qcom_scm_ice_set_key() are available.
+ */
+bool qcom_scm_ice_available(void)
+{
+ return __qcom_scm_is_call_available(__scm->dev, QCOM_SCM_SVC_ES,
+ QCOM_SCM_ES_INVALIDATE_ICE_KEY) &&
+ __qcom_scm_is_call_available(__scm->dev, QCOM_SCM_SVC_ES,
+ QCOM_SCM_ES_CONFIG_SET_ICE_KEY);
+}
+EXPORT_SYMBOL(qcom_scm_ice_available);
+
+/**
+ * qcom_scm_ice_invalidate_key() - Invalidate an inline encryption key
+ * @index: the keyslot to invalidate
+ *
+ * The UFSHCI standard defines a standard way to do this, but it doesn't work on
+ * these SoCs; only this SCM call does.
+ *
+ * Return: 0 on success; -errno on failure.
+ */
+int qcom_scm_ice_invalidate_key(u32 index)
+{
+ struct qcom_scm_desc desc = {
+ .svc = QCOM_SCM_SVC_ES,
+ .cmd = QCOM_SCM_ES_INVALIDATE_ICE_KEY,
+ .arginfo = QCOM_SCM_ARGS(1),
+ .args[0] = index,
+ .owner = ARM_SMCCC_OWNER_SIP,
+ };
+
+ return qcom_scm_call(__scm->dev, &desc, NULL);
+}
+EXPORT_SYMBOL(qcom_scm_ice_invalidate_key);
+
+/**
+ * qcom_scm_ice_set_key() - Set an inline encryption key
+ * @index: the keyslot into which to set the key
+ * @key: the key to program
+ * @key_size: the size of the key in bytes
+ * @cipher: the encryption algorithm the key is for
+ * @data_unit_size: the encryption data unit size, i.e. the size of each
+ * individual plaintext and ciphertext. Given in 512-byte
+ * units, e.g. 1 = 512 bytes, 8 = 4096 bytes, etc.
+ *
+ * Program a key into a keyslot of Qualcomm ICE (Inline Crypto Engine), where it
+ * can then be used to encrypt/decrypt UFS I/O requests inline.
+ *
+ * The UFSHCI standard defines a standard way to do this, but it doesn't work on
+ * these SoCs; only this SCM call does.
+ *
+ * Return: 0 on success; -errno on failure.
+ */
+int qcom_scm_ice_set_key(u32 index, const u8 *key, u32 key_size,
+ enum qcom_scm_ice_cipher cipher, u32 data_unit_size)
+{
+ struct qcom_scm_desc desc = {
+ .svc = QCOM_SCM_SVC_ES,
+ .cmd = QCOM_SCM_ES_CONFIG_SET_ICE_KEY,
+ .arginfo = QCOM_SCM_ARGS(5, QCOM_SCM_VAL, QCOM_SCM_RW,
+ QCOM_SCM_VAL, QCOM_SCM_VAL,
+ QCOM_SCM_VAL),
+ .args[0] = index,
+ .args[2] = key_size,
+ .args[3] = cipher,
+ .args[4] = data_unit_size,
+ .owner = ARM_SMCCC_OWNER_SIP,
+ };
+ void *keybuf;
+ dma_addr_t key_phys;
+ int ret;
+
+ /*
+ * 'key' may point to vmalloc()'ed memory, but we need to pass a
+ * physical address that's been properly flushed. The sanctioned way to
+ * do this is by using the DMA API. But as is best practice for crypto
+ * keys, we also must wipe the key after use. This makes kmemdup() +
+ * dma_map_single() not clearly correct, since the DMA API can use
+ * bounce buffers. Instead, just use dma_alloc_coherent(). Programming
+ * keys is normally rare and thus not performance-critical.
+ */
+
+ keybuf = dma_alloc_coherent(__scm->dev, key_size, &key_phys,
+ GFP_KERNEL);
+ if (!keybuf)
+ return -ENOMEM;
+ memcpy(keybuf, key, key_size);
+ desc.args[1] = key_phys;
+
+ ret = qcom_scm_call(__scm->dev, &desc, NULL);
+
+ memzero_explicit(keybuf, key_size);
+
+ dma_free_coherent(__scm->dev, key_size, keybuf, key_phys);
+ return ret;
+}
+EXPORT_SYMBOL(qcom_scm_ice_set_key);
+
+/**
* qcom_scm_hdcp_available() - Check if secure environment supports HDCP.
*
* Return true if HDCP is supported, false if not.
diff --git a/drivers/firmware/qcom_scm.h b/drivers/firmware/qcom_scm.h
index d9ed670..38ea614 100644
--- a/drivers/firmware/qcom_scm.h
+++ b/drivers/firmware/qcom_scm.h
@@ -103,6 +103,10 @@
#define QCOM_SCM_OCMEM_LOCK_CMD 0x01
#define QCOM_SCM_OCMEM_UNLOCK_CMD 0x02
+#define QCOM_SCM_SVC_ES 0x10 /* Enterprise Security */
+#define QCOM_SCM_ES_INVALIDATE_ICE_KEY 0x03
+#define QCOM_SCM_ES_CONFIG_SET_ICE_KEY 0x04
+
#define QCOM_SCM_SVC_HDCP 0x11
#define QCOM_SCM_HDCP_INVOKE 0x01
diff --git a/drivers/infiniband/Kconfig b/drivers/infiniband/Kconfig
index a83f9eb..91b0233 100644
--- a/drivers/infiniband/Kconfig
+++ b/drivers/infiniband/Kconfig
@@ -37,14 +37,6 @@
libibverbs, libibcm and a hardware driver library from
rdma-core <https://github.com/linux-rdma/rdma-core>.
-config INFINIBAND_EXP_LEGACY_VERBS_NEW_UAPI
- bool "Allow experimental legacy verbs in new ioctl uAPI (EXPERIMENTAL)"
- depends on INFINIBAND_USER_ACCESS
- help
- IOCTL based uAPI support for Infiniband is enabled by default for
- new verbs only. This allows userspace to invoke the IOCTL based uAPI
- for current legacy verbs too.
-
config INFINIBAND_USER_MEM
bool
depends on INFINIBAND_USER_ACCESS != n
diff --git a/drivers/infiniband/core/cache.c b/drivers/infiniband/core/cache.c
index a670209..ffad73b 100644
--- a/drivers/infiniband/core/cache.c
+++ b/drivers/infiniband/core/cache.c
@@ -1054,7 +1054,7 @@
cache = device->port_data[port_num].cache.pkey;
- if (index < 0 || index >= cache->table_len)
+ if (!cache || index < 0 || index >= cache->table_len)
ret = -EINVAL;
else
*pkey = cache->table[index];
@@ -1099,6 +1099,10 @@
read_lock_irqsave(&device->cache_lock, flags);
cache = device->port_data[port_num].cache.pkey;
+ if (!cache) {
+ ret = -EINVAL;
+ goto err;
+ }
*index = -1;
@@ -1117,6 +1121,7 @@
ret = 0;
}
+err:
read_unlock_irqrestore(&device->cache_lock, flags);
return ret;
@@ -1139,6 +1144,10 @@
read_lock_irqsave(&device->cache_lock, flags);
cache = device->port_data[port_num].cache.pkey;
+ if (!cache) {
+ ret = -EINVAL;
+ goto err;
+ }
*index = -1;
@@ -1149,6 +1158,7 @@
break;
}
+err:
read_unlock_irqrestore(&device->cache_lock, flags);
return ret;
@@ -1425,24 +1435,27 @@
goto err;
}
- pkey_cache = kmalloc(struct_size(pkey_cache, table,
- tprops->pkey_tbl_len),
- GFP_KERNEL);
- if (!pkey_cache) {
- ret = -ENOMEM;
- goto err;
- }
-
- pkey_cache->table_len = tprops->pkey_tbl_len;
-
- for (i = 0; i < pkey_cache->table_len; ++i) {
- ret = ib_query_pkey(device, port, i, pkey_cache->table + i);
- if (ret) {
- dev_warn(&device->dev,
- "ib_query_pkey failed (%d) for index %d\n",
- ret, i);
+ if (tprops->pkey_tbl_len) {
+ pkey_cache = kmalloc(struct_size(pkey_cache, table,
+ tprops->pkey_tbl_len),
+ GFP_KERNEL);
+ if (!pkey_cache) {
+ ret = -ENOMEM;
goto err;
}
+
+ pkey_cache->table_len = tprops->pkey_tbl_len;
+
+ for (i = 0; i < pkey_cache->table_len; ++i) {
+ ret = ib_query_pkey(device, port, i,
+ pkey_cache->table + i);
+ if (ret) {
+ dev_warn(&device->dev,
+ "ib_query_pkey failed (%d) for index %d\n",
+ ret, i);
+ goto err;
+ }
+ }
}
write_lock_irq(&device->cache_lock);
diff --git a/drivers/infiniband/core/cma.c b/drivers/infiniband/core/cma.c
index c30cf53..26de0da 100644
--- a/drivers/infiniband/core/cma.c
+++ b/drivers/infiniband/core/cma.c
@@ -428,19 +428,6 @@
return ret;
}
-static enum rdma_cm_state cma_exch(struct rdma_id_private *id_priv,
- enum rdma_cm_state exch)
-{
- unsigned long flags;
- enum rdma_cm_state old;
-
- spin_lock_irqsave(&id_priv->lock, flags);
- old = id_priv->state;
- id_priv->state = exch;
- spin_unlock_irqrestore(&id_priv->lock, flags);
- return old;
-}
-
static inline u8 cma_get_ip_ver(const struct cma_hdr *hdr)
{
return hdr->ip_version >> 4;
@@ -1829,23 +1816,11 @@
}
}
-void rdma_destroy_id(struct rdma_cm_id *id)
+static void _destroy_id(struct rdma_id_private *id_priv,
+ enum rdma_cm_state state)
{
- struct rdma_id_private *id_priv;
- enum rdma_cm_state state;
-
- id_priv = container_of(id, struct rdma_id_private, id);
- trace_cm_id_destroy(id_priv);
- state = cma_exch(id_priv, RDMA_CM_DESTROYING);
cma_cancel_operation(id_priv, state);
- /*
- * Wait for any active callback to finish. New callbacks will find
- * the id_priv state set to destroying and abort.
- */
- mutex_lock(&id_priv->handler_mutex);
- mutex_unlock(&id_priv->handler_mutex);
-
rdma_restrack_del(&id_priv->res);
if (id_priv->cma_dev) {
if (rdma_cap_ib_cm(id_priv->id.device, 1)) {
@@ -1874,6 +1849,42 @@
put_net(id_priv->id.route.addr.dev_addr.net);
kfree(id_priv);
}
+
+/*
+ * destroy an ID from within the handler_mutex. This ensures that no other
+ * handlers can start running concurrently.
+ */
+static void destroy_id_handler_unlock(struct rdma_id_private *id_priv)
+ __releases(&idprv->handler_mutex)
+{
+ enum rdma_cm_state state;
+ unsigned long flags;
+
+ trace_cm_id_destroy(id_priv);
+
+ /*
+ * Setting the state to destroyed under the handler mutex provides a
+ * fence against calling handler callbacks. If this is invoked due to
+ * the failure of a handler callback then it guarentees that no future
+ * handlers will be called.
+ */
+ lockdep_assert_held(&id_priv->handler_mutex);
+ spin_lock_irqsave(&id_priv->lock, flags);
+ state = id_priv->state;
+ id_priv->state = RDMA_CM_DESTROYING;
+ spin_unlock_irqrestore(&id_priv->lock, flags);
+ mutex_unlock(&id_priv->handler_mutex);
+ _destroy_id(id_priv, state);
+}
+
+void rdma_destroy_id(struct rdma_cm_id *id)
+{
+ struct rdma_id_private *id_priv =
+ container_of(id, struct rdma_id_private, id);
+
+ mutex_lock(&id_priv->handler_mutex);
+ destroy_id_handler_unlock(id_priv);
+}
EXPORT_SYMBOL(rdma_destroy_id);
static int cma_rep_recv(struct rdma_id_private *id_priv)
@@ -1925,6 +1936,8 @@
{
int ret;
+ lockdep_assert_held(&id_priv->handler_mutex);
+
trace_cm_event_handler(id_priv, event);
ret = id_priv->id.event_handler(&id_priv->id, event);
trace_cm_event_done(id_priv, event, ret);
@@ -1936,7 +1949,7 @@
{
struct rdma_id_private *id_priv = cm_id->context;
struct rdma_cm_event event = {};
- int ret = 0;
+ int ret;
mutex_lock(&id_priv->handler_mutex);
if ((ib_event->event != IB_CM_TIMEWAIT_EXIT &&
@@ -2005,14 +2018,12 @@
if (ret) {
/* Destroy the CM ID by returning a non-zero value. */
id_priv->cm_id.ib = NULL;
- cma_exch(id_priv, RDMA_CM_DESTROYING);
- mutex_unlock(&id_priv->handler_mutex);
- rdma_destroy_id(&id_priv->id);
+ destroy_id_handler_unlock(id_priv);
return ret;
}
out:
mutex_unlock(&id_priv->handler_mutex);
- return ret;
+ return 0;
}
static struct rdma_id_private *
@@ -2174,7 +2185,7 @@
mutex_lock(&listen_id->handler_mutex);
if (listen_id->state != RDMA_CM_LISTEN) {
ret = -ECONNABORTED;
- goto err1;
+ goto err_unlock;
}
offset = cma_user_data_offset(listen_id);
@@ -2191,55 +2202,38 @@
}
if (!conn_id) {
ret = -ENOMEM;
- goto err1;
+ goto err_unlock;
}
mutex_lock_nested(&conn_id->handler_mutex, SINGLE_DEPTH_NESTING);
ret = cma_ib_acquire_dev(conn_id, listen_id, &req);
- if (ret)
- goto err2;
+ if (ret) {
+ destroy_id_handler_unlock(conn_id);
+ goto err_unlock;
+ }
conn_id->cm_id.ib = cm_id;
cm_id->context = conn_id;
cm_id->cm_handler = cma_ib_handler;
- /*
- * Protect against the user destroying conn_id from another thread
- * until we're done accessing it.
- */
- cma_id_get(conn_id);
ret = cma_cm_event_handler(conn_id, &event);
- if (ret)
- goto err3;
- /*
- * Acquire mutex to prevent user executing rdma_destroy_id()
- * while we're accessing the cm_id.
- */
- mutex_lock(&lock);
+ if (ret) {
+ /* Destroy the CM ID by returning a non-zero value. */
+ conn_id->cm_id.ib = NULL;
+ mutex_unlock(&listen_id->handler_mutex);
+ destroy_id_handler_unlock(conn_id);
+ goto net_dev_put;
+ }
+
if (cma_comp(conn_id, RDMA_CM_CONNECT) &&
(conn_id->id.qp_type != IB_QPT_UD)) {
trace_cm_send_mra(cm_id->context);
ib_send_cm_mra(cm_id, CMA_CM_MRA_SETTING, NULL, 0);
}
- mutex_unlock(&lock);
mutex_unlock(&conn_id->handler_mutex);
- mutex_unlock(&listen_id->handler_mutex);
- cma_id_put(conn_id);
- if (net_dev)
- dev_put(net_dev);
- return 0;
-err3:
- cma_id_put(conn_id);
- /* Destroy the CM ID by returning a non-zero value. */
- conn_id->cm_id.ib = NULL;
-err2:
- cma_exch(conn_id, RDMA_CM_DESTROYING);
- mutex_unlock(&conn_id->handler_mutex);
-err1:
+err_unlock:
mutex_unlock(&listen_id->handler_mutex);
- if (conn_id)
- rdma_destroy_id(&conn_id->id);
net_dev_put:
if (net_dev)
@@ -2339,9 +2333,7 @@
if (ret) {
/* Destroy the CM ID by returning a non-zero value. */
id_priv->cm_id.iw = NULL;
- cma_exch(id_priv, RDMA_CM_DESTROYING);
- mutex_unlock(&id_priv->handler_mutex);
- rdma_destroy_id(&id_priv->id);
+ destroy_id_handler_unlock(id_priv);
return ret;
}
@@ -2388,16 +2380,16 @@
ret = rdma_translate_ip(laddr, &conn_id->id.route.addr.dev_addr);
if (ret) {
- mutex_unlock(&conn_id->handler_mutex);
- rdma_destroy_id(new_cm_id);
- goto out;
+ mutex_unlock(&listen_id->handler_mutex);
+ destroy_id_handler_unlock(conn_id);
+ return ret;
}
ret = cma_iw_acquire_dev(conn_id, listen_id);
if (ret) {
- mutex_unlock(&conn_id->handler_mutex);
- rdma_destroy_id(new_cm_id);
- goto out;
+ mutex_unlock(&listen_id->handler_mutex);
+ destroy_id_handler_unlock(conn_id);
+ return ret;
}
conn_id->cm_id.iw = cm_id;
@@ -2407,25 +2399,16 @@
memcpy(cma_src_addr(conn_id), laddr, rdma_addr_size(laddr));
memcpy(cma_dst_addr(conn_id), raddr, rdma_addr_size(raddr));
- /*
- * Protect against the user destroying conn_id from another thread
- * until we're done accessing it.
- */
- cma_id_get(conn_id);
ret = cma_cm_event_handler(conn_id, &event);
if (ret) {
/* User wants to destroy the CM ID */
conn_id->cm_id.iw = NULL;
- cma_exch(conn_id, RDMA_CM_DESTROYING);
- mutex_unlock(&conn_id->handler_mutex);
mutex_unlock(&listen_id->handler_mutex);
- cma_id_put(conn_id);
- rdma_destroy_id(&conn_id->id);
+ destroy_id_handler_unlock(conn_id);
return ret;
}
mutex_unlock(&conn_id->handler_mutex);
- cma_id_put(conn_id);
out:
mutex_unlock(&listen_id->handler_mutex);
@@ -2482,6 +2465,10 @@
{
struct rdma_id_private *id_priv = id->context;
+ /* Listening IDs are always destroyed on removal */
+ if (event->event == RDMA_CM_EVENT_DEVICE_REMOVAL)
+ return -1;
+
id->context = id_priv->id.context;
id->event_handler = id_priv->id.event_handler;
trace_cm_event_handler(id_priv, event);
@@ -2657,21 +2644,21 @@
{
struct cma_work *work = container_of(_work, struct cma_work, work);
struct rdma_id_private *id_priv = work->id;
- int destroy = 0;
mutex_lock(&id_priv->handler_mutex);
if (!cma_comp_exch(id_priv, work->old_state, work->new_state))
- goto out;
+ goto out_unlock;
if (cma_cm_event_handler(id_priv, &work->event)) {
- cma_exch(id_priv, RDMA_CM_DESTROYING);
- destroy = 1;
+ cma_id_put(id_priv);
+ destroy_id_handler_unlock(id_priv);
+ goto out_free;
}
-out:
+
+out_unlock:
mutex_unlock(&id_priv->handler_mutex);
cma_id_put(id_priv);
- if (destroy)
- rdma_destroy_id(&id_priv->id);
+out_free:
kfree(work);
}
@@ -2679,23 +2666,22 @@
{
struct cma_ndev_work *work = container_of(_work, struct cma_ndev_work, work);
struct rdma_id_private *id_priv = work->id;
- int destroy = 0;
mutex_lock(&id_priv->handler_mutex);
if (id_priv->state == RDMA_CM_DESTROYING ||
id_priv->state == RDMA_CM_DEVICE_REMOVAL)
- goto out;
+ goto out_unlock;
if (cma_cm_event_handler(id_priv, &work->event)) {
- cma_exch(id_priv, RDMA_CM_DESTROYING);
- destroy = 1;
+ cma_id_put(id_priv);
+ destroy_id_handler_unlock(id_priv);
+ goto out_free;
}
-out:
+out_unlock:
mutex_unlock(&id_priv->handler_mutex);
cma_id_put(id_priv);
- if (destroy)
- rdma_destroy_id(&id_priv->id);
+out_free:
kfree(work);
}
@@ -3171,9 +3157,7 @@
event.event = RDMA_CM_EVENT_ADDR_RESOLVED;
if (cma_cm_event_handler(id_priv, &event)) {
- cma_exch(id_priv, RDMA_CM_DESTROYING);
- mutex_unlock(&id_priv->handler_mutex);
- rdma_destroy_id(&id_priv->id);
+ destroy_id_handler_unlock(id_priv);
return;
}
out:
@@ -3790,7 +3774,7 @@
struct rdma_cm_event event = {};
const struct ib_cm_sidr_rep_event_param *rep =
&ib_event->param.sidr_rep_rcvd;
- int ret = 0;
+ int ret;
mutex_lock(&id_priv->handler_mutex);
if (id_priv->state != RDMA_CM_CONNECT)
@@ -3840,14 +3824,12 @@
if (ret) {
/* Destroy the CM ID by returning a non-zero value. */
id_priv->cm_id.ib = NULL;
- cma_exch(id_priv, RDMA_CM_DESTROYING);
- mutex_unlock(&id_priv->handler_mutex);
- rdma_destroy_id(&id_priv->id);
+ destroy_id_handler_unlock(id_priv);
return ret;
}
out:
mutex_unlock(&id_priv->handler_mutex);
- return ret;
+ return 0;
}
static int cma_resolve_ib_udp(struct rdma_id_private *id_priv,
@@ -4372,9 +4354,7 @@
rdma_destroy_ah_attr(&event.param.ud.ah_attr);
if (ret) {
- cma_exch(id_priv, RDMA_CM_DESTROYING);
- mutex_unlock(&id_priv->handler_mutex);
- rdma_destroy_id(&id_priv->id);
+ destroy_id_handler_unlock(id_priv);
return 0;
}
@@ -4789,50 +4769,59 @@
return ret;
}
-static int cma_remove_id_dev(struct rdma_id_private *id_priv)
+static void cma_send_device_removal_put(struct rdma_id_private *id_priv)
{
- struct rdma_cm_event event = {};
+ struct rdma_cm_event event = { .event = RDMA_CM_EVENT_DEVICE_REMOVAL };
enum rdma_cm_state state;
- int ret = 0;
+ unsigned long flags;
- /* Record that we want to remove the device */
- state = cma_exch(id_priv, RDMA_CM_DEVICE_REMOVAL);
- if (state == RDMA_CM_DESTROYING)
- return 0;
-
- cma_cancel_operation(id_priv, state);
mutex_lock(&id_priv->handler_mutex);
+ /* Record that we want to remove the device */
+ spin_lock_irqsave(&id_priv->lock, flags);
+ state = id_priv->state;
+ if (state == RDMA_CM_DESTROYING || state == RDMA_CM_DEVICE_REMOVAL) {
+ spin_unlock_irqrestore(&id_priv->lock, flags);
+ mutex_unlock(&id_priv->handler_mutex);
+ cma_id_put(id_priv);
+ return;
+ }
+ id_priv->state = RDMA_CM_DEVICE_REMOVAL;
+ spin_unlock_irqrestore(&id_priv->lock, flags);
- /* Check for destruction from another callback. */
- if (!cma_comp(id_priv, RDMA_CM_DEVICE_REMOVAL))
- goto out;
-
- event.event = RDMA_CM_EVENT_DEVICE_REMOVAL;
- ret = cma_cm_event_handler(id_priv, &event);
-out:
+ if (cma_cm_event_handler(id_priv, &event)) {
+ /*
+ * At this point the ULP promises it won't call
+ * rdma_destroy_id() concurrently
+ */
+ cma_id_put(id_priv);
+ mutex_unlock(&id_priv->handler_mutex);
+ trace_cm_id_destroy(id_priv);
+ _destroy_id(id_priv, state);
+ return;
+ }
mutex_unlock(&id_priv->handler_mutex);
- return ret;
+
+ /*
+ * If this races with destroy then the thread that first assigns state
+ * to a destroying does the cancel.
+ */
+ cma_cancel_operation(id_priv, state);
+ cma_id_put(id_priv);
}
static void cma_process_remove(struct cma_device *cma_dev)
{
- struct rdma_id_private *id_priv;
- int ret;
-
mutex_lock(&lock);
while (!list_empty(&cma_dev->id_list)) {
- id_priv = list_entry(cma_dev->id_list.next,
- struct rdma_id_private, list);
+ struct rdma_id_private *id_priv = list_first_entry(
+ &cma_dev->id_list, struct rdma_id_private, list);
list_del(&id_priv->listen_list);
list_del_init(&id_priv->list);
cma_id_get(id_priv);
mutex_unlock(&lock);
- ret = id_priv->internal_id ? 1 : cma_remove_id_dev(id_priv);
- cma_id_put(id_priv);
- if (ret)
- rdma_destroy_id(&id_priv->id);
+ cma_send_device_removal_put(id_priv);
mutex_lock(&lock);
}
diff --git a/drivers/infiniband/core/counters.c b/drivers/infiniband/core/counters.c
index 738d1fa..6361668 100644
--- a/drivers/infiniband/core/counters.c
+++ b/drivers/infiniband/core/counters.c
@@ -8,7 +8,7 @@
#include "core_priv.h"
#include "restrack.h"
-#define ALL_AUTO_MODE_MASKS (RDMA_COUNTER_MASK_QP_TYPE)
+#define ALL_AUTO_MODE_MASKS (RDMA_COUNTER_MASK_QP_TYPE | RDMA_COUNTER_MASK_PID)
static int __counter_set_mode(struct rdma_counter_mode *curr,
enum rdma_nl_counter_mode new_mode,
@@ -149,23 +149,13 @@
struct auto_mode_param *param = &counter->mode.param;
bool match = true;
- /*
- * Ensure that counter belongs to the right PID. This operation can
- * race with user space which kills the process and leaves QP and
- * counters orphans.
- *
- * It is not a big deal because exitted task will leave both QP and
- * counter in the same bucket of zombie process. Just ensure that
- * process is still alive before procedding.
- *
- */
- if (task_pid_nr(counter->res.task) != task_pid_nr(qp->res.task) ||
- !task_pid_nr(qp->res.task))
- return false;
-
if (auto_mask & RDMA_COUNTER_MASK_QP_TYPE)
match &= (param->qp_type == qp->qp_type);
+ if (auto_mask & RDMA_COUNTER_MASK_PID)
+ match &= (task_pid_nr(counter->res.task) ==
+ task_pid_nr(qp->res.task));
+
return match;
}
@@ -288,7 +278,7 @@
struct rdma_counter *counter;
int ret;
- if (!qp->res.valid)
+ if (!qp->res.valid || rdma_is_kernel_res(&qp->res))
return 0;
if (!rdma_is_port_valid(dev, port))
@@ -483,7 +473,7 @@
goto err;
}
- if (counter->res.task != qp->res.task) {
+ if (rdma_is_kernel_res(&counter->res) != rdma_is_kernel_res(&qp->res)) {
ret = -EINVAL;
goto err_task;
}
diff --git a/drivers/infiniband/core/device.c b/drivers/infiniband/core/device.c
index 2927a9d..ef0cd29 100644
--- a/drivers/infiniband/core/device.c
+++ b/drivers/infiniband/core/device.c
@@ -272,7 +272,6 @@
} mandatory_table[] = {
IB_MANDATORY_FUNC(query_device),
IB_MANDATORY_FUNC(query_port),
- IB_MANDATORY_FUNC(query_pkey),
IB_MANDATORY_FUNC(alloc_pd),
IB_MANDATORY_FUNC(dealloc_pd),
IB_MANDATORY_FUNC(create_qp),
@@ -1343,6 +1342,10 @@
return ret;
}
+static void prevent_dealloc_device(struct ib_device *ib_dev)
+{
+}
+
/**
* ib_register_device - Register an IB device with IB core
* @device: Device to register
@@ -1413,11 +1416,11 @@
* possibility for a parallel unregistration along with this
* error flow. Since we have a refcount here we know any
* parallel flow is stopped in disable_device and will see the
- * NULL pointers, causing the responsibility to
+ * special dealloc_driver pointer, causing the responsibility to
* ib_dealloc_device() to revert back to this thread.
*/
dealloc_fn = device->ops.dealloc_driver;
- device->ops.dealloc_driver = NULL;
+ device->ops.dealloc_driver = prevent_dealloc_device;
ib_device_put(device);
__ib_unregister_device(device);
device->ops.dealloc_driver = dealloc_fn;
@@ -1466,7 +1469,8 @@
* Drivers using the new flow may not call ib_dealloc_device except
* in error unwind prior to registration success.
*/
- if (ib_dev->ops.dealloc_driver) {
+ if (ib_dev->ops.dealloc_driver &&
+ ib_dev->ops.dealloc_driver != prevent_dealloc_device) {
WARN_ON(kref_read(&ib_dev->dev.kobj.kref) <= 1);
ib_dealloc_device(ib_dev);
}
@@ -2361,6 +2365,9 @@
if (!rdma_is_port_valid(device, port_num))
return -EINVAL;
+ if (!device->ops.query_pkey)
+ return -EOPNOTSUPP;
+
return device->ops.query_pkey(device, port_num, index, pkey);
}
EXPORT_SYMBOL(ib_query_pkey);
@@ -2621,8 +2628,14 @@
SET_DEVICE_OP(dev_ops, drain_rq);
SET_DEVICE_OP(dev_ops, drain_sq);
SET_DEVICE_OP(dev_ops, enable_driver);
- SET_DEVICE_OP(dev_ops, fill_res_entry);
- SET_DEVICE_OP(dev_ops, fill_stat_entry);
+ SET_DEVICE_OP(dev_ops, fill_res_cm_id_entry);
+ SET_DEVICE_OP(dev_ops, fill_res_cq_entry);
+ SET_DEVICE_OP(dev_ops, fill_res_cq_entry_raw);
+ SET_DEVICE_OP(dev_ops, fill_res_mr_entry);
+ SET_DEVICE_OP(dev_ops, fill_res_mr_entry_raw);
+ SET_DEVICE_OP(dev_ops, fill_res_qp_entry);
+ SET_DEVICE_OP(dev_ops, fill_res_qp_entry_raw);
+ SET_DEVICE_OP(dev_ops, fill_stat_mr_entry);
SET_DEVICE_OP(dev_ops, get_dev_fw_str);
SET_DEVICE_OP(dev_ops, get_dma_mr);
SET_DEVICE_OP(dev_ops, get_hw_stats);
@@ -2667,6 +2680,7 @@
SET_DEVICE_OP(dev_ops, query_port);
SET_DEVICE_OP(dev_ops, query_qp);
SET_DEVICE_OP(dev_ops, query_srq);
+ SET_DEVICE_OP(dev_ops, query_ucontext);
SET_DEVICE_OP(dev_ops, rdma_netdev_get_params);
SET_DEVICE_OP(dev_ops, read_counters);
SET_DEVICE_OP(dev_ops, reg_dm_mr);
@@ -2679,10 +2693,12 @@
SET_DEVICE_OP(dev_ops, set_vf_link_state);
SET_OBJ_SIZE(dev_ops, ib_ah);
+ SET_OBJ_SIZE(dev_ops, ib_counters);
SET_OBJ_SIZE(dev_ops, ib_cq);
SET_OBJ_SIZE(dev_ops, ib_pd);
SET_OBJ_SIZE(dev_ops, ib_srq);
SET_OBJ_SIZE(dev_ops, ib_ucontext);
+ SET_OBJ_SIZE(dev_ops, ib_xrcd);
}
EXPORT_SYMBOL(ib_set_device_ops);
diff --git a/drivers/infiniband/core/mad.c b/drivers/infiniband/core/mad.c
index a09f8e3..9355e52 100644
--- a/drivers/infiniband/core/mad.c
+++ b/drivers/infiniband/core/mad.c
@@ -402,7 +402,7 @@
INIT_DELAYED_WORK(&mad_agent_priv->timed_work, timeout_sends);
INIT_LIST_HEAD(&mad_agent_priv->local_list);
INIT_WORK(&mad_agent_priv->local_work, local_completions);
- atomic_set(&mad_agent_priv->refcount, 1);
+ refcount_set(&mad_agent_priv->refcount, 1);
init_completion(&mad_agent_priv->comp);
ret2 = ib_mad_agent_security_setup(&mad_agent_priv->agent, qp_type);
@@ -484,7 +484,7 @@
static inline void deref_mad_agent(struct ib_mad_agent_private *mad_agent_priv)
{
- if (atomic_dec_and_test(&mad_agent_priv->refcount))
+ if (refcount_dec_and_test(&mad_agent_priv->refcount))
complete(&mad_agent_priv->comp);
}
@@ -718,7 +718,7 @@
* Reference MAD agent until receive
* side of local completion handled
*/
- atomic_inc(&mad_agent_priv->refcount);
+ refcount_inc(&mad_agent_priv->refcount);
} else
kfree(mad_priv);
break;
@@ -758,7 +758,7 @@
local->return_wc_byte_len = mad_size;
}
/* Reference MAD agent until send side of local completion handled */
- atomic_inc(&mad_agent_priv->refcount);
+ refcount_inc(&mad_agent_priv->refcount);
/* Queue local completion to local list */
spin_lock_irqsave(&mad_agent_priv->lock, flags);
list_add_tail(&local->completion_list, &mad_agent_priv->local_list);
@@ -916,7 +916,7 @@
}
mad_send_wr->send_buf.mad_agent = mad_agent;
- atomic_inc(&mad_agent_priv->refcount);
+ refcount_inc(&mad_agent_priv->refcount);
return &mad_send_wr->send_buf;
}
EXPORT_SYMBOL(ib_create_send_mad);
@@ -1131,7 +1131,7 @@
mad_send_wr->status = IB_WC_SUCCESS;
/* Reference MAD agent until send completes */
- atomic_inc(&mad_agent_priv->refcount);
+ refcount_inc(&mad_agent_priv->refcount);
spin_lock_irqsave(&mad_agent_priv->lock, flags);
list_add_tail(&mad_send_wr->agent_list,
&mad_agent_priv->send_list);
@@ -1148,7 +1148,7 @@
spin_lock_irqsave(&mad_agent_priv->lock, flags);
list_del(&mad_send_wr->agent_list);
spin_unlock_irqrestore(&mad_agent_priv->lock, flags);
- atomic_dec(&mad_agent_priv->refcount);
+ deref_mad_agent(mad_agent_priv);
goto error;
}
}
@@ -1554,7 +1554,7 @@
hi_tid = be64_to_cpu(mad_hdr->tid) >> 32;
rcu_read_lock();
mad_agent = xa_load(&ib_mad_clients, hi_tid);
- if (mad_agent && !atomic_inc_not_zero(&mad_agent->refcount))
+ if (mad_agent && !refcount_inc_not_zero(&mad_agent->refcount))
mad_agent = NULL;
rcu_read_unlock();
} else {
@@ -1606,7 +1606,7 @@
}
}
if (mad_agent)
- atomic_inc(&mad_agent->refcount);
+ refcount_inc(&mad_agent->refcount);
out:
spin_unlock_irqrestore(&port_priv->reg_lock, flags);
}
@@ -1831,7 +1831,7 @@
mad_agent_priv->agent.recv_handler(
&mad_agent_priv->agent, NULL,
mad_recv_wc);
- atomic_dec(&mad_agent_priv->refcount);
+ deref_mad_agent(mad_agent_priv);
} else {
/* not user rmpp, revert to normal behavior and
* drop the mad */
@@ -1848,7 +1848,7 @@
&mad_agent_priv->agent,
&mad_send_wr->send_buf,
mad_recv_wc);
- atomic_dec(&mad_agent_priv->refcount);
+ deref_mad_agent(mad_agent_priv);
mad_send_wc.status = IB_WC_SUCCESS;
mad_send_wc.vendor_err = 0;
@@ -2438,7 +2438,7 @@
list_del(&mad_send_wr->agent_list);
mad_agent_priv->agent.send_handler(&mad_agent_priv->agent,
&mad_send_wc);
- atomic_dec(&mad_agent_priv->refcount);
+ deref_mad_agent(mad_agent_priv);
}
}
@@ -2572,7 +2572,7 @@
&local->mad_send_wr->send_buf,
&local->mad_priv->header.recv_wc);
spin_lock_irqsave(&recv_mad_agent->lock, flags);
- atomic_dec(&recv_mad_agent->refcount);
+ deref_mad_agent(recv_mad_agent);
spin_unlock_irqrestore(&recv_mad_agent->lock, flags);
}
@@ -2585,7 +2585,7 @@
&mad_send_wc);
spin_lock_irqsave(&mad_agent_priv->lock, flags);
- atomic_dec(&mad_agent_priv->refcount);
+ deref_mad_agent(mad_agent_priv);
if (free_mad)
kfree(local->mad_priv);
kfree(local);
@@ -2671,7 +2671,7 @@
mad_agent_priv->agent.send_handler(&mad_agent_priv->agent,
&mad_send_wc);
- atomic_dec(&mad_agent_priv->refcount);
+ deref_mad_agent(mad_agent_priv);
spin_lock_irqsave(&mad_agent_priv->lock, flags);
}
spin_unlock_irqrestore(&mad_agent_priv->lock, flags);
diff --git a/drivers/infiniband/core/mad_priv.h b/drivers/infiniband/core/mad_priv.h
index 403d867..4aa16b3 100644
--- a/drivers/infiniband/core/mad_priv.h
+++ b/drivers/infiniband/core/mad_priv.h
@@ -103,7 +103,7 @@
struct work_struct local_work;
struct list_head rmpp_list;
- atomic_t refcount;
+ refcount_t refcount;
union {
struct completion comp;
struct rcu_head rcu;
diff --git a/drivers/infiniband/core/mad_rmpp.c b/drivers/infiniband/core/mad_rmpp.c
index 5ec57ab..e0573e4 100644
--- a/drivers/infiniband/core/mad_rmpp.c
+++ b/drivers/infiniband/core/mad_rmpp.c
@@ -40,8 +40,7 @@
enum rmpp_state {
RMPP_STATE_ACTIVE,
RMPP_STATE_TIMEOUT,
- RMPP_STATE_COMPLETE,
- RMPP_STATE_CANCELING
+ RMPP_STATE_COMPLETE
};
struct mad_rmpp_recv {
@@ -52,7 +51,7 @@
struct completion comp;
enum rmpp_state state;
spinlock_t lock;
- atomic_t refcount;
+ refcount_t refcount;
struct ib_ah *ah;
struct ib_mad_recv_wc *rmpp_wc;
@@ -73,7 +72,7 @@
static inline void deref_rmpp_recv(struct mad_rmpp_recv *rmpp_recv)
{
- if (atomic_dec_and_test(&rmpp_recv->refcount))
+ if (refcount_dec_and_test(&rmpp_recv->refcount))
complete(&rmpp_recv->comp);
}
@@ -92,22 +91,18 @@
spin_lock_irqsave(&agent->lock, flags);
list_for_each_entry(rmpp_recv, &agent->rmpp_list, list) {
- if (rmpp_recv->state != RMPP_STATE_COMPLETE)
- ib_free_recv_mad(rmpp_recv->rmpp_wc);
- rmpp_recv->state = RMPP_STATE_CANCELING;
- }
- spin_unlock_irqrestore(&agent->lock, flags);
-
- list_for_each_entry(rmpp_recv, &agent->rmpp_list, list) {
cancel_delayed_work(&rmpp_recv->timeout_work);
cancel_delayed_work(&rmpp_recv->cleanup_work);
}
+ spin_unlock_irqrestore(&agent->lock, flags);
flush_workqueue(agent->qp_info->port_priv->wq);
list_for_each_entry_safe(rmpp_recv, temp_rmpp_recv,
&agent->rmpp_list, list) {
list_del(&rmpp_recv->list);
+ if (rmpp_recv->state != RMPP_STATE_COMPLETE)
+ ib_free_recv_mad(rmpp_recv->rmpp_wc);
destroy_rmpp_recv(rmpp_recv);
}
}
@@ -272,10 +267,6 @@
unsigned long flags;
spin_lock_irqsave(&rmpp_recv->agent->lock, flags);
- if (rmpp_recv->state == RMPP_STATE_CANCELING) {
- spin_unlock_irqrestore(&rmpp_recv->agent->lock, flags);
- return;
- }
list_del(&rmpp_recv->list);
spin_unlock_irqrestore(&rmpp_recv->agent->lock, flags);
destroy_rmpp_recv(rmpp_recv);
@@ -305,7 +296,7 @@
INIT_DELAYED_WORK(&rmpp_recv->cleanup_work, recv_cleanup_handler);
spin_lock_init(&rmpp_recv->lock);
rmpp_recv->state = RMPP_STATE_ACTIVE;
- atomic_set(&rmpp_recv->refcount, 1);
+ refcount_set(&rmpp_recv->refcount, 1);
rmpp_recv->rmpp_wc = mad_recv_wc;
rmpp_recv->cur_seg_buf = &mad_recv_wc->recv_buf;
@@ -357,7 +348,7 @@
spin_lock_irqsave(&agent->lock, flags);
rmpp_recv = find_rmpp_recv(agent, mad_recv_wc);
if (rmpp_recv)
- atomic_inc(&rmpp_recv->refcount);
+ refcount_inc(&rmpp_recv->refcount);
spin_unlock_irqrestore(&agent->lock, flags);
return rmpp_recv;
}
@@ -553,7 +544,7 @@
destroy_rmpp_recv(rmpp_recv);
return continue_rmpp(agent, mad_recv_wc);
}
- atomic_inc(&rmpp_recv->refcount);
+ refcount_inc(&rmpp_recv->refcount);
if (get_last_flag(&mad_recv_wc->recv_buf)) {
rmpp_recv->state = RMPP_STATE_COMPLETE;
diff --git a/drivers/infiniband/core/nldev.c b/drivers/infiniband/core/nldev.c
index e16105b..12d29d5 100644
--- a/drivers/infiniband/core/nldev.c
+++ b/drivers/infiniband/core/nldev.c
@@ -114,6 +114,7 @@
[RDMA_NLDEV_ATTR_RES_PS] = { .type = NLA_U32 },
[RDMA_NLDEV_ATTR_RES_QP] = { .type = NLA_NESTED },
[RDMA_NLDEV_ATTR_RES_QP_ENTRY] = { .type = NLA_NESTED },
+ [RDMA_NLDEV_ATTR_RES_RAW] = { .type = NLA_BINARY },
[RDMA_NLDEV_ATTR_RES_RKEY] = { .type = NLA_U32 },
[RDMA_NLDEV_ATTR_RES_RQPN] = { .type = NLA_U32 },
[RDMA_NLDEV_ATTR_RES_RQ_PSN] = { .type = NLA_U32 },
@@ -446,27 +447,11 @@
return err ? -EMSGSIZE : 0;
}
-static bool fill_res_entry(struct ib_device *dev, struct sk_buff *msg,
- struct rdma_restrack_entry *res)
+static int fill_res_qp_entry_query(struct sk_buff *msg,
+ struct rdma_restrack_entry *res,
+ struct ib_device *dev,
+ struct ib_qp *qp)
{
- if (!dev->ops.fill_res_entry)
- return false;
- return dev->ops.fill_res_entry(msg, res);
-}
-
-static bool fill_stat_entry(struct ib_device *dev, struct sk_buff *msg,
- struct rdma_restrack_entry *res)
-{
- if (!dev->ops.fill_stat_entry)
- return false;
- return dev->ops.fill_stat_entry(msg, res);
-}
-
-static int fill_res_qp_entry(struct sk_buff *msg, bool has_cap_net_admin,
- struct rdma_restrack_entry *res, uint32_t port)
-{
- struct ib_qp *qp = container_of(res, struct ib_qp, res);
- struct ib_device *dev = qp->device;
struct ib_qp_init_attr qp_init_attr;
struct ib_qp_attr qp_attr;
int ret;
@@ -475,16 +460,6 @@
if (ret)
return ret;
- if (port && port != qp_attr.port_num)
- return -EAGAIN;
-
- /* In create_qp() port is not set yet */
- if (qp_attr.port_num &&
- nla_put_u32(msg, RDMA_NLDEV_ATTR_PORT_INDEX, qp_attr.port_num))
- goto err;
-
- if (nla_put_u32(msg, RDMA_NLDEV_ATTR_RES_LQPN, qp->qp_num))
- goto err;
if (qp->qp_type == IB_QPT_RC || qp->qp_type == IB_QPT_UC) {
if (nla_put_u32(msg, RDMA_NLDEV_ATTR_RES_RQPN,
qp_attr.dest_qp_num))
@@ -508,21 +483,55 @@
if (nla_put_u8(msg, RDMA_NLDEV_ATTR_RES_STATE, qp_attr.qp_state))
goto err;
- if (!rdma_is_kernel_res(res) &&
- nla_put_u32(msg, RDMA_NLDEV_ATTR_RES_PDN, qp->pd->res.id))
- goto err;
-
- if (fill_res_name_pid(msg, res))
- goto err;
-
- if (fill_res_entry(dev, msg, res))
- goto err;
-
+ if (dev->ops.fill_res_qp_entry)
+ return dev->ops.fill_res_qp_entry(msg, qp);
return 0;
err: return -EMSGSIZE;
}
+static int fill_res_qp_entry(struct sk_buff *msg, bool has_cap_net_admin,
+ struct rdma_restrack_entry *res, uint32_t port)
+{
+ struct ib_qp *qp = container_of(res, struct ib_qp, res);
+ struct ib_device *dev = qp->device;
+ int ret;
+
+ if (port && port != qp->port)
+ return -EAGAIN;
+
+ /* In create_qp() port is not set yet */
+ if (qp->port && nla_put_u32(msg, RDMA_NLDEV_ATTR_PORT_INDEX, qp->port))
+ return -EINVAL;
+
+ ret = nla_put_u32(msg, RDMA_NLDEV_ATTR_RES_LQPN, qp->qp_num);
+ if (ret)
+ return -EMSGSIZE;
+
+ if (!rdma_is_kernel_res(res) &&
+ nla_put_u32(msg, RDMA_NLDEV_ATTR_RES_PDN, qp->pd->res.id))
+ return -EMSGSIZE;
+
+ ret = fill_res_name_pid(msg, res);
+ if (ret)
+ return -EMSGSIZE;
+
+ return fill_res_qp_entry_query(msg, res, dev, qp);
+}
+
+static int fill_res_qp_raw_entry(struct sk_buff *msg, bool has_cap_net_admin,
+ struct rdma_restrack_entry *res, uint32_t port)
+{
+ struct ib_qp *qp = container_of(res, struct ib_qp, res);
+ struct ib_device *dev = qp->device;
+
+ if (port && port != qp->port)
+ return -EAGAIN;
+ if (!dev->ops.fill_res_qp_entry_raw)
+ return -EINVAL;
+ return dev->ops.fill_res_qp_entry_raw(msg, qp);
+}
+
static int fill_res_cm_id_entry(struct sk_buff *msg, bool has_cap_net_admin,
struct rdma_restrack_entry *res, uint32_t port)
{
@@ -568,9 +577,8 @@
if (fill_res_name_pid(msg, res))
goto err;
- if (fill_res_entry(dev, msg, res))
- goto err;
-
+ if (dev->ops.fill_res_cm_id_entry)
+ return dev->ops.fill_res_cm_id_entry(msg, cm_id);
return 0;
err: return -EMSGSIZE;
@@ -583,35 +591,42 @@
struct ib_device *dev = cq->device;
if (nla_put_u32(msg, RDMA_NLDEV_ATTR_RES_CQE, cq->cqe))
- goto err;
+ return -EMSGSIZE;
if (nla_put_u64_64bit(msg, RDMA_NLDEV_ATTR_RES_USECNT,
atomic_read(&cq->usecnt), RDMA_NLDEV_ATTR_PAD))
- goto err;
+ return -EMSGSIZE;
/* Poll context is only valid for kernel CQs */
if (rdma_is_kernel_res(res) &&
nla_put_u8(msg, RDMA_NLDEV_ATTR_RES_POLL_CTX, cq->poll_ctx))
- goto err;
+ return -EMSGSIZE;
if (nla_put_u8(msg, RDMA_NLDEV_ATTR_DEV_DIM, (cq->dim != NULL)))
- goto err;
+ return -EMSGSIZE;
if (nla_put_u32(msg, RDMA_NLDEV_ATTR_RES_CQN, res->id))
- goto err;
+ return -EMSGSIZE;
if (!rdma_is_kernel_res(res) &&
nla_put_u32(msg, RDMA_NLDEV_ATTR_RES_CTXN,
cq->uobject->uevent.uobject.context->res.id))
- goto err;
+ return -EMSGSIZE;
if (fill_res_name_pid(msg, res))
- goto err;
+ return -EMSGSIZE;
- if (fill_res_entry(dev, msg, res))
- goto err;
+ return (dev->ops.fill_res_cq_entry) ?
+ dev->ops.fill_res_cq_entry(msg, cq) : 0;
+}
- return 0;
+static int fill_res_cq_raw_entry(struct sk_buff *msg, bool has_cap_net_admin,
+ struct rdma_restrack_entry *res, uint32_t port)
+{
+ struct ib_cq *cq = container_of(res, struct ib_cq, res);
+ struct ib_device *dev = cq->device;
-err: return -EMSGSIZE;
+ if (!dev->ops.fill_res_cq_entry_raw)
+ return -EINVAL;
+ return dev->ops.fill_res_cq_entry_raw(msg, cq);
}
static int fill_res_mr_entry(struct sk_buff *msg, bool has_cap_net_admin,
@@ -622,38 +637,45 @@
if (has_cap_net_admin) {
if (nla_put_u32(msg, RDMA_NLDEV_ATTR_RES_RKEY, mr->rkey))
- goto err;
+ return -EMSGSIZE;
if (nla_put_u32(msg, RDMA_NLDEV_ATTR_RES_LKEY, mr->lkey))
- goto err;
+ return -EMSGSIZE;
}
if (nla_put_u64_64bit(msg, RDMA_NLDEV_ATTR_RES_MRLEN, mr->length,
RDMA_NLDEV_ATTR_PAD))
- goto err;
+ return -EMSGSIZE;
if (nla_put_u32(msg, RDMA_NLDEV_ATTR_RES_MRN, res->id))
- goto err;
+ return -EMSGSIZE;
if (!rdma_is_kernel_res(res) &&
nla_put_u32(msg, RDMA_NLDEV_ATTR_RES_PDN, mr->pd->res.id))
- goto err;
+ return -EMSGSIZE;
if (fill_res_name_pid(msg, res))
- goto err;
+ return -EMSGSIZE;
- if (fill_res_entry(dev, msg, res))
- goto err;
+ return (dev->ops.fill_res_mr_entry) ?
+ dev->ops.fill_res_mr_entry(msg, mr) :
+ 0;
+}
- return 0;
+static int fill_res_mr_raw_entry(struct sk_buff *msg, bool has_cap_net_admin,
+ struct rdma_restrack_entry *res, uint32_t port)
+{
+ struct ib_mr *mr = container_of(res, struct ib_mr, res);
+ struct ib_device *dev = mr->pd->device;
-err: return -EMSGSIZE;
+ if (!dev->ops.fill_res_mr_entry_raw)
+ return -EINVAL;
+ return dev->ops.fill_res_mr_entry_raw(msg, mr);
}
static int fill_res_pd_entry(struct sk_buff *msg, bool has_cap_net_admin,
struct rdma_restrack_entry *res, uint32_t port)
{
struct ib_pd *pd = container_of(res, struct ib_pd, res);
- struct ib_device *dev = pd->device;
if (has_cap_net_admin) {
if (nla_put_u32(msg, RDMA_NLDEV_ATTR_RES_LOCAL_DMA_LKEY,
@@ -676,13 +698,7 @@
pd->uobject->context->res.id))
goto err;
- if (fill_res_name_pid(msg, res))
- goto err;
-
- if (fill_res_entry(dev, msg, res))
- goto err;
-
- return 0;
+ return fill_res_name_pid(msg, res);
err: return -EMSGSIZE;
}
@@ -695,11 +711,16 @@
if (nla_put_u32(msg, RDMA_NLDEV_ATTR_STAT_MODE, m->mode))
return -EMSGSIZE;
- if (m->mode == RDMA_COUNTER_MODE_AUTO)
+ if (m->mode == RDMA_COUNTER_MODE_AUTO) {
if ((m->mask & RDMA_COUNTER_MASK_QP_TYPE) &&
nla_put_u8(msg, RDMA_NLDEV_ATTR_RES_TYPE, m->param.qp_type))
return -EMSGSIZE;
+ if ((m->mask & RDMA_COUNTER_MASK_PID) &&
+ fill_res_name_pid(msg, &counter->res))
+ return -EMSGSIZE;
+ }
+
return 0;
}
@@ -738,9 +759,6 @@
xa_lock(&rt->xa);
xa_for_each(&rt->xa, id, res) {
qp = container_of(res, struct ib_qp, res);
- if (qp->qp_type == IB_QPT_RAW_PACKET && !capable(CAP_NET_RAW))
- continue;
-
if (!qp->counter || (qp->counter->id != counter->id))
continue;
@@ -793,9 +811,8 @@
if (nla_put_u32(msg, RDMA_NLDEV_ATTR_RES_MRN, res->id))
goto err;
- if (fill_stat_entry(dev, msg, res))
- goto err;
-
+ if (dev->ops.fill_stat_mr_entry)
+ return dev->ops.fill_stat_mr_entry(msg, mr);
return 0;
err:
@@ -840,7 +857,6 @@
if (nla_put_u32(msg, RDMA_NLDEV_ATTR_PORT_INDEX, counter->port) ||
nla_put_u32(msg, RDMA_NLDEV_ATTR_STAT_COUNTER_ID, counter->id) ||
- fill_res_name_pid(msg, &counter->res) ||
fill_stat_counter_mode(msg, counter) ||
fill_stat_counter_qps(msg, counter) ||
fill_stat_counter_hwcounters(msg, counter))
@@ -1177,7 +1193,6 @@
struct nldev_fill_res_entry {
enum rdma_nldev_attr nldev_attr;
- enum rdma_nldev_command nldev_cmd;
u8 flags;
u32 entry;
u32 id;
@@ -1189,40 +1204,34 @@
static const struct nldev_fill_res_entry fill_entries[RDMA_RESTRACK_MAX] = {
[RDMA_RESTRACK_QP] = {
- .nldev_cmd = RDMA_NLDEV_CMD_RES_QP_GET,
.nldev_attr = RDMA_NLDEV_ATTR_RES_QP,
.entry = RDMA_NLDEV_ATTR_RES_QP_ENTRY,
.id = RDMA_NLDEV_ATTR_RES_LQPN,
},
[RDMA_RESTRACK_CM_ID] = {
- .nldev_cmd = RDMA_NLDEV_CMD_RES_CM_ID_GET,
.nldev_attr = RDMA_NLDEV_ATTR_RES_CM_ID,
.entry = RDMA_NLDEV_ATTR_RES_CM_ID_ENTRY,
.id = RDMA_NLDEV_ATTR_RES_CM_IDN,
},
[RDMA_RESTRACK_CQ] = {
- .nldev_cmd = RDMA_NLDEV_CMD_RES_CQ_GET,
.nldev_attr = RDMA_NLDEV_ATTR_RES_CQ,
.flags = NLDEV_PER_DEV,
.entry = RDMA_NLDEV_ATTR_RES_CQ_ENTRY,
.id = RDMA_NLDEV_ATTR_RES_CQN,
},
[RDMA_RESTRACK_MR] = {
- .nldev_cmd = RDMA_NLDEV_CMD_RES_MR_GET,
.nldev_attr = RDMA_NLDEV_ATTR_RES_MR,
.flags = NLDEV_PER_DEV,
.entry = RDMA_NLDEV_ATTR_RES_MR_ENTRY,
.id = RDMA_NLDEV_ATTR_RES_MRN,
},
[RDMA_RESTRACK_PD] = {
- .nldev_cmd = RDMA_NLDEV_CMD_RES_PD_GET,
.nldev_attr = RDMA_NLDEV_ATTR_RES_PD,
.flags = NLDEV_PER_DEV,
.entry = RDMA_NLDEV_ATTR_RES_PD_ENTRY,
.id = RDMA_NLDEV_ATTR_RES_PDN,
},
[RDMA_RESTRACK_COUNTER] = {
- .nldev_cmd = RDMA_NLDEV_CMD_STAT_GET,
.nldev_attr = RDMA_NLDEV_ATTR_STAT_COUNTER,
.entry = RDMA_NLDEV_ATTR_STAT_COUNTER_ENTRY,
.id = RDMA_NLDEV_ATTR_STAT_COUNTER_ID,
@@ -1281,7 +1290,8 @@
}
nlh = nlmsg_put(msg, NETLINK_CB(skb).portid, nlh->nlmsg_seq,
- RDMA_NL_GET_TYPE(RDMA_NL_NLDEV, fe->nldev_cmd),
+ RDMA_NL_GET_TYPE(RDMA_NL_NLDEV,
+ RDMA_NL_GET_OP(nlh->nlmsg_type)),
0, 0);
if (fill_nldev_handle(msg, device)) {
@@ -1359,7 +1369,8 @@
}
nlh = nlmsg_put(skb, NETLINK_CB(cb->skb).portid, cb->nlh->nlmsg_seq,
- RDMA_NL_GET_TYPE(RDMA_NL_NLDEV, fe->nldev_cmd),
+ RDMA_NL_GET_TYPE(RDMA_NL_NLDEV,
+ RDMA_NL_GET_OP(cb->nlh->nlmsg_type)),
0, NLM_F_MULTI);
if (fill_nldev_handle(skb, device)) {
@@ -1441,26 +1452,29 @@
return ret;
}
-#define RES_GET_FUNCS(name, type) \
- static int nldev_res_get_##name##_dumpit(struct sk_buff *skb, \
+#define RES_GET_FUNCS(name, type) \
+ static int nldev_res_get_##name##_dumpit(struct sk_buff *skb, \
struct netlink_callback *cb) \
- { \
- return res_get_common_dumpit(skb, cb, type, \
- fill_res_##name##_entry); \
- } \
- static int nldev_res_get_##name##_doit(struct sk_buff *skb, \
- struct nlmsghdr *nlh, \
+ { \
+ return res_get_common_dumpit(skb, cb, type, \
+ fill_res_##name##_entry); \
+ } \
+ static int nldev_res_get_##name##_doit(struct sk_buff *skb, \
+ struct nlmsghdr *nlh, \
struct netlink_ext_ack *extack) \
- { \
- return res_get_common_doit(skb, nlh, extack, type, \
- fill_res_##name##_entry); \
+ { \
+ return res_get_common_doit(skb, nlh, extack, type, \
+ fill_res_##name##_entry); \
}
RES_GET_FUNCS(qp, RDMA_RESTRACK_QP);
+RES_GET_FUNCS(qp_raw, RDMA_RESTRACK_QP);
RES_GET_FUNCS(cm_id, RDMA_RESTRACK_CM_ID);
RES_GET_FUNCS(cq, RDMA_RESTRACK_CQ);
+RES_GET_FUNCS(cq_raw, RDMA_RESTRACK_CQ);
RES_GET_FUNCS(pd, RDMA_RESTRACK_PD);
RES_GET_FUNCS(mr, RDMA_RESTRACK_MR);
+RES_GET_FUNCS(mr_raw, RDMA_RESTRACK_MR);
RES_GET_FUNCS(counter, RDMA_RESTRACK_COUNTER);
static LIST_HEAD(link_ops);
@@ -2145,6 +2159,21 @@
.doit = nldev_stat_del_doit,
.flags = RDMA_NL_ADMIN_PERM,
},
+ [RDMA_NLDEV_CMD_RES_QP_GET_RAW] = {
+ .doit = nldev_res_get_qp_raw_doit,
+ .dump = nldev_res_get_qp_raw_dumpit,
+ .flags = RDMA_NL_ADMIN_PERM,
+ },
+ [RDMA_NLDEV_CMD_RES_CQ_GET_RAW] = {
+ .doit = nldev_res_get_cq_raw_doit,
+ .dump = nldev_res_get_cq_raw_dumpit,
+ .flags = RDMA_NL_ADMIN_PERM,
+ },
+ [RDMA_NLDEV_CMD_RES_MR_GET_RAW] = {
+ .doit = nldev_res_get_mr_raw_doit,
+ .dump = nldev_res_get_mr_raw_dumpit,
+ .flags = RDMA_NL_ADMIN_PERM,
+ },
};
void __init nldev_init(void)
diff --git a/drivers/infiniband/core/sysfs.c b/drivers/infiniband/core/sysfs.c
index defe9cd..c11e505 100644
--- a/drivers/infiniband/core/sysfs.c
+++ b/drivers/infiniband/core/sysfs.c
@@ -58,7 +58,7 @@
struct ib_device *ibdev;
struct gid_attr_group *gid_attr_group;
struct attribute_group gid_group;
- struct attribute_group pkey_group;
+ struct attribute_group *pkey_group;
struct attribute_group *pma_table;
struct attribute_group *hw_stats_ag;
struct rdma_hw_stats *hw_stats;
@@ -681,11 +681,16 @@
kfree(p->gid_group.attrs);
}
- if (p->pkey_group.attrs) {
- for (i = 0; (a = p->pkey_group.attrs[i]); ++i)
- kfree(a);
+ if (p->pkey_group) {
+ if (p->pkey_group->attrs) {
+ for (i = 0; (a = p->pkey_group->attrs[i]); ++i)
+ kfree(a);
- kfree(p->pkey_group.attrs);
+ kfree(p->pkey_group->attrs);
+ }
+
+ kfree(p->pkey_group);
+ p->pkey_group = NULL;
}
kfree(p);
@@ -1118,17 +1123,26 @@
if (ret)
goto err_free_gid_type;
- p->pkey_group.name = "pkeys";
- p->pkey_group.attrs = alloc_group_attrs(show_port_pkey,
- attr.pkey_tbl_len);
- if (!p->pkey_group.attrs) {
- ret = -ENOMEM;
- goto err_remove_gid_type;
+ if (attr.pkey_tbl_len) {
+ p->pkey_group = kzalloc(sizeof(*p->pkey_group), GFP_KERNEL);
+ if (!p->pkey_group) {
+ ret = -ENOMEM;
+ goto err_remove_gid_type;
+ }
+
+ p->pkey_group->name = "pkeys";
+ p->pkey_group->attrs = alloc_group_attrs(show_port_pkey,
+ attr.pkey_tbl_len);
+ if (!p->pkey_group->attrs) {
+ ret = -ENOMEM;
+ goto err_free_pkey_group;
+ }
+
+ ret = sysfs_create_group(&p->kobj, p->pkey_group);
+ if (ret)
+ goto err_free_pkey;
}
- ret = sysfs_create_group(&p->kobj, &p->pkey_group);
- if (ret)
- goto err_free_pkey;
if (device->ops.init_port && is_full_dev) {
ret = device->ops.init_port(device, port_num, &p->kobj);
@@ -1150,14 +1164,20 @@
return 0;
err_remove_pkey:
- sysfs_remove_group(&p->kobj, &p->pkey_group);
+ if (p->pkey_group)
+ sysfs_remove_group(&p->kobj, p->pkey_group);
err_free_pkey:
- for (i = 0; i < attr.pkey_tbl_len; ++i)
- kfree(p->pkey_group.attrs[i]);
+ if (p->pkey_group) {
+ for (i = 0; i < attr.pkey_tbl_len; ++i)
+ kfree(p->pkey_group->attrs[i]);
- kfree(p->pkey_group.attrs);
- p->pkey_group.attrs = NULL;
+ kfree(p->pkey_group->attrs);
+ p->pkey_group->attrs = NULL;
+ }
+
+err_free_pkey_group:
+ kfree(p->pkey_group);
err_remove_gid_type:
sysfs_remove_group(&p->gid_attr_group->kobj,
@@ -1317,7 +1337,8 @@
if (port->pma_table)
sysfs_remove_group(p, port->pma_table);
- sysfs_remove_group(p, &port->pkey_group);
+ if (port->pkey_group)
+ sysfs_remove_group(p, port->pkey_group);
sysfs_remove_group(p, &port->gid_group);
sysfs_remove_group(&port->gid_attr_group->kobj,
&port->gid_attr_group->ndev);
diff --git a/drivers/infiniband/core/trace.c b/drivers/infiniband/core/trace.c
index 6c3514b..31e7860 100644
--- a/drivers/infiniband/core/trace.c
+++ b/drivers/infiniband/core/trace.c
@@ -9,6 +9,4 @@
#define CREATE_TRACE_POINTS
-#include <rdma/ib_verbs.h>
-
#include <trace/events/rdma_core.h>
diff --git a/drivers/infiniband/core/umem.c b/drivers/infiniband/core/umem.c
index 82455a1..831bff8 100644
--- a/drivers/infiniband/core/umem.c
+++ b/drivers/infiniband/core/umem.c
@@ -261,6 +261,7 @@
sg = umem->sg_head.sgl;
while (npages) {
+ cond_resched();
ret = pin_user_pages_fast(cur_base,
min_t(unsigned long, npages,
PAGE_SIZE /
diff --git a/drivers/infiniband/core/umem_odp.c b/drivers/infiniband/core/umem_odp.c
index ccd2840..5e32f61 100644
--- a/drivers/infiniband/core/umem_odp.c
+++ b/drivers/infiniband/core/umem_odp.c
@@ -152,6 +152,7 @@
* ib_alloc_implicit_odp_umem()
* @addr: The starting userspace VA
* @size: The length of the userspace VA
+ * @ops: MMU interval ops, currently only @invalidate
*/
struct ib_umem_odp *
ib_umem_odp_alloc_child(struct ib_umem_odp *root, unsigned long addr,
@@ -213,6 +214,7 @@
* @addr: userspace virtual address to start at
* @size: length of region to pin
* @access: IB_ACCESS_xxx flags for memory being pinned
+ * @ops: MMU interval ops, currently only @invalidate
*
* The driver should use when the access flags indicate ODP memory. It avoids
* pinning, instead, stores the mm for future page fault handling in
diff --git a/drivers/infiniband/core/uverbs_cmd.c b/drivers/infiniband/core/uverbs_cmd.c
index 76e7ec0..2fbc583 100644
--- a/drivers/infiniband/core/uverbs_cmd.c
+++ b/drivers/infiniband/core/uverbs_cmd.c
@@ -415,8 +415,8 @@
static int ib_uverbs_alloc_pd(struct uverbs_attr_bundle *attrs)
{
+ struct ib_uverbs_alloc_pd_resp resp = {};
struct ib_uverbs_alloc_pd cmd;
- struct ib_uverbs_alloc_pd_resp resp;
struct ib_uobject *uobj;
struct ib_pd *pd;
int ret;
@@ -438,29 +438,20 @@
pd->device = ib_dev;
pd->uobject = uobj;
- pd->__internal_mr = NULL;
atomic_set(&pd->usecnt, 0);
pd->res.type = RDMA_RESTRACK_PD;
ret = ib_dev->ops.alloc_pd(pd, &attrs->driver_udata);
if (ret)
goto err_alloc;
-
- uobj->object = pd;
- memset(&resp, 0, sizeof resp);
- resp.pd_handle = uobj->id;
rdma_restrack_uadd(&pd->res);
- ret = uverbs_response(attrs, &resp, sizeof(resp));
- if (ret)
- goto err_copy;
+ uobj->object = pd;
+ uobj_finalize_uobj_create(uobj, attrs);
- rdma_alloc_commit_uobject(uobj, attrs);
- return 0;
+ resp.pd_handle = uobj->id;
+ return uverbs_response(attrs, &resp, sizeof(resp));
-err_copy:
- ib_dealloc_pd_user(pd, uverbs_get_cleared_udata(attrs));
- pd = NULL;
err_alloc:
kfree(pd);
err:
@@ -568,15 +559,15 @@
static int ib_uverbs_open_xrcd(struct uverbs_attr_bundle *attrs)
{
struct ib_uverbs_device *ibudev = attrs->ufile->device;
+ struct ib_uverbs_open_xrcd_resp resp = {};
struct ib_uverbs_open_xrcd cmd;
- struct ib_uverbs_open_xrcd_resp resp;
struct ib_uxrcd_object *obj;
struct ib_xrcd *xrcd = NULL;
- struct fd f = {NULL, 0};
struct inode *inode = NULL;
- int ret = 0;
int new_xrcd = 0;
struct ib_device *ib_dev;
+ struct fd f = {};
+ int ret;
ret = uverbs_request(attrs, &cmd, sizeof(cmd));
if (ret)
@@ -614,24 +605,16 @@
}
if (!xrcd) {
- xrcd = ib_dev->ops.alloc_xrcd(ib_dev, &attrs->driver_udata);
+ xrcd = ib_alloc_xrcd_user(ib_dev, inode, &attrs->driver_udata);
if (IS_ERR(xrcd)) {
ret = PTR_ERR(xrcd);
goto err;
}
-
- xrcd->inode = inode;
- xrcd->device = ib_dev;
- atomic_set(&xrcd->usecnt, 0);
- mutex_init(&xrcd->tgt_qp_mutex);
- INIT_LIST_HEAD(&xrcd->tgt_qp_list);
new_xrcd = 1;
}
atomic_set(&obj->refcnt, 0);
obj->uobject.object = xrcd;
- memset(&resp, 0, sizeof resp);
- resp.xrcd_handle = obj->uobject.id;
if (inode) {
if (new_xrcd) {
@@ -643,27 +626,17 @@
atomic_inc(&xrcd->usecnt);
}
- ret = uverbs_response(attrs, &resp, sizeof(resp));
- if (ret)
- goto err_copy;
-
if (f.file)
fdput(f);
mutex_unlock(&ibudev->xrcd_tree_mutex);
+ uobj_finalize_uobj_create(&obj->uobject, attrs);
- rdma_alloc_commit_uobject(&obj->uobject, attrs);
- return 0;
-
-err_copy:
- if (inode) {
- if (new_xrcd)
- xrcd_table_delete(ibudev, inode);
- atomic_dec(&xrcd->usecnt);
- }
+ resp.xrcd_handle = obj->uobject.id;
+ return uverbs_response(attrs, &resp, sizeof(resp));
err_dealloc_xrcd:
- ib_dealloc_xrcd(xrcd, uverbs_get_cleared_udata(attrs));
+ ib_dealloc_xrcd_user(xrcd, uverbs_get_cleared_udata(attrs));
err:
uobj_alloc_abort(&obj->uobject, attrs);
@@ -701,7 +674,7 @@
if (inode && !atomic_dec_and_test(&xrcd->usecnt))
return 0;
- ret = ib_dealloc_xrcd(xrcd, &attrs->driver_udata);
+ ret = ib_dealloc_xrcd_user(xrcd, &attrs->driver_udata);
if (ib_is_destroy_retryable(ret, why, uobject)) {
atomic_inc(&xrcd->usecnt);
@@ -716,8 +689,8 @@
static int ib_uverbs_reg_mr(struct uverbs_attr_bundle *attrs)
{
+ struct ib_uverbs_reg_mr_resp resp = {};
struct ib_uverbs_reg_mr cmd;
- struct ib_uverbs_reg_mr_resp resp;
struct ib_uobject *uobj;
struct ib_pd *pd;
struct ib_mr *mr;
@@ -770,30 +743,20 @@
mr->uobject = uobj;
atomic_inc(&pd->usecnt);
mr->res.type = RDMA_RESTRACK_MR;
+ mr->iova = cmd.hca_va;
rdma_restrack_uadd(&mr->res);
uobj->object = mr;
-
- memset(&resp, 0, sizeof resp);
- resp.lkey = mr->lkey;
- resp.rkey = mr->rkey;
- resp.mr_handle = uobj->id;
-
- ret = uverbs_response(attrs, &resp, sizeof(resp));
- if (ret)
- goto err_copy;
-
uobj_put_obj_read(pd);
+ uobj_finalize_uobj_create(uobj, attrs);
- rdma_alloc_commit_uobject(uobj, attrs);
- return 0;
-
-err_copy:
- ib_dereg_mr_user(mr, uverbs_get_cleared_udata(attrs));
+ resp.lkey = mr->lkey;
+ resp.rkey = mr->rkey;
+ resp.mr_handle = uobj->id;
+ return uverbs_response(attrs, &resp, sizeof(resp));
err_put:
uobj_put_obj_read(pd);
-
err_free:
uobj_alloc_abort(uobj, attrs);
return ret;
@@ -861,6 +824,9 @@
atomic_dec(&old_pd->usecnt);
}
+ if (cmd.flags & IB_MR_REREG_TRANS)
+ mr->iova = cmd.hca_va;
+
memset(&resp, 0, sizeof(resp));
resp.lkey = mr->lkey;
resp.rkey = mr->rkey;
@@ -930,21 +896,13 @@
atomic_inc(&pd->usecnt);
uobj->object = mw;
-
- memset(&resp, 0, sizeof(resp));
- resp.rkey = mw->rkey;
- resp.mw_handle = uobj->id;
-
- ret = uverbs_response(attrs, &resp, sizeof(resp));
- if (ret)
- goto err_copy;
-
uobj_put_obj_read(pd);
- rdma_alloc_commit_uobject(uobj, attrs);
- return 0;
+ uobj_finalize_uobj_create(uobj, attrs);
-err_copy:
- uverbs_dealloc_mw(mw);
+ resp.rkey = mw->rkey;
+ resp.mw_handle = uobj->id;
+ return uverbs_response(attrs, &resp, sizeof(resp));
+
err_put:
uobj_put_obj_read(pd);
err_free:
@@ -981,40 +939,33 @@
if (IS_ERR(uobj))
return PTR_ERR(uobj);
- resp.fd = uobj->id;
-
ev_file = container_of(uobj, struct ib_uverbs_completion_event_file,
uobj);
ib_uverbs_init_event_queue(&ev_file->ev_queue);
+ uobj_finalize_uobj_create(uobj, attrs);
- ret = uverbs_response(attrs, &resp, sizeof(resp));
- if (ret) {
- uobj_alloc_abort(uobj, attrs);
- return ret;
- }
-
- rdma_alloc_commit_uobject(uobj, attrs);
- return 0;
+ resp.fd = uobj->id;
+ return uverbs_response(attrs, &resp, sizeof(resp));
}
-static struct ib_ucq_object *create_cq(struct uverbs_attr_bundle *attrs,
- struct ib_uverbs_ex_create_cq *cmd)
+static int create_cq(struct uverbs_attr_bundle *attrs,
+ struct ib_uverbs_ex_create_cq *cmd)
{
struct ib_ucq_object *obj;
struct ib_uverbs_completion_event_file *ev_file = NULL;
struct ib_cq *cq;
int ret;
- struct ib_uverbs_ex_create_cq_resp resp;
+ struct ib_uverbs_ex_create_cq_resp resp = {};
struct ib_cq_init_attr attr = {};
struct ib_device *ib_dev;
if (cmd->comp_vector >= attrs->ufile->device->num_comp_vectors)
- return ERR_PTR(-EINVAL);
+ return -EINVAL;
obj = (struct ib_ucq_object *)uobj_alloc(UVERBS_OBJECT_CQ, attrs,
&ib_dev);
if (IS_ERR(obj))
- return obj;
+ return PTR_ERR(obj);
if (cmd->comp_channel >= 0) {
ev_file = ib_uverbs_lookup_comp_file(cmd->comp_channel, attrs);
@@ -1043,53 +994,38 @@
cq->event_handler = ib_uverbs_cq_event_handler;
cq->cq_context = ev_file ? &ev_file->ev_queue : NULL;
atomic_set(&cq->usecnt, 0);
+ cq->res.type = RDMA_RESTRACK_CQ;
ret = ib_dev->ops.create_cq(cq, &attr, &attrs->driver_udata);
if (ret)
goto err_free;
+ rdma_restrack_uadd(&cq->res);
obj->uevent.uobject.object = cq;
obj->uevent.event_file = READ_ONCE(attrs->ufile->default_async_file);
if (obj->uevent.event_file)
uverbs_uobject_get(&obj->uevent.event_file->uobj);
+ uobj_finalize_uobj_create(&obj->uevent.uobject, attrs);
- memset(&resp, 0, sizeof resp);
resp.base.cq_handle = obj->uevent.uobject.id;
- resp.base.cqe = cq->cqe;
+ resp.base.cqe = cq->cqe;
resp.response_length = uverbs_response_length(attrs, sizeof(resp));
+ return uverbs_response(attrs, &resp, sizeof(resp));
- cq->res.type = RDMA_RESTRACK_CQ;
- rdma_restrack_uadd(&cq->res);
-
- ret = uverbs_response(attrs, &resp, sizeof(resp));
- if (ret)
- goto err_cb;
-
- rdma_alloc_commit_uobject(&obj->uevent.uobject, attrs);
- return obj;
-
-err_cb:
- if (obj->uevent.event_file)
- uverbs_uobject_put(&obj->uevent.event_file->uobj);
- ib_destroy_cq_user(cq, uverbs_get_cleared_udata(attrs));
- cq = NULL;
err_free:
kfree(cq);
err_file:
if (ev_file)
ib_uverbs_release_ucq(ev_file, obj);
-
err:
uobj_alloc_abort(&obj->uevent.uobject, attrs);
-
- return ERR_PTR(ret);
+ return ret;
}
static int ib_uverbs_create_cq(struct uverbs_attr_bundle *attrs)
{
struct ib_uverbs_create_cq cmd;
struct ib_uverbs_ex_create_cq cmd_ex;
- struct ib_ucq_object *obj;
int ret;
ret = uverbs_request(attrs, &cmd, sizeof(cmd));
@@ -1102,14 +1038,12 @@
cmd_ex.comp_vector = cmd.comp_vector;
cmd_ex.comp_channel = cmd.comp_channel;
- obj = create_cq(attrs, &cmd_ex);
- return PTR_ERR_OR_ZERO(obj);
+ return create_cq(attrs, &cmd_ex);
}
static int ib_uverbs_ex_create_cq(struct uverbs_attr_bundle *attrs)
{
struct ib_uverbs_ex_create_cq cmd;
- struct ib_ucq_object *obj;
int ret;
ret = uverbs_request(attrs, &cmd, sizeof(cmd));
@@ -1122,8 +1056,7 @@
if (cmd.reserved)
return -EINVAL;
- obj = create_cq(attrs, &cmd);
- return PTR_ERR_OR_ZERO(obj);
+ return create_cq(attrs, &cmd);
}
static int ib_uverbs_resize_cq(struct uverbs_attr_bundle *attrs)
@@ -1131,7 +1064,7 @@
struct ib_uverbs_resize_cq cmd;
struct ib_uverbs_resize_cq_resp resp = {};
struct ib_cq *cq;
- int ret = -EINVAL;
+ int ret;
ret = uverbs_request(attrs, &cmd, sizeof(cmd));
if (ret)
@@ -1298,7 +1231,7 @@
struct ib_srq *srq = NULL;
struct ib_qp *qp;
struct ib_qp_init_attr attr = {};
- struct ib_uverbs_ex_create_qp_resp resp;
+ struct ib_uverbs_ex_create_qp_resp resp = {};
int ret;
struct ib_rwq_ind_table *ind_tbl = NULL;
bool has_sq = true;
@@ -1468,20 +1401,6 @@
if (obj->uevent.event_file)
uverbs_uobject_get(&obj->uevent.event_file->uobj);
- memset(&resp, 0, sizeof resp);
- resp.base.qpn = qp->qp_num;
- resp.base.qp_handle = obj->uevent.uobject.id;
- resp.base.max_recv_sge = attr.cap.max_recv_sge;
- resp.base.max_send_sge = attr.cap.max_send_sge;
- resp.base.max_recv_wr = attr.cap.max_recv_wr;
- resp.base.max_send_wr = attr.cap.max_send_wr;
- resp.base.max_inline_data = attr.cap.max_inline_data;
- resp.response_length = uverbs_response_length(attrs, sizeof(resp));
-
- ret = uverbs_response(attrs, &resp, sizeof(resp));
- if (ret)
- goto err_uevent;
-
if (xrcd) {
obj->uxrcd = container_of(xrcd_uobj, struct ib_uxrcd_object,
uobject);
@@ -1502,12 +1421,18 @@
UVERBS_LOOKUP_READ);
if (ind_tbl)
uobj_put_obj_read(ind_tbl);
+ uobj_finalize_uobj_create(&obj->uevent.uobject, attrs);
- rdma_alloc_commit_uobject(&obj->uevent.uobject, attrs);
- return 0;
-err_uevent:
- if (obj->uevent.event_file)
- uverbs_uobject_put(&obj->uevent.event_file->uobj);
+ resp.base.qpn = qp->qp_num;
+ resp.base.qp_handle = obj->uevent.uobject.id;
+ resp.base.max_recv_sge = attr.cap.max_recv_sge;
+ resp.base.max_send_sge = attr.cap.max_send_sge;
+ resp.base.max_recv_wr = attr.cap.max_recv_wr;
+ resp.base.max_send_wr = attr.cap.max_send_wr;
+ resp.base.max_inline_data = attr.cap.max_inline_data;
+ resp.response_length = uverbs_response_length(attrs, sizeof(resp));
+ return uverbs_response(attrs, &resp, sizeof(resp));
+
err_cb:
ib_destroy_qp_user(qp, uverbs_get_cleared_udata(attrs));
@@ -1580,14 +1505,14 @@
static int ib_uverbs_open_qp(struct uverbs_attr_bundle *attrs)
{
+ struct ib_uverbs_create_qp_resp resp = {};
struct ib_uverbs_open_qp cmd;
- struct ib_uverbs_create_qp_resp resp;
struct ib_uqp_object *obj;
struct ib_xrcd *xrcd;
- struct ib_uobject *xrcd_uobj;
struct ib_qp *qp;
struct ib_qp_open_attr attr = {};
int ret;
+ struct ib_uobject *xrcd_uobj;
struct ib_device *ib_dev;
ret = uverbs_request(attrs, &cmd, sizeof(cmd));
@@ -1627,24 +1552,16 @@
obj->uevent.uobject.object = qp;
obj->uevent.uobject.user_handle = cmd.user_handle;
- memset(&resp, 0, sizeof resp);
- resp.qpn = qp->qp_num;
- resp.qp_handle = obj->uevent.uobject.id;
-
- ret = uverbs_response(attrs, &resp, sizeof(resp));
- if (ret)
- goto err_destroy;
-
obj->uxrcd = container_of(xrcd_uobj, struct ib_uxrcd_object, uobject);
atomic_inc(&obj->uxrcd->refcnt);
qp->uobject = obj;
uobj_put_read(xrcd_uobj);
+ uobj_finalize_uobj_create(&obj->uevent.uobject, attrs);
- rdma_alloc_commit_uobject(&obj->uevent.uobject, attrs);
- return 0;
+ resp.qpn = qp->qp_num;
+ resp.qp_handle = obj->uevent.uobject.id;
+ return uverbs_response(attrs, &resp, sizeof(resp));
-err_destroy:
- ib_destroy_qp_user(qp, uverbs_get_cleared_udata(attrs));
err_xrcd:
uobj_put_read(xrcd_uobj);
err_put:
@@ -1980,7 +1897,7 @@
* Last bit is reserved for extending the attr_mask by
* using another field.
*/
- BUILD_BUG_ON(IB_USER_LAST_QP_ATTR_MASK == (1 << 31));
+ BUILD_BUG_ON(IB_USER_LAST_QP_ATTR_MASK == (1ULL << 31));
if (cmd.base.attr_mask &
~((IB_USER_LAST_QP_ATTR_MASK << 1) - 1))
@@ -2480,24 +2397,14 @@
ah->uobject = uobj;
uobj->user_handle = cmd.user_handle;
uobj->object = ah;
+ uobj_put_obj_read(pd);
+ uobj_finalize_uobj_create(uobj, attrs);
resp.ah_handle = uobj->id;
-
- ret = uverbs_response(attrs, &resp, sizeof(resp));
- if (ret)
- goto err_copy;
-
- uobj_put_obj_read(pd);
- rdma_alloc_commit_uobject(uobj, attrs);
- return 0;
-
-err_copy:
- rdma_destroy_ah_user(ah, RDMA_DESTROY_AH_SLEEPABLE,
- uverbs_get_cleared_udata(attrs));
+ return uverbs_response(attrs, &resp, sizeof(resp));
err_put:
uobj_put_obj_read(pd);
-
err:
uobj_alloc_abort(uobj, attrs);
return ret;
@@ -2989,26 +2896,18 @@
if (obj->uevent.event_file)
uverbs_uobject_get(&obj->uevent.event_file->uobj);
- memset(&resp, 0, sizeof(resp));
+ uobj_put_obj_read(pd);
+ rdma_lookup_put_uobject(&cq->uobject->uevent.uobject,
+ UVERBS_LOOKUP_READ);
+ uobj_finalize_uobj_create(&obj->uevent.uobject, attrs);
+
resp.wq_handle = obj->uevent.uobject.id;
resp.max_sge = wq_init_attr.max_sge;
resp.max_wr = wq_init_attr.max_wr;
resp.wqn = wq->wq_num;
resp.response_length = uverbs_response_length(attrs, sizeof(resp));
- err = uverbs_response(attrs, &resp, sizeof(resp));
- if (err)
- goto err_copy;
+ return uverbs_response(attrs, &resp, sizeof(resp));
- uobj_put_obj_read(pd);
- rdma_lookup_put_uobject(&cq->uobject->uevent.uobject,
- UVERBS_LOOKUP_READ);
- rdma_alloc_commit_uobject(&obj->uevent.uobject, attrs);
- return 0;
-
-err_copy:
- if (obj->uevent.event_file)
- uverbs_uobject_put(&obj->uevent.event_file->uobj);
- ib_destroy_wq(wq, uverbs_get_cleared_udata(attrs));
err_put_cq:
rdma_lookup_put_uobject(&cq->uobject->uevent.uobject,
UVERBS_LOOKUP_READ);
@@ -3093,7 +2992,7 @@
struct ib_wq **wqs = NULL;
u32 *wqs_handles = NULL;
struct ib_wq *wq = NULL;
- int i, j, num_read_wqs;
+ int i, num_read_wqs;
u32 num_wq_handles;
struct uverbs_req_iter iter;
struct ib_device *ib_dev;
@@ -3139,6 +3038,7 @@
}
wqs[num_read_wqs] = wq;
+ atomic_inc(&wqs[num_read_wqs]->usecnt);
}
uobj = uobj_alloc(UVERBS_OBJECT_RWQ_IND_TBL, attrs, &ib_dev);
@@ -3166,33 +3066,24 @@
atomic_set(&rwq_ind_tbl->usecnt, 0);
for (i = 0; i < num_wq_handles; i++)
- atomic_inc(&wqs[i]->usecnt);
+ rdma_lookup_put_uobject(&wqs[i]->uobject->uevent.uobject,
+ UVERBS_LOOKUP_READ);
+ kfree(wqs_handles);
+ uobj_finalize_uobj_create(uobj, attrs);
resp.ind_tbl_handle = uobj->id;
resp.ind_tbl_num = rwq_ind_tbl->ind_tbl_num;
resp.response_length = uverbs_response_length(attrs, sizeof(resp));
+ return uverbs_response(attrs, &resp, sizeof(resp));
- err = uverbs_response(attrs, &resp, sizeof(resp));
- if (err)
- goto err_copy;
-
- kfree(wqs_handles);
-
- for (j = 0; j < num_read_wqs; j++)
- rdma_lookup_put_uobject(&wqs[j]->uobject->uevent.uobject,
- UVERBS_LOOKUP_READ);
-
- rdma_alloc_commit_uobject(uobj, attrs);
- return 0;
-
-err_copy:
- ib_destroy_rwq_ind_table(rwq_ind_tbl);
err_uobj:
uobj_alloc_abort(uobj, attrs);
put_wqs:
- for (j = 0; j < num_read_wqs; j++)
- rdma_lookup_put_uobject(&wqs[j]->uobject->uevent.uobject,
+ for (i = 0; i < num_read_wqs; i++) {
+ rdma_lookup_put_uobject(&wqs[i]->uobject->uevent.uobject,
UVERBS_LOOKUP_READ);
+ atomic_dec(&wqs[i]->usecnt);
+ }
err_free:
kfree(wqs_handles);
kfree(wqs);
@@ -3218,7 +3109,7 @@
static int ib_uverbs_ex_create_flow(struct uverbs_attr_bundle *attrs)
{
struct ib_uverbs_create_flow cmd;
- struct ib_uverbs_create_flow_resp resp;
+ struct ib_uverbs_create_flow_resp resp = {};
struct ib_uobject *uobj;
struct ib_flow *flow_id;
struct ib_uverbs_flow_attr *kern_flow_attr;
@@ -3351,23 +3242,17 @@
ib_set_flow(uobj, flow_id, qp, qp->device, uflow_res);
- memset(&resp, 0, sizeof(resp));
- resp.flow_handle = uobj->id;
-
- err = uverbs_response(attrs, &resp, sizeof(resp));
- if (err)
- goto err_copy;
-
rdma_lookup_put_uobject(&qp->uobject->uevent.uobject,
UVERBS_LOOKUP_READ);
kfree(flow_attr);
+
if (cmd.flow_attr.num_of_specs)
kfree(kern_flow_attr);
- rdma_alloc_commit_uobject(uobj, attrs);
- return 0;
-err_copy:
- if (!qp->device->ops.destroy_flow(flow_id))
- atomic_dec(&qp->usecnt);
+ uobj_finalize_uobj_create(uobj, attrs);
+
+ resp.flow_handle = uobj->id;
+ return uverbs_response(attrs, &resp, sizeof(resp));
+
err_free:
ib_uverbs_flow_resources_free(uflow_res);
err_free_flow_attr:
@@ -3402,13 +3287,13 @@
struct ib_uverbs_create_xsrq *cmd,
struct ib_udata *udata)
{
- struct ib_uverbs_create_srq_resp resp;
+ struct ib_uverbs_create_srq_resp resp = {};
struct ib_usrq_object *obj;
struct ib_pd *pd;
struct ib_srq *srq;
- struct ib_uobject *xrcd_uobj;
struct ib_srq_init_attr attr;
int ret;
+ struct ib_uobject *xrcd_uobj;
struct ib_device *ib_dev;
obj = (struct ib_usrq_object *)uobj_alloc(UVERBS_OBJECT_SRQ, attrs,
@@ -3473,17 +3358,9 @@
if (obj->uevent.event_file)
uverbs_uobject_get(&obj->uevent.event_file->uobj);
- memset(&resp, 0, sizeof resp);
- resp.srq_handle = obj->uevent.uobject.id;
- resp.max_wr = attr.attr.max_wr;
- resp.max_sge = attr.attr.max_sge;
if (cmd->srq_type == IB_SRQT_XRC)
resp.srqn = srq->ext.xrc.srq_num;
- ret = uverbs_response(attrs, &resp, sizeof(resp));
- if (ret)
- goto err_copy;
-
if (cmd->srq_type == IB_SRQT_XRC)
uobj_put_read(xrcd_uobj);
@@ -3492,13 +3369,13 @@
UVERBS_LOOKUP_READ);
uobj_put_obj_read(pd);
- rdma_alloc_commit_uobject(&obj->uevent.uobject, attrs);
- return 0;
+ uobj_finalize_uobj_create(&obj->uevent.uobject, attrs);
-err_copy:
- if (obj->uevent.event_file)
- uverbs_uobject_put(&obj->uevent.event_file->uobj);
- ib_destroy_srq_user(srq, uverbs_get_cleared_udata(attrs));
+ resp.srq_handle = obj->uevent.uobject.id;
+ resp.max_wr = attr.attr.max_wr;
+ resp.max_sge = attr.attr.max_sge;
+ return uverbs_response(attrs, &resp, sizeof(resp));
+
err_put_pd:
uobj_put_obj_read(pd);
err_put_cq:
diff --git a/drivers/infiniband/core/uverbs_ioctl.c b/drivers/infiniband/core/uverbs_ioctl.c
index 2d882c0..ef04a26 100644
--- a/drivers/infiniband/core/uverbs_ioctl.c
+++ b/drivers/infiniband/core/uverbs_ioctl.c
@@ -790,6 +790,7 @@
}
return uverbs_copy_to(bundle, idx, from, size);
}
+EXPORT_SYMBOL(uverbs_copy_to_struct_or_zero);
/* Once called an abort will call through to the type's destroy_hw() */
void uverbs_finalize_uobj_create(const struct uverbs_attr_bundle *bundle,
diff --git a/drivers/infiniband/core/uverbs_main.c b/drivers/infiniband/core/uverbs_main.c
index 69e4755..37794d8 100644
--- a/drivers/infiniband/core/uverbs_main.c
+++ b/drivers/infiniband/core/uverbs_main.c
@@ -601,6 +601,7 @@
memset(bundle.attr_present, 0, sizeof(bundle.attr_present));
bundle.ufile = file;
bundle.context = NULL; /* only valid if bundle has uobject */
+ bundle.uobject = NULL;
if (!method_elm->is_ex) {
size_t in_len = hdr.in_words * 4 - sizeof(hdr);
size_t out_len = hdr.out_words * 4;
@@ -664,6 +665,9 @@
}
ret = method_elm->handler(&bundle);
+ if (bundle.uobject)
+ uverbs_finalize_object(bundle.uobject, UVERBS_ACCESS_NEW, true,
+ !ret, &bundle);
out_unlock:
srcu_read_unlock(&file->device->disassociate_srcu, srcu_key);
return (ret) ? : count;
diff --git a/drivers/infiniband/core/uverbs_std_types_counters.c b/drivers/infiniband/core/uverbs_std_types_counters.c
index 9f01330..c7e7438 100644
--- a/drivers/infiniband/core/uverbs_std_types_counters.c
+++ b/drivers/infiniband/core/uverbs_std_types_counters.c
@@ -46,7 +46,9 @@
if (ret)
return ret;
- return counters->device->ops.destroy_counters(counters);
+ counters->device->ops.destroy_counters(counters);
+ kfree(counters);
+ return 0;
}
static int UVERBS_HANDLER(UVERBS_METHOD_COUNTERS_CREATE)(
@@ -66,20 +68,19 @@
if (!ib_dev->ops.create_counters)
return -EOPNOTSUPP;
- counters = ib_dev->ops.create_counters(ib_dev, attrs);
- if (IS_ERR(counters)) {
- ret = PTR_ERR(counters);
- goto err_create_counters;
- }
+ counters = rdma_zalloc_drv_obj(ib_dev, ib_counters);
+ if (!counters)
+ return -ENOMEM;
counters->device = ib_dev;
counters->uobject = uobj;
uobj->object = counters;
atomic_set(&counters->usecnt, 0);
- return 0;
+ ret = ib_dev->ops.create_counters(counters, attrs);
+ if (ret)
+ kfree(counters);
-err_create_counters:
return ret;
}
diff --git a/drivers/infiniband/core/uverbs_std_types_cq.c b/drivers/infiniband/core/uverbs_std_types_cq.c
index 5dce2c7..b1c7dac 100644
--- a/drivers/infiniband/core/uverbs_std_types_cq.c
+++ b/drivers/infiniband/core/uverbs_std_types_cq.c
@@ -207,11 +207,8 @@
DECLARE_UVERBS_NAMED_OBJECT(
UVERBS_OBJECT_CQ,
UVERBS_TYPE_ALLOC_IDR_SZ(sizeof(struct ib_ucq_object), uverbs_free_cq),
-
-#if IS_ENABLED(CONFIG_INFINIBAND_EXP_LEGACY_VERBS_NEW_UAPI)
&UVERBS_METHOD(UVERBS_METHOD_CQ_CREATE),
&UVERBS_METHOD(UVERBS_METHOD_CQ_DESTROY)
-#endif
);
const struct uapi_definition uverbs_def_obj_cq[] = {
diff --git a/drivers/infiniband/core/uverbs_std_types_device.c b/drivers/infiniband/core/uverbs_std_types_device.c
index ae4a59d..75df2094 100644
--- a/drivers/infiniband/core/uverbs_std_types_device.c
+++ b/drivers/infiniband/core/uverbs_std_types_device.c
@@ -38,7 +38,12 @@
attrs->ucore.outlen < method_elm->resp_size)
return -ENOSPC;
- return method_elm->handler(attrs);
+ attrs->uobject = NULL;
+ rc = method_elm->handler(attrs);
+ if (attrs->uobject)
+ uverbs_finalize_object(attrs->uobject, UVERBS_ACCESS_NEW, true,
+ !rc, attrs);
+ return rc;
}
DECLARE_UVERBS_NAMED_METHOD(UVERBS_METHOD_INVOKE_WRITE,
@@ -229,6 +234,37 @@
return 0;
}
+static int UVERBS_HANDLER(UVERBS_METHOD_QUERY_CONTEXT)(
+ struct uverbs_attr_bundle *attrs)
+{
+ u64 core_support = IB_UVERBS_CORE_SUPPORT_OPTIONAL_MR_ACCESS;
+ struct ib_ucontext *ucontext;
+ struct ib_device *ib_dev;
+ u32 num_comp;
+ int ret;
+
+ ucontext = ib_uverbs_get_ucontext(attrs);
+ if (IS_ERR(ucontext))
+ return PTR_ERR(ucontext);
+ ib_dev = ucontext->device;
+
+ if (!ib_dev->ops.query_ucontext)
+ return -EOPNOTSUPP;
+
+ num_comp = attrs->ufile->device->num_comp_vectors;
+ ret = uverbs_copy_to(attrs, UVERBS_ATTR_QUERY_CONTEXT_NUM_COMP_VECTORS,
+ &num_comp, sizeof(num_comp));
+ if (IS_UVERBS_COPY_ERR(ret))
+ return ret;
+
+ ret = uverbs_copy_to(attrs, UVERBS_ATTR_QUERY_CONTEXT_CORE_SUPPORT,
+ &core_support, sizeof(core_support));
+ if (IS_UVERBS_COPY_ERR(ret))
+ return ret;
+
+ return ucontext->device->ops.query_ucontext(ucontext, attrs);
+}
+
DECLARE_UVERBS_NAMED_METHOD(
UVERBS_METHOD_GET_CONTEXT,
UVERBS_ATTR_PTR_OUT(UVERBS_ATTR_GET_CONTEXT_NUM_COMP_VECTORS,
@@ -238,6 +274,13 @@
UVERBS_ATTR_UHW());
DECLARE_UVERBS_NAMED_METHOD(
+ UVERBS_METHOD_QUERY_CONTEXT,
+ UVERBS_ATTR_PTR_OUT(UVERBS_ATTR_QUERY_CONTEXT_NUM_COMP_VECTORS,
+ UVERBS_ATTR_TYPE(u32), UA_OPTIONAL),
+ UVERBS_ATTR_PTR_OUT(UVERBS_ATTR_QUERY_CONTEXT_CORE_SUPPORT,
+ UVERBS_ATTR_TYPE(u64), UA_OPTIONAL));
+
+DECLARE_UVERBS_NAMED_METHOD(
UVERBS_METHOD_INFO_HANDLES,
/* Also includes any device specific object ids */
UVERBS_ATTR_CONST_IN(UVERBS_ATTR_INFO_OBJECT_ID,
@@ -260,7 +303,8 @@
&UVERBS_METHOD(UVERBS_METHOD_GET_CONTEXT),
&UVERBS_METHOD(UVERBS_METHOD_INVOKE_WRITE),
&UVERBS_METHOD(UVERBS_METHOD_INFO_HANDLES),
- &UVERBS_METHOD(UVERBS_METHOD_QUERY_PORT));
+ &UVERBS_METHOD(UVERBS_METHOD_QUERY_PORT),
+ &UVERBS_METHOD(UVERBS_METHOD_QUERY_CONTEXT));
const struct uapi_definition uverbs_def_obj_device[] = {
UAPI_DEF_CHAIN_OBJ_TREE_NAMED(UVERBS_OBJECT_DEVICE),
diff --git a/drivers/infiniband/core/uverbs_std_types_mr.c b/drivers/infiniband/core/uverbs_std_types_mr.c
index a2722ef..9b22bb5 100644
--- a/drivers/infiniband/core/uverbs_std_types_mr.c
+++ b/drivers/infiniband/core/uverbs_std_types_mr.c
@@ -69,7 +69,7 @@
num_sge = uverbs_attr_ptr_get_array_size(
attrs, UVERBS_ATTR_ADVISE_MR_SGE_LIST, sizeof(struct ib_sge));
- if (num_sge < 0)
+ if (num_sge <= 0)
return num_sge;
sg_list = uverbs_attr_get_alloced_ptr(attrs,
@@ -148,6 +148,36 @@
return ret;
}
+static int UVERBS_HANDLER(UVERBS_METHOD_QUERY_MR)(
+ struct uverbs_attr_bundle *attrs)
+{
+ struct ib_mr *mr =
+ uverbs_attr_get_obj(attrs, UVERBS_ATTR_QUERY_MR_HANDLE);
+ int ret;
+
+ ret = uverbs_copy_to(attrs, UVERBS_ATTR_QUERY_MR_RESP_LKEY, &mr->lkey,
+ sizeof(mr->lkey));
+ if (ret)
+ return ret;
+
+ ret = uverbs_copy_to(attrs, UVERBS_ATTR_QUERY_MR_RESP_RKEY,
+ &mr->rkey, sizeof(mr->rkey));
+
+ if (ret)
+ return ret;
+
+ ret = uverbs_copy_to(attrs, UVERBS_ATTR_QUERY_MR_RESP_LENGTH,
+ &mr->length, sizeof(mr->length));
+
+ if (ret)
+ return ret;
+
+ ret = uverbs_copy_to(attrs, UVERBS_ATTR_QUERY_MR_RESP_IOVA,
+ &mr->iova, sizeof(mr->iova));
+
+ return IS_UVERBS_COPY_ERR(ret) ? ret : 0;
+}
+
DECLARE_UVERBS_NAMED_METHOD(
UVERBS_METHOD_ADVISE_MR,
UVERBS_ATTR_IDR(UVERBS_ATTR_ADVISE_MR_PD_HANDLE,
@@ -166,6 +196,25 @@
UA_ALLOC_AND_COPY));
DECLARE_UVERBS_NAMED_METHOD(
+ UVERBS_METHOD_QUERY_MR,
+ UVERBS_ATTR_IDR(UVERBS_ATTR_QUERY_MR_HANDLE,
+ UVERBS_OBJECT_MR,
+ UVERBS_ACCESS_READ,
+ UA_MANDATORY),
+ UVERBS_ATTR_PTR_OUT(UVERBS_ATTR_QUERY_MR_RESP_RKEY,
+ UVERBS_ATTR_TYPE(u32),
+ UA_MANDATORY),
+ UVERBS_ATTR_PTR_OUT(UVERBS_ATTR_QUERY_MR_RESP_LKEY,
+ UVERBS_ATTR_TYPE(u32),
+ UA_MANDATORY),
+ UVERBS_ATTR_PTR_OUT(UVERBS_ATTR_QUERY_MR_RESP_LENGTH,
+ UVERBS_ATTR_TYPE(u64),
+ UA_MANDATORY),
+ UVERBS_ATTR_PTR_OUT(UVERBS_ATTR_QUERY_MR_RESP_IOVA,
+ UVERBS_ATTR_TYPE(u64),
+ UA_OPTIONAL));
+
+DECLARE_UVERBS_NAMED_METHOD(
UVERBS_METHOD_DM_MR_REG,
UVERBS_ATTR_IDR(UVERBS_ATTR_REG_DM_MR_HANDLE,
UVERBS_OBJECT_MR,
@@ -206,7 +255,8 @@
UVERBS_TYPE_ALLOC_IDR(uverbs_free_mr),
&UVERBS_METHOD(UVERBS_METHOD_DM_MR_REG),
&UVERBS_METHOD(UVERBS_METHOD_MR_DESTROY),
- &UVERBS_METHOD(UVERBS_METHOD_ADVISE_MR));
+ &UVERBS_METHOD(UVERBS_METHOD_ADVISE_MR),
+ &UVERBS_METHOD(UVERBS_METHOD_QUERY_MR));
const struct uapi_definition uverbs_def_obj_mr[] = {
UAPI_DEF_CHAIN_OBJ_TREE_NAMED(UVERBS_OBJECT_MR,
diff --git a/drivers/infiniband/core/verbs.c b/drivers/infiniband/core/verbs.c
index 53d6505c..3096e73 100644
--- a/drivers/infiniband/core/verbs.c
+++ b/drivers/infiniband/core/verbs.c
@@ -988,8 +988,8 @@
* @srq_init_attr: A list of initial attributes required to create the
* SRQ. If SRQ creation succeeds, then the attributes are updated to
* the actual capabilities of the created SRQ.
- * @uobject - uobject pointer if this is not a kernel SRQ
- * @udata - udata pointer if this is not a kernel SRQ
+ * @uobject: uobject pointer if this is not a kernel SRQ
+ * @udata: udata pointer if this is not a kernel SRQ
*
* srq_attr->max_wr and srq_attr->max_sge are read the determine the
* requested size of the SRQ, and set to the actual values allocated
@@ -1090,13 +1090,6 @@
spin_unlock_irqrestore(&qp->device->qp_open_list_lock, flags);
}
-static void __ib_insert_xrcd_qp(struct ib_xrcd *xrcd, struct ib_qp *qp)
-{
- mutex_lock(&xrcd->tgt_qp_mutex);
- list_add(&qp->xrcd_list, &xrcd->tgt_qp_list);
- mutex_unlock(&xrcd->tgt_qp_mutex);
-}
-
static struct ib_qp *__ib_open_qp(struct ib_qp *real_qp,
void (*event_handler)(struct ib_event *, void *),
void *qp_context)
@@ -1139,16 +1132,15 @@
if (qp_open_attr->qp_type != IB_QPT_XRC_TGT)
return ERR_PTR(-EINVAL);
- qp = ERR_PTR(-EINVAL);
- mutex_lock(&xrcd->tgt_qp_mutex);
- list_for_each_entry(real_qp, &xrcd->tgt_qp_list, xrcd_list) {
- if (real_qp->qp_num == qp_open_attr->qp_num) {
- qp = __ib_open_qp(real_qp, qp_open_attr->event_handler,
- qp_open_attr->qp_context);
- break;
- }
+ down_read(&xrcd->tgt_qps_rwsem);
+ real_qp = xa_load(&xrcd->tgt_qps, qp_open_attr->qp_num);
+ if (!real_qp) {
+ up_read(&xrcd->tgt_qps_rwsem);
+ return ERR_PTR(-EINVAL);
}
- mutex_unlock(&xrcd->tgt_qp_mutex);
+ qp = __ib_open_qp(real_qp, qp_open_attr->event_handler,
+ qp_open_attr->qp_context);
+ up_read(&xrcd->tgt_qps_rwsem);
return qp;
}
EXPORT_SYMBOL(ib_open_qp);
@@ -1157,6 +1149,7 @@
struct ib_qp_init_attr *qp_init_attr)
{
struct ib_qp *real_qp = qp;
+ int err;
qp->event_handler = __ib_shared_qp_event_handler;
qp->qp_context = qp;
@@ -1172,7 +1165,12 @@
if (IS_ERR(qp))
return qp;
- __ib_insert_xrcd_qp(qp_init_attr->xrcd, real_qp);
+ err = xa_err(xa_store(&qp_init_attr->xrcd->tgt_qps, real_qp->qp_num,
+ real_qp, GFP_KERNEL));
+ if (err) {
+ ib_close_qp(qp);
+ return ERR_PTR(err);
+ }
return qp;
}
@@ -1712,7 +1710,7 @@
if (!(rdma_protocol_ib(qp->device,
attr->alt_ah_attr.port_num) &&
rdma_protocol_ib(qp->device, port))) {
- ret = EINVAL;
+ ret = -EINVAL;
goto out;
}
}
@@ -1887,21 +1885,18 @@
real_qp = qp->real_qp;
xrcd = real_qp->xrcd;
-
- mutex_lock(&xrcd->tgt_qp_mutex);
+ down_write(&xrcd->tgt_qps_rwsem);
ib_close_qp(qp);
if (atomic_read(&real_qp->usecnt) == 0)
- list_del(&real_qp->xrcd_list);
+ xa_erase(&xrcd->tgt_qps, real_qp->qp_num);
else
real_qp = NULL;
- mutex_unlock(&xrcd->tgt_qp_mutex);
+ up_write(&xrcd->tgt_qps_rwsem);
if (real_qp) {
ret = ib_destroy_qp(real_qp);
if (!ret)
atomic_dec(&xrcd->usecnt);
- else
- __ib_insert_xrcd_qp(xrcd, real_qp);
}
return 0;
@@ -2077,6 +2072,9 @@
if (!pd->device->ops.advise_mr)
return -EOPNOTSUPP;
+ if (!num_sge)
+ return 0;
+
return pd->device->ops.advise_mr(pd, advice, flags, sg_list, num_sge,
NULL);
}
@@ -2104,11 +2102,10 @@
EXPORT_SYMBOL(ib_dereg_mr_user);
/**
- * ib_alloc_mr_user() - Allocates a memory region
+ * ib_alloc_mr() - Allocates a memory region
* @pd: protection domain associated with the region
* @mr_type: memory region type
* @max_num_sg: maximum sg entries available for registration.
- * @udata: user data or null for kernel objects
*
* Notes:
* Memory registeration page/sg lists must not exceed max_num_sg.
@@ -2116,8 +2113,8 @@
* max_num_sg * used_page_size.
*
*/
-struct ib_mr *ib_alloc_mr_user(struct ib_pd *pd, enum ib_mr_type mr_type,
- u32 max_num_sg, struct ib_udata *udata)
+struct ib_mr *ib_alloc_mr(struct ib_pd *pd, enum ib_mr_type mr_type,
+ u32 max_num_sg)
{
struct ib_mr *mr;
@@ -2132,25 +2129,26 @@
goto out;
}
- mr = pd->device->ops.alloc_mr(pd, mr_type, max_num_sg, udata);
- if (!IS_ERR(mr)) {
- mr->device = pd->device;
- mr->pd = pd;
- mr->dm = NULL;
- mr->uobject = NULL;
- atomic_inc(&pd->usecnt);
- mr->need_inval = false;
- mr->res.type = RDMA_RESTRACK_MR;
- rdma_restrack_kadd(&mr->res);
- mr->type = mr_type;
- mr->sig_attrs = NULL;
- }
+ mr = pd->device->ops.alloc_mr(pd, mr_type, max_num_sg);
+ if (IS_ERR(mr))
+ goto out;
+
+ mr->device = pd->device;
+ mr->pd = pd;
+ mr->dm = NULL;
+ mr->uobject = NULL;
+ atomic_inc(&pd->usecnt);
+ mr->need_inval = false;
+ mr->res.type = RDMA_RESTRACK_MR;
+ rdma_restrack_kadd(&mr->res);
+ mr->type = mr_type;
+ mr->sig_attrs = NULL;
out:
trace_mr_alloc(pd, mr_type, max_num_sg, mr);
return mr;
}
-EXPORT_SYMBOL(ib_alloc_mr_user);
+EXPORT_SYMBOL(ib_alloc_mr);
/**
* ib_alloc_mr_integrity() - Allocates an integrity memory region
@@ -2288,45 +2286,57 @@
}
EXPORT_SYMBOL(ib_detach_mcast);
-struct ib_xrcd *__ib_alloc_xrcd(struct ib_device *device, const char *caller)
+/**
+ * ib_alloc_xrcd_user - Allocates an XRC domain.
+ * @device: The device on which to allocate the XRC domain.
+ * @inode: inode to connect XRCD
+ * @udata: Valid user data or NULL for kernel object
+ */
+struct ib_xrcd *ib_alloc_xrcd_user(struct ib_device *device,
+ struct inode *inode, struct ib_udata *udata)
{
struct ib_xrcd *xrcd;
+ int ret;
if (!device->ops.alloc_xrcd)
return ERR_PTR(-EOPNOTSUPP);
- xrcd = device->ops.alloc_xrcd(device, NULL);
- if (!IS_ERR(xrcd)) {
- xrcd->device = device;
- xrcd->inode = NULL;
- atomic_set(&xrcd->usecnt, 0);
- mutex_init(&xrcd->tgt_qp_mutex);
- INIT_LIST_HEAD(&xrcd->tgt_qp_list);
- }
+ xrcd = rdma_zalloc_drv_obj(device, ib_xrcd);
+ if (!xrcd)
+ return ERR_PTR(-ENOMEM);
+ xrcd->device = device;
+ xrcd->inode = inode;
+ atomic_set(&xrcd->usecnt, 0);
+ init_rwsem(&xrcd->tgt_qps_rwsem);
+ xa_init(&xrcd->tgt_qps);
+
+ ret = device->ops.alloc_xrcd(xrcd, udata);
+ if (ret)
+ goto err;
return xrcd;
+err:
+ kfree(xrcd);
+ return ERR_PTR(ret);
}
-EXPORT_SYMBOL(__ib_alloc_xrcd);
+EXPORT_SYMBOL(ib_alloc_xrcd_user);
-int ib_dealloc_xrcd(struct ib_xrcd *xrcd, struct ib_udata *udata)
+/**
+ * ib_dealloc_xrcd_user - Deallocates an XRC domain.
+ * @xrcd: The XRC domain to deallocate.
+ * @udata: Valid user data or NULL for kernel object
+ */
+int ib_dealloc_xrcd_user(struct ib_xrcd *xrcd, struct ib_udata *udata)
{
- struct ib_qp *qp;
- int ret;
-
if (atomic_read(&xrcd->usecnt))
return -EBUSY;
- while (!list_empty(&xrcd->tgt_qp_list)) {
- qp = list_entry(xrcd->tgt_qp_list.next, struct ib_qp, xrcd_list);
- ret = ib_destroy_qp(qp);
- if (ret)
- return ret;
- }
- mutex_destroy(&xrcd->tgt_qp_mutex);
-
- return xrcd->device->ops.dealloc_xrcd(xrcd, udata);
+ WARN_ON(!xa_empty(&xrcd->tgt_qps));
+ xrcd->device->ops.dealloc_xrcd(xrcd, udata);
+ kfree(xrcd);
+ return 0;
}
-EXPORT_SYMBOL(ib_dealloc_xrcd);
+EXPORT_SYMBOL(ib_dealloc_xrcd_user);
/**
* ib_create_wq - Creates a WQ associated with the specified protection
@@ -2410,45 +2420,6 @@
EXPORT_SYMBOL(ib_modify_wq);
/*
- * ib_create_rwq_ind_table - Creates a RQ Indirection Table.
- * @device: The device on which to create the rwq indirection table.
- * @ib_rwq_ind_table_init_attr: A list of initial attributes required to
- * create the Indirection Table.
- *
- * Note: The life time of ib_rwq_ind_table_init_attr->ind_tbl is not less
- * than the created ib_rwq_ind_table object and the caller is responsible
- * for its memory allocation/free.
- */
-struct ib_rwq_ind_table *ib_create_rwq_ind_table(struct ib_device *device,
- struct ib_rwq_ind_table_init_attr *init_attr)
-{
- struct ib_rwq_ind_table *rwq_ind_table;
- int i;
- u32 table_size;
-
- if (!device->ops.create_rwq_ind_table)
- return ERR_PTR(-EOPNOTSUPP);
-
- table_size = (1 << init_attr->log_ind_tbl_size);
- rwq_ind_table = device->ops.create_rwq_ind_table(device,
- init_attr, NULL);
- if (IS_ERR(rwq_ind_table))
- return rwq_ind_table;
-
- rwq_ind_table->ind_tbl = init_attr->ind_tbl;
- rwq_ind_table->log_ind_tbl_size = init_attr->log_ind_tbl_size;
- rwq_ind_table->device = device;
- rwq_ind_table->uobject = NULL;
- atomic_set(&rwq_ind_table->usecnt, 0);
-
- for (i = 0; i < table_size; i++)
- atomic_inc(&rwq_ind_table->ind_tbl[i]->usecnt);
-
- return rwq_ind_table;
-}
-EXPORT_SYMBOL(ib_create_rwq_ind_table);
-
-/*
* ib_destroy_rwq_ind_table - Destroys the specified Indirection Table.
* @wq_ind_table: The Indirection Table to destroy.
*/
diff --git a/drivers/infiniband/hw/bnxt_re/ib_verbs.c b/drivers/infiniband/hw/bnxt_re/ib_verbs.c
index 8b6ad5c..3f18efc 100644
--- a/drivers/infiniband/hw/bnxt_re/ib_verbs.c
+++ b/drivers/infiniband/hw/bnxt_re/ib_verbs.c
@@ -842,16 +842,79 @@
}
}
+static u16 bnxt_re_setup_rwqe_size(struct bnxt_qplib_qp *qplqp,
+ int rsge, int max)
+{
+ if (qplqp->wqe_mode == BNXT_QPLIB_WQE_MODE_STATIC)
+ rsge = max;
+ return bnxt_re_get_rwqe_size(rsge);
+}
+
+static u16 bnxt_re_get_wqe_size(int ilsize, int nsge)
+{
+ u16 wqe_size, calc_ils;
+
+ wqe_size = bnxt_re_get_swqe_size(nsge);
+ if (ilsize) {
+ calc_ils = sizeof(struct sq_send_hdr) + ilsize;
+ wqe_size = max_t(u16, calc_ils, wqe_size);
+ wqe_size = ALIGN(wqe_size, sizeof(struct sq_send_hdr));
+ }
+ return wqe_size;
+}
+
+static int bnxt_re_setup_swqe_size(struct bnxt_re_qp *qp,
+ struct ib_qp_init_attr *init_attr)
+{
+ struct bnxt_qplib_dev_attr *dev_attr;
+ struct bnxt_qplib_qp *qplqp;
+ struct bnxt_re_dev *rdev;
+ struct bnxt_qplib_q *sq;
+ int align, ilsize;
+
+ rdev = qp->rdev;
+ qplqp = &qp->qplib_qp;
+ sq = &qplqp->sq;
+ dev_attr = &rdev->dev_attr;
+
+ align = sizeof(struct sq_send_hdr);
+ ilsize = ALIGN(init_attr->cap.max_inline_data, align);
+
+ sq->wqe_size = bnxt_re_get_wqe_size(ilsize, sq->max_sge);
+ if (sq->wqe_size > bnxt_re_get_swqe_size(dev_attr->max_qp_sges))
+ return -EINVAL;
+ /* For gen p4 and gen p5 backward compatibility mode
+ * wqe size is fixed to 128 bytes
+ */
+ if (sq->wqe_size < bnxt_re_get_swqe_size(dev_attr->max_qp_sges) &&
+ qplqp->wqe_mode == BNXT_QPLIB_WQE_MODE_STATIC)
+ sq->wqe_size = bnxt_re_get_swqe_size(dev_attr->max_qp_sges);
+
+ if (init_attr->cap.max_inline_data) {
+ qplqp->max_inline_data = sq->wqe_size -
+ sizeof(struct sq_send_hdr);
+ init_attr->cap.max_inline_data = qplqp->max_inline_data;
+ if (qplqp->wqe_mode == BNXT_QPLIB_WQE_MODE_STATIC)
+ sq->max_sge = qplqp->max_inline_data /
+ sizeof(struct sq_sge);
+ }
+
+ return 0;
+}
+
static int bnxt_re_init_user_qp(struct bnxt_re_dev *rdev, struct bnxt_re_pd *pd,
struct bnxt_re_qp *qp, struct ib_udata *udata)
{
+ struct bnxt_qplib_qp *qplib_qp;
+ struct bnxt_re_ucontext *cntx;
struct bnxt_re_qp_req ureq;
- struct bnxt_qplib_qp *qplib_qp = &qp->qplib_qp;
- struct ib_umem *umem;
int bytes = 0, psn_sz;
- struct bnxt_re_ucontext *cntx = rdma_udata_to_drv_context(
- udata, struct bnxt_re_ucontext, ib_uctx);
+ struct ib_umem *umem;
+ int psn_nume;
+ qplib_qp = &qp->qplib_qp;
+ cntx = rdma_udata_to_drv_context(udata, struct bnxt_re_ucontext,
+ ib_uctx);
if (ib_copy_from_udata(&ureq, udata, sizeof(ureq)))
return -EFAULT;
@@ -859,10 +922,15 @@
/* Consider mapping PSN search memory only for RC QPs. */
if (qplib_qp->type == CMDQ_CREATE_QP_TYPE_RC) {
psn_sz = bnxt_qplib_is_chip_gen_p5(rdev->chip_ctx) ?
- sizeof(struct sq_psn_search_ext) :
- sizeof(struct sq_psn_search);
- bytes += (qplib_qp->sq.max_wqe * psn_sz);
+ sizeof(struct sq_psn_search_ext) :
+ sizeof(struct sq_psn_search);
+ psn_nume = (qplib_qp->wqe_mode == BNXT_QPLIB_WQE_MODE_STATIC) ?
+ qplib_qp->sq.max_wqe :
+ ((qplib_qp->sq.max_wqe * qplib_qp->sq.wqe_size) /
+ sizeof(struct bnxt_qplib_sge));
+ bytes += (psn_nume * psn_sz);
}
+
bytes = PAGE_ALIGN(bytes);
umem = ib_umem_get(&rdev->ibdev, ureq.qpsva, bytes,
IB_ACCESS_LOCAL_WRITE);
@@ -975,7 +1043,7 @@
qp->qplib_qp.sig_type = true;
/* Shadow QP SQ depth should be same as QP1 RQ depth */
- qp->qplib_qp.sq.wqe_size = bnxt_re_get_swqe_size();
+ qp->qplib_qp.sq.wqe_size = bnxt_re_get_wqe_size(0, 6);
qp->qplib_qp.sq.max_wqe = qp1_qp->rq.max_wqe;
qp->qplib_qp.sq.max_sge = 2;
/* Q full delta can be 1 since it is internal QP */
@@ -986,7 +1054,7 @@
qp->qplib_qp.scq = qp1_qp->scq;
qp->qplib_qp.rcq = qp1_qp->rcq;
- qp->qplib_qp.rq.wqe_size = bnxt_re_get_rwqe_size();
+ qp->qplib_qp.rq.wqe_size = bnxt_re_get_rwqe_size(6);
qp->qplib_qp.rq.max_wqe = qp1_qp->rq.max_wqe;
qp->qplib_qp.rq.max_sge = qp1_qp->rq.max_sge;
/* Q full delta can be 1 since it is internal QP */
@@ -1041,19 +1109,21 @@
qplqp->srq = &srq->qplib_srq;
rq->max_wqe = 0;
} else {
- rq->wqe_size = bnxt_re_get_rwqe_size();
+ rq->max_sge = init_attr->cap.max_recv_sge;
+ if (rq->max_sge > dev_attr->max_qp_sges)
+ rq->max_sge = dev_attr->max_qp_sges;
+ init_attr->cap.max_recv_sge = rq->max_sge;
+ rq->wqe_size = bnxt_re_setup_rwqe_size(qplqp, rq->max_sge,
+ dev_attr->max_qp_sges);
/* Allocate 1 more than what's provided so posting max doesn't
* mean empty.
*/
entries = roundup_pow_of_two(init_attr->cap.max_recv_wr + 1);
rq->max_wqe = min_t(u32, entries, dev_attr->max_qp_wqes + 1);
- rq->q_full_delta = rq->max_wqe - init_attr->cap.max_recv_wr;
- rq->max_sge = init_attr->cap.max_recv_sge;
- if (rq->max_sge > dev_attr->max_qp_sges)
- rq->max_sge = dev_attr->max_qp_sges;
+ rq->q_full_delta = 0;
+ rq->sg_info.pgsize = PAGE_SIZE;
+ rq->sg_info.pgshft = PAGE_SHIFT;
}
- rq->sg_info.pgsize = PAGE_SIZE;
- rq->sg_info.pgshft = PAGE_SHIFT;
return 0;
}
@@ -1068,41 +1138,48 @@
qplqp = &qp->qplib_qp;
dev_attr = &rdev->dev_attr;
- qplqp->rq.max_sge = dev_attr->max_qp_sges;
- if (qplqp->rq.max_sge > dev_attr->max_qp_sges)
+ if (!bnxt_qplib_is_chip_gen_p5(rdev->chip_ctx)) {
qplqp->rq.max_sge = dev_attr->max_qp_sges;
- qplqp->rq.max_sge = 6;
+ if (qplqp->rq.max_sge > dev_attr->max_qp_sges)
+ qplqp->rq.max_sge = dev_attr->max_qp_sges;
+ qplqp->rq.max_sge = 6;
+ }
}
-static void bnxt_re_init_sq_attr(struct bnxt_re_qp *qp,
- struct ib_qp_init_attr *init_attr,
- struct ib_udata *udata)
+static int bnxt_re_init_sq_attr(struct bnxt_re_qp *qp,
+ struct ib_qp_init_attr *init_attr,
+ struct ib_udata *udata)
{
struct bnxt_qplib_dev_attr *dev_attr;
struct bnxt_qplib_qp *qplqp;
struct bnxt_re_dev *rdev;
struct bnxt_qplib_q *sq;
int entries;
+ int diff;
+ int rc;
rdev = qp->rdev;
qplqp = &qp->qplib_qp;
sq = &qplqp->sq;
dev_attr = &rdev->dev_attr;
- sq->wqe_size = bnxt_re_get_swqe_size();
sq->max_sge = init_attr->cap.max_send_sge;
- if (sq->max_sge > dev_attr->max_qp_sges)
+ if (sq->max_sge > dev_attr->max_qp_sges) {
sq->max_sge = dev_attr->max_qp_sges;
- /*
- * Change the SQ depth if user has requested minimum using
- * configfs. Only supported for kernel consumers
- */
+ init_attr->cap.max_send_sge = sq->max_sge;
+ }
+
+ rc = bnxt_re_setup_swqe_size(qp, init_attr);
+ if (rc)
+ return rc;
+
entries = init_attr->cap.max_send_wr;
/* Allocate 128 + 1 more than what's provided */
- entries = roundup_pow_of_two(entries + BNXT_QPLIB_RESERVED_QP_WRS + 1);
- sq->max_wqe = min_t(u32, entries, dev_attr->max_qp_wqes +
- BNXT_QPLIB_RESERVED_QP_WRS + 1);
- sq->q_full_delta = BNXT_QPLIB_RESERVED_QP_WRS + 1;
+ diff = (qplqp->wqe_mode == BNXT_QPLIB_WQE_MODE_VARIABLE) ?
+ 0 : BNXT_QPLIB_RESERVED_QP_WRS;
+ entries = roundup_pow_of_two(entries + diff + 1);
+ sq->max_wqe = min_t(u32, entries, dev_attr->max_qp_wqes + diff + 1);
+ sq->q_full_delta = diff + 1;
/*
* Reserving one slot for Phantom WQE. Application can
* post one extra entry in this case. But allowing this to avoid
@@ -1111,6 +1188,8 @@
qplqp->sq.q_full_delta -= 1;
qplqp->sq.sg_info.pgsize = PAGE_SIZE;
qplqp->sq.sg_info.pgshft = PAGE_SHIFT;
+
+ return 0;
}
static void bnxt_re_adjust_gsi_sq_attr(struct bnxt_re_qp *qp,
@@ -1125,13 +1204,16 @@
qplqp = &qp->qplib_qp;
dev_attr = &rdev->dev_attr;
- entries = roundup_pow_of_two(init_attr->cap.max_send_wr + 1);
- qplqp->sq.max_wqe = min_t(u32, entries, dev_attr->max_qp_wqes + 1);
- qplqp->sq.q_full_delta = qplqp->sq.max_wqe -
- init_attr->cap.max_send_wr;
- qplqp->sq.max_sge++; /* Need one extra sge to put UD header */
- if (qplqp->sq.max_sge > dev_attr->max_qp_sges)
- qplqp->sq.max_sge = dev_attr->max_qp_sges;
+ if (!bnxt_qplib_is_chip_gen_p5(rdev->chip_ctx)) {
+ entries = roundup_pow_of_two(init_attr->cap.max_send_wr + 1);
+ qplqp->sq.max_wqe = min_t(u32, entries,
+ dev_attr->max_qp_wqes + 1);
+ qplqp->sq.q_full_delta = qplqp->sq.max_wqe -
+ init_attr->cap.max_send_wr;
+ qplqp->sq.max_sge++; /* Need one extra sge to put UD header */
+ if (qplqp->sq.max_sge > dev_attr->max_qp_sges)
+ qplqp->sq.max_sge = dev_attr->max_qp_sges;
+ }
}
static int bnxt_re_init_qp_type(struct bnxt_re_dev *rdev,
@@ -1183,6 +1265,7 @@
goto out;
}
qplqp->type = (u8)qptype;
+ qplqp->wqe_mode = rdev->chip_ctx->modes.wqe_mode;
if (init_attr->qp_type == IB_QPT_RC) {
qplqp->max_rd_atomic = dev_attr->max_qp_rd_atom;
@@ -1226,7 +1309,9 @@
bnxt_re_adjust_gsi_rq_attr(qp);
/* Setup SQ */
- bnxt_re_init_sq_attr(qp, init_attr, udata);
+ rc = bnxt_re_init_sq_attr(qp, init_attr, udata);
+ if (rc)
+ goto out;
if (init_attr->qp_type == IB_QPT_GSI)
bnxt_re_adjust_gsi_sq_attr(qp, init_attr);
@@ -1574,8 +1659,9 @@
entries = dev_attr->max_srq_wqes + 1;
srq->qplib_srq.max_wqe = entries;
- srq->qplib_srq.wqe_size = bnxt_re_get_rwqe_size();
srq->qplib_srq.max_sge = srq_init_attr->attr.max_sge;
+ srq->qplib_srq.wqe_size =
+ bnxt_re_get_rwqe_size(srq->qplib_srq.max_sge);
srq->qplib_srq.threshold = srq_init_attr->attr.srq_limit;
srq->srq_limit = srq_init_attr->attr.srq_limit;
srq->qplib_srq.eventq_hw_ring_id = rdev->nq[0].ring_id;
@@ -3569,7 +3655,7 @@
}
struct ib_mr *bnxt_re_alloc_mr(struct ib_pd *ib_pd, enum ib_mr_type type,
- u32 max_num_sg, struct ib_udata *udata)
+ u32 max_num_sg)
{
struct bnxt_re_pd *pd = container_of(ib_pd, struct bnxt_re_pd, ib_pd);
struct bnxt_re_dev *rdev = pd->rdev;
diff --git a/drivers/infiniband/hw/bnxt_re/ib_verbs.h b/drivers/infiniband/hw/bnxt_re/ib_verbs.h
index e5fbbeb..1daeb30 100644
--- a/drivers/infiniband/hw/bnxt_re/ib_verbs.h
+++ b/drivers/infiniband/hw/bnxt_re/ib_verbs.h
@@ -136,14 +136,14 @@
spinlock_t sh_lock; /* protect shpg */
};
-static inline u16 bnxt_re_get_swqe_size(void)
+static inline u16 bnxt_re_get_swqe_size(int nsge)
{
- return sizeof(struct sq_send);
+ return sizeof(struct sq_send_hdr) + nsge * sizeof(struct sq_sge);
}
-static inline u16 bnxt_re_get_rwqe_size(void)
+static inline u16 bnxt_re_get_rwqe_size(int nsge)
{
- return sizeof(struct rq_wqe);
+ return sizeof(struct rq_wqe_hdr) + (nsge * sizeof(struct sq_sge));
}
int bnxt_re_query_device(struct ib_device *ibdev,
@@ -201,7 +201,7 @@
int bnxt_re_map_mr_sg(struct ib_mr *ib_mr, struct scatterlist *sg, int sg_nents,
unsigned int *sg_offset);
struct ib_mr *bnxt_re_alloc_mr(struct ib_pd *ib_pd, enum ib_mr_type mr_type,
- u32 max_num_sg, struct ib_udata *udata);
+ u32 max_num_sg);
int bnxt_re_dereg_mr(struct ib_mr *mr, struct ib_udata *udata);
struct ib_mw *bnxt_re_alloc_mw(struct ib_pd *ib_pd, enum ib_mw_type type,
struct ib_udata *udata);
diff --git a/drivers/infiniband/hw/bnxt_re/main.c b/drivers/infiniband/hw/bnxt_re/main.c
index b12fbc8..dad0df8 100644
--- a/drivers/infiniband/hw/bnxt_re/main.c
+++ b/drivers/infiniband/hw/bnxt_re/main.c
@@ -82,6 +82,15 @@
static void bnxt_re_dealloc_driver(struct ib_device *ib_dev);
static void bnxt_re_stop_irq(void *handle);
+static void bnxt_re_set_drv_mode(struct bnxt_re_dev *rdev, u8 mode)
+{
+ struct bnxt_qplib_chip_ctx *cctx;
+
+ cctx = rdev->chip_ctx;
+ cctx->modes.wqe_mode = bnxt_qplib_is_chip_gen_p5(rdev->chip_ctx) ?
+ mode : BNXT_QPLIB_WQE_MODE_STATIC;
+}
+
static void bnxt_re_destroy_chip_ctx(struct bnxt_re_dev *rdev)
{
struct bnxt_qplib_chip_ctx *chip_ctx;
@@ -97,7 +106,7 @@
kfree(chip_ctx);
}
-static int bnxt_re_setup_chip_ctx(struct bnxt_re_dev *rdev)
+static int bnxt_re_setup_chip_ctx(struct bnxt_re_dev *rdev, u8 wqe_mode)
{
struct bnxt_qplib_chip_ctx *chip_ctx;
struct bnxt_en_dev *en_dev;
@@ -117,6 +126,7 @@
rdev->qplib_res.cctx = rdev->chip_ctx;
rdev->rcfw.res = &rdev->qplib_res;
+ bnxt_re_set_drv_mode(rdev, wqe_mode);
return 0;
}
@@ -1386,7 +1396,7 @@
schedule_delayed_work(&rdev->worker, msecs_to_jiffies(30000));
}
-static int bnxt_re_dev_init(struct bnxt_re_dev *rdev)
+static int bnxt_re_dev_init(struct bnxt_re_dev *rdev, u8 wqe_mode)
{
struct bnxt_qplib_creq_ctx *creq;
struct bnxt_re_ring_attr rattr;
@@ -1406,7 +1416,7 @@
}
set_bit(BNXT_RE_FLAG_NETDEV_REGISTERED, &rdev->flags);
- rc = bnxt_re_setup_chip_ctx(rdev);
+ rc = bnxt_re_setup_chip_ctx(rdev, wqe_mode);
if (rc) {
ibdev_err(&rdev->ibdev, "Failed to get chip context\n");
return -EINVAL;
@@ -1585,7 +1595,7 @@
}
static int bnxt_re_add_device(struct bnxt_re_dev **rdev,
- struct net_device *netdev)
+ struct net_device *netdev, u8 wqe_mode)
{
int rc;
@@ -1599,7 +1609,7 @@
}
pci_dev_get((*rdev)->en_dev->pdev);
- rc = bnxt_re_dev_init(*rdev);
+ rc = bnxt_re_dev_init(*rdev, wqe_mode);
if (rc) {
pci_dev_put((*rdev)->en_dev->pdev);
bnxt_re_dev_unreg(*rdev);
@@ -1711,7 +1721,8 @@
case NETDEV_REGISTER:
if (rdev)
break;
- rc = bnxt_re_add_device(&rdev, real_dev);
+ rc = bnxt_re_add_device(&rdev, real_dev,
+ BNXT_QPLIB_WQE_MODE_STATIC);
if (!rc)
sch_work = true;
release = false;
diff --git a/drivers/infiniband/hw/bnxt_re/qplib_fp.c b/drivers/infiniband/hw/bnxt_re/qplib_fp.c
index c5e2957..117b423 100644
--- a/drivers/infiniband/hw/bnxt_re/qplib_fp.c
+++ b/drivers/infiniband/hw/bnxt_re/qplib_fp.c
@@ -178,11 +178,11 @@
if (qp->rq_hdr_buf)
dma_free_coherent(&res->pdev->dev,
- rq->hwq.max_elements * qp->rq_hdr_buf_size,
+ rq->max_wqe * qp->rq_hdr_buf_size,
qp->rq_hdr_buf, qp->rq_hdr_buf_map);
if (qp->sq_hdr_buf)
dma_free_coherent(&res->pdev->dev,
- sq->hwq.max_elements * qp->sq_hdr_buf_size,
+ sq->max_wqe * qp->sq_hdr_buf_size,
qp->sq_hdr_buf, qp->sq_hdr_buf_map);
qp->rq_hdr_buf = NULL;
qp->sq_hdr_buf = NULL;
@@ -199,10 +199,9 @@
struct bnxt_qplib_q *sq = &qp->sq;
int rc = 0;
- if (qp->sq_hdr_buf_size && sq->hwq.max_elements) {
+ if (qp->sq_hdr_buf_size && sq->max_wqe) {
qp->sq_hdr_buf = dma_alloc_coherent(&res->pdev->dev,
- sq->hwq.max_elements *
- qp->sq_hdr_buf_size,
+ sq->max_wqe * qp->sq_hdr_buf_size,
&qp->sq_hdr_buf_map, GFP_KERNEL);
if (!qp->sq_hdr_buf) {
rc = -ENOMEM;
@@ -212,9 +211,9 @@
}
}
- if (qp->rq_hdr_buf_size && rq->hwq.max_elements) {
+ if (qp->rq_hdr_buf_size && rq->max_wqe) {
qp->rq_hdr_buf = dma_alloc_coherent(&res->pdev->dev,
- rq->hwq.max_elements *
+ rq->max_wqe *
qp->rq_hdr_buf_size,
&qp->rq_hdr_buf_map,
GFP_KERNEL);
@@ -661,6 +660,7 @@
srq->dbinfo.hwq = &srq->hwq;
srq->dbinfo.xid = srq->id;
srq->dbinfo.db = srq->dpi->dbr;
+ srq->dbinfo.max_slot = 1;
srq->dbinfo.priv_db = res->dpi_tbl.dbr_bar_reg_iomem;
if (srq->threshold)
bnxt_qplib_armen_db(&srq->dbinfo, DBC_DBC_TYPE_SRQ_ARMENA);
@@ -784,6 +784,28 @@
}
/* QP */
+
+static int bnxt_qplib_alloc_init_swq(struct bnxt_qplib_q *que)
+{
+ int rc = 0;
+ int indx;
+
+ que->swq = kcalloc(que->max_wqe, sizeof(*que->swq), GFP_KERNEL);
+ if (!que->swq) {
+ rc = -ENOMEM;
+ goto out;
+ }
+
+ que->swq_start = 0;
+ que->swq_last = que->max_wqe - 1;
+ for (indx = 0; indx < que->max_wqe; indx++)
+ que->swq[indx].next_idx = indx + 1;
+ que->swq[que->swq_last].next_idx = 0; /* Make it circular */
+ que->swq_last = 0;
+out:
+ return rc;
+}
+
int bnxt_qplib_create_qp1(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp)
{
struct bnxt_qplib_hwq_attr hwq_attr = {};
@@ -808,71 +830,63 @@
/* SQ */
hwq_attr.res = res;
hwq_attr.sginfo = &sq->sg_info;
- hwq_attr.depth = sq->max_wqe;
- hwq_attr.stride = sq->wqe_size;
+ hwq_attr.stride = sizeof(struct sq_sge);
+ hwq_attr.depth = bnxt_qplib_get_depth(sq);
hwq_attr.type = HWQ_TYPE_QUEUE;
rc = bnxt_qplib_alloc_init_hwq(&sq->hwq, &hwq_attr);
if (rc)
goto exit;
- sq->swq = kcalloc(sq->hwq.max_elements, sizeof(*sq->swq), GFP_KERNEL);
- if (!sq->swq) {
- rc = -ENOMEM;
+ rc = bnxt_qplib_alloc_init_swq(sq);
+ if (rc)
goto fail_sq;
- }
+
+ req.sq_size = cpu_to_le32(bnxt_qplib_set_sq_size(sq, qp->wqe_mode));
pbl = &sq->hwq.pbl[PBL_LVL_0];
req.sq_pbl = cpu_to_le64(pbl->pg_map_arr[0]);
pg_sz_lvl = (bnxt_qplib_base_pg_size(&sq->hwq) <<
CMDQ_CREATE_QP1_SQ_PG_SIZE_SFT);
pg_sz_lvl |= (sq->hwq.level & CMDQ_CREATE_QP1_SQ_LVL_MASK);
req.sq_pg_size_sq_lvl = pg_sz_lvl;
+ req.sq_fwo_sq_sge =
+ cpu_to_le16((sq->max_sge & CMDQ_CREATE_QP1_SQ_SGE_MASK) <<
+ CMDQ_CREATE_QP1_SQ_SGE_SFT);
+ req.scq_cid = cpu_to_le32(qp->scq->id);
- if (qp->scq)
- req.scq_cid = cpu_to_le32(qp->scq->id);
/* RQ */
if (rq->max_wqe) {
hwq_attr.res = res;
hwq_attr.sginfo = &rq->sg_info;
- hwq_attr.stride = rq->wqe_size;
- hwq_attr.depth = qp->rq.max_wqe;
+ hwq_attr.stride = sizeof(struct sq_sge);
+ hwq_attr.depth = bnxt_qplib_get_depth(rq);
hwq_attr.type = HWQ_TYPE_QUEUE;
rc = bnxt_qplib_alloc_init_hwq(&rq->hwq, &hwq_attr);
if (rc)
- goto fail_sq;
-
- rq->swq = kcalloc(rq->hwq.max_elements, sizeof(*rq->swq),
- GFP_KERNEL);
- if (!rq->swq) {
- rc = -ENOMEM;
+ goto sq_swq;
+ rc = bnxt_qplib_alloc_init_swq(rq);
+ if (rc)
goto fail_rq;
- }
+ req.rq_size = cpu_to_le32(rq->max_wqe);
pbl = &rq->hwq.pbl[PBL_LVL_0];
req.rq_pbl = cpu_to_le64(pbl->pg_map_arr[0]);
pg_sz_lvl = (bnxt_qplib_base_pg_size(&rq->hwq) <<
CMDQ_CREATE_QP1_RQ_PG_SIZE_SFT);
pg_sz_lvl |= (rq->hwq.level & CMDQ_CREATE_QP1_RQ_LVL_MASK);
req.rq_pg_size_rq_lvl = pg_sz_lvl;
- if (qp->rcq)
- req.rcq_cid = cpu_to_le32(qp->rcq->id);
+ req.rq_fwo_rq_sge =
+ cpu_to_le16((rq->max_sge &
+ CMDQ_CREATE_QP1_RQ_SGE_MASK) <<
+ CMDQ_CREATE_QP1_RQ_SGE_SFT);
}
+ req.rcq_cid = cpu_to_le32(qp->rcq->id);
/* Header buffer - allow hdr_buf pass in */
rc = bnxt_qplib_alloc_qp_hdr_buf(res, qp);
if (rc) {
rc = -ENOMEM;
- goto fail;
+ goto rq_rwq;
}
qp_flags |= CMDQ_CREATE_QP1_QP_FLAGS_RESERVED_LKEY_ENABLE;
req.qp_flags = cpu_to_le32(qp_flags);
- req.sq_size = cpu_to_le32(sq->hwq.max_elements);
- req.rq_size = cpu_to_le32(rq->hwq.max_elements);
-
- req.sq_fwo_sq_sge =
- cpu_to_le16((sq->max_sge & CMDQ_CREATE_QP1_SQ_SGE_MASK) <<
- CMDQ_CREATE_QP1_SQ_SGE_SFT);
- req.rq_fwo_rq_sge =
- cpu_to_le16((rq->max_sge & CMDQ_CREATE_QP1_RQ_SGE_MASK) <<
- CMDQ_CREATE_QP1_RQ_SGE_SFT);
-
req.pd_id = cpu_to_le32(qp->pd->id);
rc = bnxt_qplib_rcfw_send_message(rcfw, (void *)&req,
@@ -886,10 +900,12 @@
sq->dbinfo.hwq = &sq->hwq;
sq->dbinfo.xid = qp->id;
sq->dbinfo.db = qp->dpi->dbr;
+ sq->dbinfo.max_slot = bnxt_qplib_set_sq_max_slot(qp->wqe_mode);
if (rq->max_wqe) {
rq->dbinfo.hwq = &rq->hwq;
rq->dbinfo.xid = qp->id;
rq->dbinfo.db = qp->dpi->dbr;
+ rq->dbinfo.max_slot = bnxt_qplib_set_rq_max_slot(rq->wqe_size);
}
rcfw->qp_tbl[qp->id].qp_id = qp->id;
rcfw->qp_tbl[qp->id].qp_handle = (void *)qp;
@@ -898,12 +914,14 @@
fail:
bnxt_qplib_free_qp_hdr_buf(res, qp);
+rq_rwq:
+ kfree(rq->swq);
fail_rq:
bnxt_qplib_free_hwq(res, &rq->hwq);
- kfree(rq->swq);
+sq_swq:
+ kfree(sq->swq);
fail_sq:
bnxt_qplib_free_hwq(res, &sq->hwq);
- kfree(sq->swq);
exit:
return rc;
}
@@ -912,26 +930,18 @@
{
struct bnxt_qplib_hwq *hwq;
struct bnxt_qplib_q *sq;
- u64 fpsne, psne, psn_pg;
- u16 indx_pad = 0, indx;
- u16 pg_num, pg_indx;
- u64 *page;
+ u64 fpsne, psn_pg;
+ u16 indx_pad = 0;
sq = &qp->sq;
hwq = &sq->hwq;
-
- fpsne = (u64)bnxt_qplib_get_qe(hwq, hwq->max_elements, &psn_pg);
+ fpsne = (u64)bnxt_qplib_get_qe(hwq, hwq->depth, &psn_pg);
if (!IS_ALIGNED(fpsne, PAGE_SIZE))
indx_pad = ALIGN(fpsne, PAGE_SIZE) / size;
- page = (u64 *)psn_pg;
- for (indx = 0; indx < hwq->max_elements; indx++) {
- pg_num = (indx + indx_pad) / (PAGE_SIZE / size);
- pg_indx = (indx + indx_pad) % (PAGE_SIZE / size);
- psne = page[pg_num] + pg_indx * size;
- sq->swq[indx].psn_ext = (struct sq_psn_search_ext *)psne;
- sq->swq[indx].psn_search = (struct sq_psn_search *)psne;
- }
+ hwq->pad_pgofft = indx_pad;
+ hwq->pad_pg = (u64 *)psn_pg;
+ hwq->pad_stride = size;
}
int bnxt_qplib_create_qp(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp)
@@ -944,12 +954,12 @@
struct creq_create_qp_resp resp;
int rc, req_size, psn_sz = 0;
struct bnxt_qplib_hwq *xrrq;
- u16 cmd_flags = 0, max_ssge;
struct bnxt_qplib_pbl *pbl;
struct cmdq_create_qp req;
+ u16 cmd_flags = 0;
u32 qp_flags = 0;
u8 pg_sz_lvl;
- u16 max_rsge;
+ u16 nsge;
RCFW_CMD_PREP(req, CREATE_QP, cmd_flags);
@@ -967,97 +977,78 @@
hwq_attr.res = res;
hwq_attr.sginfo = &sq->sg_info;
- hwq_attr.stride = sq->wqe_size;
- hwq_attr.depth = sq->max_wqe;
+ hwq_attr.stride = sizeof(struct sq_sge);
+ hwq_attr.depth = bnxt_qplib_get_depth(sq);
hwq_attr.aux_stride = psn_sz;
- hwq_attr.aux_depth = hwq_attr.depth;
+ hwq_attr.aux_depth = bnxt_qplib_set_sq_size(sq, qp->wqe_mode);
hwq_attr.type = HWQ_TYPE_QUEUE;
rc = bnxt_qplib_alloc_init_hwq(&sq->hwq, &hwq_attr);
if (rc)
goto exit;
- sq->swq = kcalloc(sq->hwq.max_elements, sizeof(*sq->swq), GFP_KERNEL);
- if (!sq->swq) {
- rc = -ENOMEM;
+ rc = bnxt_qplib_alloc_init_swq(sq);
+ if (rc)
goto fail_sq;
- }
if (psn_sz)
bnxt_qplib_init_psn_ptr(qp, psn_sz);
+ req.sq_size = cpu_to_le32(bnxt_qplib_set_sq_size(sq, qp->wqe_mode));
pbl = &sq->hwq.pbl[PBL_LVL_0];
req.sq_pbl = cpu_to_le64(pbl->pg_map_arr[0]);
pg_sz_lvl = (bnxt_qplib_base_pg_size(&sq->hwq) <<
CMDQ_CREATE_QP_SQ_PG_SIZE_SFT);
pg_sz_lvl |= (sq->hwq.level & CMDQ_CREATE_QP_SQ_LVL_MASK);
req.sq_pg_size_sq_lvl = pg_sz_lvl;
-
- if (qp->scq)
- req.scq_cid = cpu_to_le32(qp->scq->id);
+ req.sq_fwo_sq_sge =
+ cpu_to_le16(((sq->max_sge & CMDQ_CREATE_QP_SQ_SGE_MASK) <<
+ CMDQ_CREATE_QP_SQ_SGE_SFT) | 0);
+ req.scq_cid = cpu_to_le32(qp->scq->id);
/* RQ */
- if (rq->max_wqe) {
+ if (!qp->srq) {
hwq_attr.res = res;
hwq_attr.sginfo = &rq->sg_info;
- hwq_attr.stride = rq->wqe_size;
- hwq_attr.depth = rq->max_wqe;
+ hwq_attr.stride = sizeof(struct sq_sge);
+ hwq_attr.depth = bnxt_qplib_get_depth(rq);
hwq_attr.aux_stride = 0;
hwq_attr.aux_depth = 0;
hwq_attr.type = HWQ_TYPE_QUEUE;
rc = bnxt_qplib_alloc_init_hwq(&rq->hwq, &hwq_attr);
if (rc)
- goto fail_sq;
-
- rq->swq = kcalloc(rq->hwq.max_elements, sizeof(*rq->swq),
- GFP_KERNEL);
- if (!rq->swq) {
- rc = -ENOMEM;
+ goto sq_swq;
+ rc = bnxt_qplib_alloc_init_swq(rq);
+ if (rc)
goto fail_rq;
- }
+
+ req.rq_size = cpu_to_le32(rq->max_wqe);
pbl = &rq->hwq.pbl[PBL_LVL_0];
req.rq_pbl = cpu_to_le64(pbl->pg_map_arr[0]);
pg_sz_lvl = (bnxt_qplib_base_pg_size(&rq->hwq) <<
CMDQ_CREATE_QP_RQ_PG_SIZE_SFT);
pg_sz_lvl |= (rq->hwq.level & CMDQ_CREATE_QP_RQ_LVL_MASK);
req.rq_pg_size_rq_lvl = pg_sz_lvl;
+ nsge = (qp->wqe_mode == BNXT_QPLIB_WQE_MODE_STATIC) ?
+ 6 : rq->max_sge;
+ req.rq_fwo_rq_sge =
+ cpu_to_le16(((nsge &
+ CMDQ_CREATE_QP_RQ_SGE_MASK) <<
+ CMDQ_CREATE_QP_RQ_SGE_SFT) | 0);
} else {
/* SRQ */
- if (qp->srq) {
- qp_flags |= CMDQ_CREATE_QP_QP_FLAGS_SRQ_USED;
- req.srq_cid = cpu_to_le32(qp->srq->id);
- }
+ qp_flags |= CMDQ_CREATE_QP_QP_FLAGS_SRQ_USED;
+ req.srq_cid = cpu_to_le32(qp->srq->id);
}
-
- if (qp->rcq)
- req.rcq_cid = cpu_to_le32(qp->rcq->id);
+ req.rcq_cid = cpu_to_le32(qp->rcq->id);
qp_flags |= CMDQ_CREATE_QP_QP_FLAGS_RESERVED_LKEY_ENABLE;
qp_flags |= CMDQ_CREATE_QP_QP_FLAGS_FR_PMR_ENABLED;
if (qp->sig_type)
qp_flags |= CMDQ_CREATE_QP_QP_FLAGS_FORCE_COMPLETION;
+ if (qp->wqe_mode == BNXT_QPLIB_WQE_MODE_VARIABLE)
+ qp_flags |= CMDQ_CREATE_QP_QP_FLAGS_VARIABLE_SIZED_WQE_ENABLED;
req.qp_flags = cpu_to_le32(qp_flags);
- req.sq_size = cpu_to_le32(sq->hwq.max_elements);
- req.rq_size = cpu_to_le32(rq->hwq.max_elements);
- qp->sq_hdr_buf = NULL;
- qp->rq_hdr_buf = NULL;
-
- rc = bnxt_qplib_alloc_qp_hdr_buf(res, qp);
- if (rc)
- goto fail_rq;
-
- /* CTRL-22434: Irrespective of the requested SGE count on the SQ
- * always create the QP with max send sges possible if the requested
- * inline size is greater than 0.
- */
- max_ssge = qp->max_inline_data ? 6 : sq->max_sge;
- req.sq_fwo_sq_sge = cpu_to_le16(
- ((max_ssge & CMDQ_CREATE_QP_SQ_SGE_MASK)
- << CMDQ_CREATE_QP_SQ_SGE_SFT) | 0);
- max_rsge = bnxt_qplib_is_chip_gen_p5(res->cctx) ? 6 : rq->max_sge;
- req.rq_fwo_rq_sge = cpu_to_le16(
- ((max_rsge & CMDQ_CREATE_QP_RQ_SGE_MASK)
- << CMDQ_CREATE_QP_RQ_SGE_SFT) | 0);
/* ORRQ and IRRQ */
if (psn_sz) {
xrrq = &qp->orrq;
@@ -1078,7 +1069,7 @@
hwq_attr.type = HWQ_TYPE_CTX;
rc = bnxt_qplib_alloc_init_hwq(xrrq, &hwq_attr);
if (rc)
- goto fail_buf_free;
+ goto rq_swq;
pbl = &xrrq->pbl[PBL_LVL_0];
req.orrq_addr = cpu_to_le64(pbl->pg_map_arr[0]);
@@ -1113,30 +1104,29 @@
sq->dbinfo.hwq = &sq->hwq;
sq->dbinfo.xid = qp->id;
sq->dbinfo.db = qp->dpi->dbr;
+ sq->dbinfo.max_slot = bnxt_qplib_set_sq_max_slot(qp->wqe_mode);
if (rq->max_wqe) {
rq->dbinfo.hwq = &rq->hwq;
rq->dbinfo.xid = qp->id;
rq->dbinfo.db = qp->dpi->dbr;
+ rq->dbinfo.max_slot = bnxt_qplib_set_rq_max_slot(rq->wqe_size);
}
rcfw->qp_tbl[qp->id].qp_id = qp->id;
rcfw->qp_tbl[qp->id].qp_handle = (void *)qp;
return 0;
-
fail:
- if (qp->irrq.max_elements)
- bnxt_qplib_free_hwq(res, &qp->irrq);
+ bnxt_qplib_free_hwq(res, &qp->irrq);
fail_orrq:
- if (qp->orrq.max_elements)
- bnxt_qplib_free_hwq(res, &qp->orrq);
-fail_buf_free:
- bnxt_qplib_free_qp_hdr_buf(res, qp);
+ bnxt_qplib_free_hwq(res, &qp->orrq);
+rq_swq:
+ kfree(rq->swq);
fail_rq:
bnxt_qplib_free_hwq(res, &rq->hwq);
- kfree(rq->swq);
+sq_swq:
+ kfree(sq->swq);
fail_sq:
bnxt_qplib_free_hwq(res, &sq->hwq);
- kfree(sq->swq);
exit:
return rc;
}
@@ -1512,7 +1502,7 @@
memset(sge, 0, sizeof(*sge));
if (qp->sq_hdr_buf) {
- sw_prod = HWQ_CMP(sq->hwq.prod, &sq->hwq);
+ sw_prod = sq->swq_start;
sge->addr = (dma_addr_t)(qp->sq_hdr_buf_map +
sw_prod * qp->sq_hdr_buf_size);
sge->lkey = 0xFFFFFFFF;
@@ -1526,7 +1516,7 @@
{
struct bnxt_qplib_q *rq = &qp->rq;
- return HWQ_CMP(rq->hwq.prod, &rq->hwq);
+ return rq->swq_start;
}
dma_addr_t bnxt_qplib_get_qp_buf_from_index(struct bnxt_qplib_qp *qp, u32 index)
@@ -1543,7 +1533,7 @@
memset(sge, 0, sizeof(*sge));
if (qp->rq_hdr_buf) {
- sw_prod = HWQ_CMP(rq->hwq.prod, &rq->hwq);
+ sw_prod = rq->swq_start;
sge->addr = (dma_addr_t)(qp->rq_hdr_buf_map +
sw_prod * qp->rq_hdr_buf_size);
sge->lkey = 0xFFFFFFFF;
@@ -1562,6 +1552,8 @@
u32 flg_npsn;
u32 op_spsn;
+ if (!swq->psn_search)
+ return;
psns = swq->psn_search;
psns_ext = swq->psn_ext;
@@ -1575,12 +1567,122 @@
if (bnxt_qplib_is_chip_gen_p5(qp->cctx)) {
psns_ext->opcode_start_psn = cpu_to_le32(op_spsn);
psns_ext->flags_next_psn = cpu_to_le32(flg_npsn);
+ psns_ext->start_slot_idx = cpu_to_le16(swq->slot_idx);
} else {
psns->opcode_start_psn = cpu_to_le32(op_spsn);
psns->flags_next_psn = cpu_to_le32(flg_npsn);
}
}
+static int bnxt_qplib_put_inline(struct bnxt_qplib_qp *qp,
+ struct bnxt_qplib_swqe *wqe,
+ u16 *idx)
+{
+ struct bnxt_qplib_hwq *hwq;
+ int len, t_len, offt;
+ bool pull_dst = true;
+ void *il_dst = NULL;
+ void *il_src = NULL;
+ int t_cplen, cplen;
+ int indx;
+
+ hwq = &qp->sq.hwq;
+ t_len = 0;
+ for (indx = 0; indx < wqe->num_sge; indx++) {
+ len = wqe->sg_list[indx].size;
+ il_src = (void *)wqe->sg_list[indx].addr;
+ t_len += len;
+ if (t_len > qp->max_inline_data)
+ goto bad;
+ while (len) {
+ if (pull_dst) {
+ pull_dst = false;
+ il_dst = bnxt_qplib_get_prod_qe(hwq, *idx);
+ (*idx)++;
+ t_cplen = 0;
+ offt = 0;
+ }
+ cplen = min_t(int, len, sizeof(struct sq_sge));
+ cplen = min_t(int, cplen,
+ (sizeof(struct sq_sge) - offt));
+ memcpy(il_dst, il_src, cplen);
+ t_cplen += cplen;
+ il_src += cplen;
+ il_dst += cplen;
+ offt += cplen;
+ len -= cplen;
+ if (t_cplen == sizeof(struct sq_sge))
+ pull_dst = true;
+ }
+ }
+
+ return t_len;
+bad:
+ return -ENOMEM;
+}
+
+static u32 bnxt_qplib_put_sges(struct bnxt_qplib_hwq *hwq,
+ struct bnxt_qplib_sge *ssge,
+ u16 nsge, u16 *idx)
+{
+ struct sq_sge *dsge;
+ int indx, len = 0;
+
+ for (indx = 0; indx < nsge; indx++, (*idx)++) {
+ dsge = bnxt_qplib_get_prod_qe(hwq, *idx);
+ dsge->va_or_pa = cpu_to_le64(ssge[indx].addr);
+ dsge->l_key = cpu_to_le32(ssge[indx].lkey);
+ dsge->size = cpu_to_le32(ssge[indx].size);
+ len += ssge[indx].size;
+ }
+
+ return len;
+}
+
+static u16 bnxt_qplib_required_slots(struct bnxt_qplib_qp *qp,
+ struct bnxt_qplib_swqe *wqe,
+ u16 *wqe_sz, u16 *qdf, u8 mode)
+{
+ u32 ilsize, bytes;
+ u16 nsge;
+ u16 slot;
+
+ nsge = wqe->num_sge;
+ /* Adding sq_send_hdr is a misnomer, for rq also hdr size is same. */
+ bytes = sizeof(struct sq_send_hdr) + nsge * sizeof(struct sq_sge);
+ if (wqe->flags & BNXT_QPLIB_SWQE_FLAGS_INLINE) {
+ ilsize = bnxt_qplib_calc_ilsize(wqe, qp->max_inline_data);
+ bytes = ALIGN(ilsize, sizeof(struct sq_sge));
+ bytes += sizeof(struct sq_send_hdr);
+ }
+
+ *qdf = __xlate_qfd(qp->sq.q_full_delta, bytes);
+ slot = bytes >> 4;
+ *wqe_sz = slot;
+ if (mode == BNXT_QPLIB_WQE_MODE_STATIC)
+ slot = 8;
+ return slot;
+}
+
+static void bnxt_qplib_pull_psn_buff(struct bnxt_qplib_q *sq,
+ struct bnxt_qplib_swq *swq)
+{
+ struct bnxt_qplib_hwq *hwq;
+ u32 pg_num, pg_indx;
+ void *buff;
+ u32 tail;
+
+ hwq = &sq->hwq;
+ if (!hwq->pad_pg)
+ return;
+ tail = swq->slot_idx / sq->dbinfo.max_slot;
+ pg_num = (tail + hwq->pad_pgofft) / (PAGE_SIZE / hwq->pad_stride);
+ pg_indx = (tail + hwq->pad_pgofft) % (PAGE_SIZE / hwq->pad_stride);
+ buff = (void *)(hwq->pad_pg[pg_num] + pg_indx * hwq->pad_stride);
+ swq->psn_ext = buff;
+ swq->psn_search = buff;
+}
+
void bnxt_qplib_post_send_db(struct bnxt_qplib_qp *qp)
{
struct bnxt_qplib_q *sq = &qp->sq;
@@ -1594,88 +1696,84 @@
struct bnxt_qplib_nq_work *nq_work = NULL;
int i, rc = 0, data_len = 0, pkt_num = 0;
struct bnxt_qplib_q *sq = &qp->sq;
- struct sq_send *hw_sq_send_hdr;
+ struct bnxt_qplib_hwq *hwq;
struct bnxt_qplib_swq *swq;
bool sch_handler = false;
- struct sq_sge *hw_sge;
- u8 wqe_size16;
+ u16 wqe_sz, qdf = 0;
+ void *base_hdr;
+ void *ext_hdr;
__le32 temp32;
- u32 sw_prod;
+ u32 wqe_idx;
+ u32 slots;
+ u16 idx;
- if (qp->state != CMDQ_MODIFY_QP_NEW_STATE_RTS) {
- if (qp->state == CMDQ_MODIFY_QP_NEW_STATE_ERR) {
- sch_handler = true;
- dev_dbg(&sq->hwq.pdev->dev,
- "%s Error QP. Scheduling for poll_cq\n",
- __func__);
- goto queue_err;
- }
+ hwq = &sq->hwq;
+ if (qp->state != CMDQ_MODIFY_QP_NEW_STATE_RTS &&
+ qp->state != CMDQ_MODIFY_QP_NEW_STATE_ERR) {
+ dev_err(&hwq->pdev->dev,
+ "QPLIB: FP: QP (0x%x) is in the 0x%x state",
+ qp->id, qp->state);
+ rc = -EINVAL;
+ goto done;
}
- if (bnxt_qplib_queue_full(sq)) {
- dev_err(&sq->hwq.pdev->dev,
+ slots = bnxt_qplib_required_slots(qp, wqe, &wqe_sz, &qdf, qp->wqe_mode);
+ if (bnxt_qplib_queue_full(sq, slots + qdf)) {
+ dev_err(&hwq->pdev->dev,
"prod = %#x cons = %#x qdepth = %#x delta = %#x\n",
- sq->hwq.prod, sq->hwq.cons, sq->hwq.max_elements,
- sq->q_full_delta);
+ hwq->prod, hwq->cons, hwq->depth, sq->q_full_delta);
rc = -ENOMEM;
goto done;
}
- sw_prod = HWQ_CMP(sq->hwq.prod, &sq->hwq);
- swq = &sq->swq[sw_prod];
+
+ swq = bnxt_qplib_get_swqe(sq, &wqe_idx);
+ bnxt_qplib_pull_psn_buff(sq, swq);
+
+ idx = 0;
+ swq->slot_idx = hwq->prod;
+ swq->slots = slots;
swq->wr_id = wqe->wr_id;
swq->type = wqe->type;
swq->flags = wqe->flags;
+ swq->start_psn = sq->psn & BTH_PSN_MASK;
if (qp->sig_type)
swq->flags |= SQ_SEND_FLAGS_SIGNAL_COMP;
- swq->start_psn = sq->psn & BTH_PSN_MASK;
- hw_sq_send_hdr = bnxt_qplib_get_qe(&sq->hwq, sw_prod, NULL);
- memset(hw_sq_send_hdr, 0, sq->wqe_size);
-
- if (wqe->flags & BNXT_QPLIB_SWQE_FLAGS_INLINE) {
- /* Copy the inline data */
- if (wqe->inline_len > BNXT_QPLIB_SWQE_MAX_INLINE_LENGTH) {
- dev_warn(&sq->hwq.pdev->dev,
- "Inline data length > 96 detected\n");
- data_len = BNXT_QPLIB_SWQE_MAX_INLINE_LENGTH;
- } else {
- data_len = wqe->inline_len;
- }
- memcpy(hw_sq_send_hdr->data, wqe->inline_data, data_len);
- wqe_size16 = (data_len + 15) >> 4;
- } else {
- for (i = 0, hw_sge = (struct sq_sge *)hw_sq_send_hdr->data;
- i < wqe->num_sge; i++, hw_sge++) {
- hw_sge->va_or_pa = cpu_to_le64(wqe->sg_list[i].addr);
- hw_sge->l_key = cpu_to_le32(wqe->sg_list[i].lkey);
- hw_sge->size = cpu_to_le32(wqe->sg_list[i].size);
- data_len += wqe->sg_list[i].size;
- }
- /* Each SGE entry = 1 WQE size16 */
- wqe_size16 = wqe->num_sge;
- /* HW requires wqe size has room for atleast one SGE even if
- * none was supplied by ULP
- */
- if (!wqe->num_sge)
- wqe_size16++;
+ if (qp->state == CMDQ_MODIFY_QP_NEW_STATE_ERR) {
+ sch_handler = true;
+ dev_dbg(&hwq->pdev->dev,
+ "%s Error QP. Scheduling for poll_cq\n", __func__);
+ goto queue_err;
}
+ base_hdr = bnxt_qplib_get_prod_qe(hwq, idx++);
+ ext_hdr = bnxt_qplib_get_prod_qe(hwq, idx++);
+ memset(base_hdr, 0, sizeof(struct sq_sge));
+ memset(ext_hdr, 0, sizeof(struct sq_sge));
+
+ if (wqe->flags & BNXT_QPLIB_SWQE_FLAGS_INLINE)
+ /* Copy the inline data */
+ data_len = bnxt_qplib_put_inline(qp, wqe, &idx);
+ else
+ data_len = bnxt_qplib_put_sges(hwq, wqe->sg_list, wqe->num_sge,
+ &idx);
+ if (data_len < 0)
+ goto queue_err;
/* Specifics */
switch (wqe->type) {
case BNXT_QPLIB_SWQE_TYPE_SEND:
if (qp->type == CMDQ_CREATE_QP1_TYPE_GSI) {
+ struct sq_send_raweth_qp1_hdr *sqe = base_hdr;
+ struct sq_raw_ext_hdr *ext_sqe = ext_hdr;
/* Assemble info for Raw Ethertype QPs */
- struct sq_send_raweth_qp1 *sqe =
- (struct sq_send_raweth_qp1 *)hw_sq_send_hdr;
sqe->wqe_type = wqe->type;
sqe->flags = wqe->flags;
- sqe->wqe_size = wqe_size16 +
- ((offsetof(typeof(*sqe), data) + 15) >> 4);
+ sqe->wqe_size = wqe_sz;
sqe->cfa_action = cpu_to_le16(wqe->rawqp1.cfa_action);
sqe->lflags = cpu_to_le16(wqe->rawqp1.lflags);
sqe->length = cpu_to_le32(data_len);
- sqe->cfa_meta = cpu_to_le32((wqe->rawqp1.cfa_meta &
+ ext_sqe->cfa_meta = cpu_to_le32((wqe->rawqp1.cfa_meta &
SQ_SEND_RAWETH_QP1_CFA_META_VLAN_VID_MASK) <<
SQ_SEND_RAWETH_QP1_CFA_META_VLAN_VID_SFT);
@@ -1685,27 +1783,24 @@
case BNXT_QPLIB_SWQE_TYPE_SEND_WITH_IMM:
case BNXT_QPLIB_SWQE_TYPE_SEND_WITH_INV:
{
- struct sq_send *sqe = (struct sq_send *)hw_sq_send_hdr;
+ struct sq_ud_ext_hdr *ext_sqe = ext_hdr;
+ struct sq_send_hdr *sqe = base_hdr;
sqe->wqe_type = wqe->type;
sqe->flags = wqe->flags;
- sqe->wqe_size = wqe_size16 +
- ((offsetof(typeof(*sqe), data) + 15) >> 4);
- sqe->inv_key_or_imm_data = cpu_to_le32(
- wqe->send.inv_key);
+ sqe->wqe_size = wqe_sz;
+ sqe->inv_key_or_imm_data = cpu_to_le32(wqe->send.inv_key);
if (qp->type == CMDQ_CREATE_QP_TYPE_UD ||
qp->type == CMDQ_CREATE_QP_TYPE_GSI) {
sqe->q_key = cpu_to_le32(wqe->send.q_key);
- sqe->dst_qp = cpu_to_le32(
- wqe->send.dst_qp & SQ_SEND_DST_QP_MASK);
sqe->length = cpu_to_le32(data_len);
- sqe->avid = cpu_to_le32(wqe->send.avid &
- SQ_SEND_AVID_MASK);
sq->psn = (sq->psn + 1) & BTH_PSN_MASK;
+ ext_sqe->dst_qp = cpu_to_le32(wqe->send.dst_qp &
+ SQ_SEND_DST_QP_MASK);
+ ext_sqe->avid = cpu_to_le32(wqe->send.avid &
+ SQ_SEND_AVID_MASK);
} else {
sqe->length = cpu_to_le32(data_len);
- sqe->dst_qp = 0;
- sqe->avid = 0;
if (qp->mtu)
pkt_num = (data_len + qp->mtu - 1) / qp->mtu;
if (!pkt_num)
@@ -1718,16 +1813,16 @@
case BNXT_QPLIB_SWQE_TYPE_RDMA_WRITE_WITH_IMM:
case BNXT_QPLIB_SWQE_TYPE_RDMA_READ:
{
- struct sq_rdma *sqe = (struct sq_rdma *)hw_sq_send_hdr;
+ struct sq_rdma_ext_hdr *ext_sqe = ext_hdr;
+ struct sq_rdma_hdr *sqe = base_hdr;
sqe->wqe_type = wqe->type;
sqe->flags = wqe->flags;
- sqe->wqe_size = wqe_size16 +
- ((offsetof(typeof(*sqe), data) + 15) >> 4);
+ sqe->wqe_size = wqe_sz;
sqe->imm_data = cpu_to_le32(wqe->rdma.inv_key);
sqe->length = cpu_to_le32((u32)data_len);
- sqe->remote_va = cpu_to_le64(wqe->rdma.remote_va);
- sqe->remote_key = cpu_to_le32(wqe->rdma.r_key);
+ ext_sqe->remote_va = cpu_to_le64(wqe->rdma.remote_va);
+ ext_sqe->remote_key = cpu_to_le32(wqe->rdma.r_key);
if (qp->mtu)
pkt_num = (data_len + qp->mtu - 1) / qp->mtu;
if (!pkt_num)
@@ -1738,14 +1833,15 @@
case BNXT_QPLIB_SWQE_TYPE_ATOMIC_CMP_AND_SWP:
case BNXT_QPLIB_SWQE_TYPE_ATOMIC_FETCH_AND_ADD:
{
- struct sq_atomic *sqe = (struct sq_atomic *)hw_sq_send_hdr;
+ struct sq_atomic_ext_hdr *ext_sqe = ext_hdr;
+ struct sq_atomic_hdr *sqe = base_hdr;
sqe->wqe_type = wqe->type;
sqe->flags = wqe->flags;
sqe->remote_key = cpu_to_le32(wqe->atomic.r_key);
sqe->remote_va = cpu_to_le64(wqe->atomic.remote_va);
- sqe->swap_data = cpu_to_le64(wqe->atomic.swap_data);
- sqe->cmp_data = cpu_to_le64(wqe->atomic.cmp_data);
+ ext_sqe->swap_data = cpu_to_le64(wqe->atomic.swap_data);
+ ext_sqe->cmp_data = cpu_to_le64(wqe->atomic.cmp_data);
if (qp->mtu)
pkt_num = (data_len + qp->mtu - 1) / qp->mtu;
if (!pkt_num)
@@ -1755,8 +1851,7 @@
}
case BNXT_QPLIB_SWQE_TYPE_LOCAL_INV:
{
- struct sq_localinvalidate *sqe =
- (struct sq_localinvalidate *)hw_sq_send_hdr;
+ struct sq_localinvalidate *sqe = base_hdr;
sqe->wqe_type = wqe->type;
sqe->flags = wqe->flags;
@@ -1766,7 +1861,8 @@
}
case BNXT_QPLIB_SWQE_TYPE_FAST_REG_MR:
{
- struct sq_fr_pmr *sqe = (struct sq_fr_pmr *)hw_sq_send_hdr;
+ struct sq_fr_pmr_ext_hdr *ext_sqe = ext_hdr;
+ struct sq_fr_pmr_hdr *sqe = base_hdr;
sqe->wqe_type = wqe->type;
sqe->flags = wqe->flags;
@@ -1790,14 +1886,15 @@
wqe->frmr.pbl_ptr[i] = cpu_to_le64(
wqe->frmr.page_list[i] |
PTU_PTE_VALID);
- sqe->pblptr = cpu_to_le64(wqe->frmr.pbl_dma_ptr);
- sqe->va = cpu_to_le64(wqe->frmr.va);
+ ext_sqe->pblptr = cpu_to_le64(wqe->frmr.pbl_dma_ptr);
+ ext_sqe->va = cpu_to_le64(wqe->frmr.va);
break;
}
case BNXT_QPLIB_SWQE_TYPE_BIND_MW:
{
- struct sq_bind *sqe = (struct sq_bind *)hw_sq_send_hdr;
+ struct sq_bind_ext_hdr *ext_sqe = ext_hdr;
+ struct sq_bind_hdr *sqe = base_hdr;
sqe->wqe_type = wqe->type;
sqe->flags = wqe->flags;
@@ -1806,9 +1903,8 @@
(wqe->bind.zero_based ? SQ_BIND_ZERO_BASED : 0);
sqe->parent_l_key = cpu_to_le32(wqe->bind.parent_l_key);
sqe->l_key = cpu_to_le32(wqe->bind.r_key);
- sqe->va = cpu_to_le64(wqe->bind.va);
- temp32 = cpu_to_le32(wqe->bind.length);
- memcpy(&sqe->length, &temp32, sizeof(wqe->bind.length));
+ ext_sqe->va = cpu_to_le64(wqe->bind.va);
+ ext_sqe->length_lo = cpu_to_le32(wqe->bind.length);
break;
}
default:
@@ -1817,23 +1913,11 @@
goto done;
}
swq->next_psn = sq->psn & BTH_PSN_MASK;
- if (qp->type == CMDQ_CREATE_QP_TYPE_RC)
- bnxt_qplib_fill_psn_search(qp, wqe, swq);
+ bnxt_qplib_fill_psn_search(qp, wqe, swq);
queue_err:
- if (sch_handler) {
- /* Store the ULP info in the software structures */
- sw_prod = HWQ_CMP(sq->hwq.prod, &sq->hwq);
- swq = &sq->swq[sw_prod];
- swq->wr_id = wqe->wr_id;
- swq->type = wqe->type;
- swq->flags = wqe->flags;
- if (qp->sig_type)
- swq->flags |= SQ_SEND_FLAGS_SIGNAL_COMP;
- swq->start_psn = sq->psn & BTH_PSN_MASK;
- }
- sq->hwq.prod++;
+ bnxt_qplib_swq_mod_start(sq, wqe_idx);
+ bnxt_qplib_hwq_incr_prod(hwq, swq->slots);
qp->wqe_cnt++;
-
done:
if (sch_handler) {
nq_work = kzalloc(sizeof(*nq_work), GFP_ATOMIC);
@@ -1843,7 +1927,7 @@
INIT_WORK(&nq_work->work, bnxt_qpn_cqn_sched_task);
queue_work(qp->scq->nq->cqn_wq, &nq_work->work);
} else {
- dev_err(&sq->hwq.pdev->dev,
+ dev_err(&hwq->pdev->dev,
"FP: Failed to allocate SQ nq_work!\n");
rc = -ENOMEM;
}
@@ -1863,58 +1947,65 @@
{
struct bnxt_qplib_nq_work *nq_work = NULL;
struct bnxt_qplib_q *rq = &qp->rq;
+ struct rq_wqe_hdr *base_hdr;
+ struct rq_ext_hdr *ext_hdr;
+ struct bnxt_qplib_hwq *hwq;
+ struct bnxt_qplib_swq *swq;
bool sch_handler = false;
- struct sq_sge *hw_sge;
- struct rq_wqe *rqe;
- int i, rc = 0;
- u32 sw_prod;
+ u16 wqe_sz, idx;
+ u32 wqe_idx;
+ int rc = 0;
- if (qp->state == CMDQ_MODIFY_QP_NEW_STATE_ERR) {
- sch_handler = true;
- dev_dbg(&rq->hwq.pdev->dev,
- "%s: Error QP. Scheduling for poll_cq\n", __func__);
- goto queue_err;
+ hwq = &rq->hwq;
+ if (qp->state == CMDQ_MODIFY_QP_NEW_STATE_RESET) {
+ dev_err(&hwq->pdev->dev,
+ "QPLIB: FP: QP (0x%x) is in the 0x%x state",
+ qp->id, qp->state);
+ rc = -EINVAL;
+ goto done;
}
- if (bnxt_qplib_queue_full(rq)) {
- dev_err(&rq->hwq.pdev->dev,
+
+ if (bnxt_qplib_queue_full(rq, rq->dbinfo.max_slot)) {
+ dev_err(&hwq->pdev->dev,
"FP: QP (0x%x) RQ is full!\n", qp->id);
rc = -EINVAL;
goto done;
}
- sw_prod = HWQ_CMP(rq->hwq.prod, &rq->hwq);
- rq->swq[sw_prod].wr_id = wqe->wr_id;
- rqe = bnxt_qplib_get_qe(&rq->hwq, sw_prod, NULL);
- memset(rqe, 0, rq->wqe_size);
+ swq = bnxt_qplib_get_swqe(rq, &wqe_idx);
+ swq->wr_id = wqe->wr_id;
+ swq->slots = rq->dbinfo.max_slot;
- /* Calculate wqe_size16 and data_len */
- for (i = 0, hw_sge = (struct sq_sge *)rqe->data;
- i < wqe->num_sge; i++, hw_sge++) {
- hw_sge->va_or_pa = cpu_to_le64(wqe->sg_list[i].addr);
- hw_sge->l_key = cpu_to_le32(wqe->sg_list[i].lkey);
- hw_sge->size = cpu_to_le32(wqe->sg_list[i].size);
+ if (qp->state == CMDQ_MODIFY_QP_NEW_STATE_ERR) {
+ sch_handler = true;
+ dev_dbg(&hwq->pdev->dev,
+ "%s: Error QP. Scheduling for poll_cq\n", __func__);
+ goto queue_err;
}
- rqe->wqe_type = wqe->type;
- rqe->flags = wqe->flags;
- rqe->wqe_size = wqe->num_sge +
- ((offsetof(typeof(*rqe), data) + 15) >> 4);
- /* HW requires wqe size has room for atleast one SGE even if none
- * was supplied by ULP
- */
- if (!wqe->num_sge)
- rqe->wqe_size++;
- /* Supply the rqe->wr_id index to the wr_id_tbl for now */
- rqe->wr_id[0] = cpu_to_le32(sw_prod);
+ idx = 0;
+ base_hdr = bnxt_qplib_get_prod_qe(hwq, idx++);
+ ext_hdr = bnxt_qplib_get_prod_qe(hwq, idx++);
+ memset(base_hdr, 0, sizeof(struct sq_sge));
+ memset(ext_hdr, 0, sizeof(struct sq_sge));
+ wqe_sz = (sizeof(struct rq_wqe_hdr) +
+ wqe->num_sge * sizeof(struct sq_sge)) >> 4;
+ bnxt_qplib_put_sges(hwq, wqe->sg_list, wqe->num_sge, &idx);
+ if (!wqe->num_sge) {
+ struct sq_sge *sge;
+ sge = bnxt_qplib_get_prod_qe(hwq, idx++);
+ sge->size = 0;
+ wqe_sz++;
+ }
+ base_hdr->wqe_type = wqe->type;
+ base_hdr->flags = wqe->flags;
+ base_hdr->wqe_size = wqe_sz;
+ base_hdr->wr_id[0] = cpu_to_le32(wqe_idx);
queue_err:
- if (sch_handler) {
- /* Store the ULP info in the software structures */
- sw_prod = HWQ_CMP(rq->hwq.prod, &rq->hwq);
- rq->swq[sw_prod].wr_id = wqe->wr_id;
- }
-
- rq->hwq.prod++;
+ bnxt_qplib_swq_mod_start(rq, wqe_idx);
+ bnxt_qplib_hwq_incr_prod(hwq, swq->slots);
+done:
if (sch_handler) {
nq_work = kzalloc(sizeof(*nq_work), GFP_ATOMIC);
if (nq_work) {
@@ -1923,12 +2014,12 @@
INIT_WORK(&nq_work->work, bnxt_qpn_cqn_sched_task);
queue_work(qp->rcq->nq->cqn_wq, &nq_work->work);
} else {
- dev_err(&rq->hwq.pdev->dev,
+ dev_err(&hwq->pdev->dev,
"FP: Failed to allocate RQ nq_work!\n");
rc = -ENOMEM;
}
}
-done:
+
return rc;
}
@@ -2026,20 +2117,19 @@
static int __flush_sq(struct bnxt_qplib_q *sq, struct bnxt_qplib_qp *qp,
struct bnxt_qplib_cqe **pcqe, int *budget)
{
- u32 sw_prod, sw_cons;
struct bnxt_qplib_cqe *cqe;
+ u32 start, last;
int rc = 0;
/* Now complete all outstanding SQEs with FLUSHED_ERR */
- sw_prod = HWQ_CMP(sq->hwq.prod, &sq->hwq);
+ start = sq->swq_start;
cqe = *pcqe;
while (*budget) {
- sw_cons = HWQ_CMP(sq->hwq.cons, &sq->hwq);
- if (sw_cons == sw_prod) {
+ last = sq->swq_last;
+ if (start == last)
break;
- }
/* Skip the FENCE WQE completions */
- if (sq->swq[sw_cons].wr_id == BNXT_QPLIB_FENCE_WRID) {
+ if (sq->swq[last].wr_id == BNXT_QPLIB_FENCE_WRID) {
bnxt_qplib_cancel_phantom_processing(qp);
goto skip_compl;
}
@@ -2047,16 +2137,17 @@
cqe->status = CQ_REQ_STATUS_WORK_REQUEST_FLUSHED_ERR;
cqe->opcode = CQ_BASE_CQE_TYPE_REQ;
cqe->qp_handle = (u64)(unsigned long)qp;
- cqe->wr_id = sq->swq[sw_cons].wr_id;
+ cqe->wr_id = sq->swq[last].wr_id;
cqe->src_qp = qp->id;
- cqe->type = sq->swq[sw_cons].type;
+ cqe->type = sq->swq[last].type;
cqe++;
(*budget)--;
skip_compl:
- sq->hwq.cons++;
+ bnxt_qplib_hwq_incr_cons(&sq->hwq, sq->swq[last].slots);
+ sq->swq_last = sq->swq[last].next_idx;
}
*pcqe = cqe;
- if (!(*budget) && HWQ_CMP(sq->hwq.cons, &sq->hwq) != sw_prod)
+ if (!(*budget) && sq->swq_last != start)
/* Out of budget */
rc = -EAGAIN;
@@ -2067,9 +2158,9 @@
struct bnxt_qplib_cqe **pcqe, int *budget)
{
struct bnxt_qplib_cqe *cqe;
- u32 sw_prod, sw_cons;
- int rc = 0;
+ u32 start, last;
int opcode = 0;
+ int rc = 0;
switch (qp->type) {
case CMDQ_CREATE_QP1_TYPE_GSI:
@@ -2085,24 +2176,25 @@
}
/* Flush the rest of the RQ */
- sw_prod = HWQ_CMP(rq->hwq.prod, &rq->hwq);
+ start = rq->swq_start;
cqe = *pcqe;
while (*budget) {
- sw_cons = HWQ_CMP(rq->hwq.cons, &rq->hwq);
- if (sw_cons == sw_prod)
+ last = rq->swq_last;
+ if (last == start)
break;
memset(cqe, 0, sizeof(*cqe));
cqe->status =
CQ_RES_RC_STATUS_WORK_REQUEST_FLUSHED_ERR;
cqe->opcode = opcode;
cqe->qp_handle = (unsigned long)qp;
- cqe->wr_id = rq->swq[sw_cons].wr_id;
+ cqe->wr_id = rq->swq[last].wr_id;
cqe++;
(*budget)--;
- rq->hwq.cons++;
+ bnxt_qplib_hwq_incr_cons(&rq->hwq, rq->swq[last].slots);
+ rq->swq_last = rq->swq[last].next_idx;
}
*pcqe = cqe;
- if (!*budget && HWQ_CMP(rq->hwq.cons, &rq->hwq) != sw_prod)
+ if (!*budget && rq->swq_last != start)
/* Out of budget */
rc = -EAGAIN;
@@ -2125,7 +2217,7 @@
* CQE is track from sw_cq_cons to max_element but valid only if VALID=1
*/
static int do_wa9060(struct bnxt_qplib_qp *qp, struct bnxt_qplib_cq *cq,
- u32 cq_cons, u32 sw_sq_cons, u32 cqe_sq_cons)
+ u32 cq_cons, u32 swq_last, u32 cqe_sq_cons)
{
u32 peek_sw_cq_cons, peek_raw_cq_cons, peek_sq_cons_idx;
struct bnxt_qplib_q *sq = &qp->sq;
@@ -2138,7 +2230,7 @@
/* Normal mode */
/* Check for the psn_search marking before completing */
- swq = &sq->swq[sw_sq_cons];
+ swq = &sq->swq[swq_last];
if (swq->psn_search &&
le32_to_cpu(swq->psn_search->flags_next_psn) & 0x80000000) {
/* Unmark */
@@ -2147,7 +2239,7 @@
& ~0x80000000);
dev_dbg(&cq->hwq.pdev->dev,
"FP: Process Req cq_cons=0x%x qp=0x%x sq cons sw=0x%x cqe=0x%x marked!\n",
- cq_cons, qp->id, sw_sq_cons, cqe_sq_cons);
+ cq_cons, qp->id, swq_last, cqe_sq_cons);
sq->condition = true;
sq->send_phantom = true;
@@ -2184,9 +2276,10 @@
le64_to_cpu
(peek_req_hwcqe->qp_handle));
peek_sq = &peek_qp->sq;
- peek_sq_cons_idx = HWQ_CMP(le16_to_cpu(
- peek_req_hwcqe->sq_cons_idx) - 1
- , &sq->hwq);
+ peek_sq_cons_idx =
+ ((le16_to_cpu(
+ peek_req_hwcqe->sq_cons_idx)
+ - 1) % sq->max_wqe);
/* If the hwcqe's sq's wr_id matches */
if (peek_sq == sq &&
sq->swq[peek_sq_cons_idx].wr_id ==
@@ -2214,7 +2307,7 @@
}
dev_err(&cq->hwq.pdev->dev,
"Should not have come here! cq_cons=0x%x qp=0x%x sq cons sw=0x%x hw=0x%x\n",
- cq_cons, qp->id, sw_sq_cons, cqe_sq_cons);
+ cq_cons, qp->id, swq_last, cqe_sq_cons);
rc = -EINVAL;
}
out:
@@ -2226,11 +2319,11 @@
struct bnxt_qplib_cqe **pcqe, int *budget,
u32 cq_cons, struct bnxt_qplib_qp **lib_qp)
{
- u32 sw_sq_cons, cqe_sq_cons;
struct bnxt_qplib_swq *swq;
struct bnxt_qplib_cqe *cqe;
struct bnxt_qplib_qp *qp;
struct bnxt_qplib_q *sq;
+ u32 cqe_sq_cons;
int rc = 0;
qp = (struct bnxt_qplib_qp *)((unsigned long)
@@ -2242,14 +2335,7 @@
}
sq = &qp->sq;
- cqe_sq_cons = HWQ_CMP(le16_to_cpu(hwcqe->sq_cons_idx), &sq->hwq);
- if (cqe_sq_cons > sq->hwq.max_elements) {
- dev_err(&cq->hwq.pdev->dev,
- "FP: CQ Process req reported sq_cons_idx 0x%x which exceeded max 0x%x\n",
- cqe_sq_cons, sq->hwq.max_elements);
- return -EINVAL;
- }
-
+ cqe_sq_cons = le16_to_cpu(hwcqe->sq_cons_idx) % sq->max_wqe;
if (qp->sq.flushed) {
dev_dbg(&cq->hwq.pdev->dev,
"%s: QP in Flush QP = %p\n", __func__, qp);
@@ -2261,12 +2347,11 @@
*/
cqe = *pcqe;
while (*budget) {
- sw_sq_cons = HWQ_CMP(sq->hwq.cons, &sq->hwq);
- if (sw_sq_cons == cqe_sq_cons)
+ if (sq->swq_last == cqe_sq_cons)
/* Done */
break;
- swq = &sq->swq[sw_sq_cons];
+ swq = &sq->swq[sq->swq_last];
memset(cqe, 0, sizeof(*cqe));
cqe->opcode = CQ_BASE_CQE_TYPE_REQ;
cqe->qp_handle = (u64)(unsigned long)qp;
@@ -2280,12 +2365,12 @@
* of the request being signaled or not, it must complete with
* the hwcqe error status
*/
- if (HWQ_CMP((sw_sq_cons + 1), &sq->hwq) == cqe_sq_cons &&
+ if (swq->next_idx == cqe_sq_cons &&
hwcqe->status != CQ_REQ_STATUS_OK) {
cqe->status = hwcqe->status;
dev_err(&cq->hwq.pdev->dev,
"FP: CQ Processed Req wr_id[%d] = 0x%llx with status 0x%x\n",
- sw_sq_cons, cqe->wr_id, cqe->status);
+ sq->swq_last, cqe->wr_id, cqe->status);
cqe++;
(*budget)--;
bnxt_qplib_mark_qp_error(qp);
@@ -2293,7 +2378,7 @@
bnxt_qplib_add_flush_qp(qp);
} else {
/* Before we complete, do WA 9060 */
- if (do_wa9060(qp, cq, cq_cons, sw_sq_cons,
+ if (do_wa9060(qp, cq, cq_cons, sq->swq_last,
cqe_sq_cons)) {
*lib_qp = qp;
goto out;
@@ -2305,13 +2390,14 @@
}
}
skip:
- sq->hwq.cons++;
+ bnxt_qplib_hwq_incr_cons(&sq->hwq, swq->slots);
+ sq->swq_last = swq->next_idx;
if (sq->single)
break;
}
out:
*pcqe = cqe;
- if (HWQ_CMP(sq->hwq.cons, &sq->hwq) != cqe_sq_cons) {
+ if (sq->swq_last != cqe_sq_cons) {
/* Out of budget */
rc = -EAGAIN;
goto done;
@@ -2386,17 +2472,23 @@
(*budget)--;
*pcqe = cqe;
} else {
+ struct bnxt_qplib_swq *swq;
+
rq = &qp->rq;
- if (wr_id_idx >= rq->hwq.max_elements) {
+ if (wr_id_idx > (rq->max_wqe - 1)) {
dev_err(&cq->hwq.pdev->dev,
"FP: CQ Process RC wr_id idx 0x%x exceeded RQ max 0x%x\n",
- wr_id_idx, rq->hwq.max_elements);
+ wr_id_idx, rq->max_wqe);
return -EINVAL;
}
- cqe->wr_id = rq->swq[wr_id_idx].wr_id;
+ if (wr_id_idx != rq->swq_last)
+ return -EINVAL;
+ swq = &rq->swq[rq->swq_last];
+ cqe->wr_id = swq->wr_id;
cqe++;
(*budget)--;
- rq->hwq.cons++;
+ bnxt_qplib_hwq_incr_cons(&rq->hwq, swq->slots);
+ rq->swq_last = swq->next_idx;
*pcqe = cqe;
if (hwcqe->status != CQ_RES_RC_STATUS_OK) {
@@ -2467,18 +2559,24 @@
(*budget)--;
*pcqe = cqe;
} else {
+ struct bnxt_qplib_swq *swq;
+
rq = &qp->rq;
- if (wr_id_idx >= rq->hwq.max_elements) {
+ if (wr_id_idx > (rq->max_wqe - 1)) {
dev_err(&cq->hwq.pdev->dev,
"FP: CQ Process UD wr_id idx 0x%x exceeded RQ max 0x%x\n",
- wr_id_idx, rq->hwq.max_elements);
+ wr_id_idx, rq->max_wqe);
return -EINVAL;
}
- cqe->wr_id = rq->swq[wr_id_idx].wr_id;
+ if (rq->swq_last != wr_id_idx)
+ return -EINVAL;
+ swq = &rq->swq[rq->swq_last];
+ cqe->wr_id = swq->wr_id;
cqe++;
(*budget)--;
- rq->hwq.cons++;
+ bnxt_qplib_hwq_incr_cons(&rq->hwq, swq->slots);
+ rq->swq_last = swq->next_idx;
*pcqe = cqe;
if (hwcqe->status != CQ_RES_RC_STATUS_OK) {
@@ -2569,17 +2667,23 @@
(*budget)--;
*pcqe = cqe;
} else {
+ struct bnxt_qplib_swq *swq;
+
rq = &qp->rq;
- if (wr_id_idx >= rq->hwq.max_elements) {
+ if (wr_id_idx > (rq->max_wqe - 1)) {
dev_err(&cq->hwq.pdev->dev,
"FP: CQ Process Raw/QP1 RQ wr_id idx 0x%x exceeded RQ max 0x%x\n",
- wr_id_idx, rq->hwq.max_elements);
+ wr_id_idx, rq->max_wqe);
return -EINVAL;
}
- cqe->wr_id = rq->swq[wr_id_idx].wr_id;
+ if (rq->swq_last != wr_id_idx)
+ return -EINVAL;
+ swq = &rq->swq[rq->swq_last];
+ cqe->wr_id = swq->wr_id;
cqe++;
(*budget)--;
- rq->hwq.cons++;
+ bnxt_qplib_hwq_incr_cons(&rq->hwq, swq->slots);
+ rq->swq_last = swq->next_idx;
*pcqe = cqe;
if (hwcqe->status != CQ_RES_RC_STATUS_OK) {
@@ -2601,7 +2705,7 @@
struct bnxt_qplib_qp *qp;
struct bnxt_qplib_q *sq, *rq;
struct bnxt_qplib_cqe *cqe;
- u32 sw_cons = 0, cqe_cons;
+ u32 swq_last = 0, cqe_cons;
int rc = 0;
/* Check the Status */
@@ -2627,13 +2731,7 @@
cqe_cons = le16_to_cpu(hwcqe->sq_cons_idx);
if (cqe_cons == 0xFFFF)
goto do_rq;
-
- if (cqe_cons > sq->hwq.max_elements) {
- dev_err(&cq->hwq.pdev->dev,
- "FP: CQ Process terminal reported sq_cons_idx 0x%x which exceeded max 0x%x\n",
- cqe_cons, sq->hwq.max_elements);
- goto do_rq;
- }
+ cqe_cons %= sq->max_wqe;
if (qp->sq.flushed) {
dev_dbg(&cq->hwq.pdev->dev,
@@ -2647,24 +2745,25 @@
*/
cqe = *pcqe;
while (*budget) {
- sw_cons = HWQ_CMP(sq->hwq.cons, &sq->hwq);
- if (sw_cons == cqe_cons)
+ swq_last = sq->swq_last;
+ if (swq_last == cqe_cons)
break;
- if (sq->swq[sw_cons].flags & SQ_SEND_FLAGS_SIGNAL_COMP) {
+ if (sq->swq[swq_last].flags & SQ_SEND_FLAGS_SIGNAL_COMP) {
memset(cqe, 0, sizeof(*cqe));
cqe->status = CQ_REQ_STATUS_OK;
cqe->opcode = CQ_BASE_CQE_TYPE_REQ;
cqe->qp_handle = (u64)(unsigned long)qp;
cqe->src_qp = qp->id;
- cqe->wr_id = sq->swq[sw_cons].wr_id;
- cqe->type = sq->swq[sw_cons].type;
+ cqe->wr_id = sq->swq[swq_last].wr_id;
+ cqe->type = sq->swq[swq_last].type;
cqe++;
(*budget)--;
}
- sq->hwq.cons++;
+ bnxt_qplib_hwq_incr_cons(&sq->hwq, sq->swq[swq_last].slots);
+ sq->swq_last = sq->swq[swq_last].next_idx;
}
*pcqe = cqe;
- if (!(*budget) && sw_cons != cqe_cons) {
+ if (!(*budget) && swq_last != cqe_cons) {
/* Out of budget */
rc = -EAGAIN;
goto sq_done;
@@ -2676,10 +2775,10 @@
cqe_cons = le16_to_cpu(hwcqe->rq_cons_idx);
if (cqe_cons == 0xFFFF) {
goto done;
- } else if (cqe_cons > rq->hwq.max_elements) {
+ } else if (cqe_cons > rq->max_wqe - 1) {
dev_err(&cq->hwq.pdev->dev,
"FP: CQ Processed terminal reported rq_cons_idx 0x%x exceeds max 0x%x\n",
- cqe_cons, rq->hwq.max_elements);
+ cqe_cons, rq->max_wqe);
goto done;
}
diff --git a/drivers/infiniband/hw/bnxt_re/qplib_fp.h b/drivers/infiniband/hw/bnxt_re/qplib_fp.h
index 568ca39..f507844 100644
--- a/drivers/infiniband/hw/bnxt_re/qplib_fp.h
+++ b/drivers/infiniband/hw/bnxt_re/qplib_fp.h
@@ -39,6 +39,51 @@
#ifndef __BNXT_QPLIB_FP_H__
#define __BNXT_QPLIB_FP_H__
+/* Few helper structures temporarily defined here
+ * should get rid of these when roce_hsi.h is updated
+ * in original code base
+ */
+struct sq_ud_ext_hdr {
+ __le32 dst_qp;
+ __le32 avid;
+ __le64 rsvd;
+};
+
+struct sq_raw_ext_hdr {
+ __le32 cfa_meta;
+ __le32 rsvd0;
+ __le64 rsvd1;
+};
+
+struct sq_rdma_ext_hdr {
+ __le64 remote_va;
+ __le32 remote_key;
+ __le32 rsvd;
+};
+
+struct sq_atomic_ext_hdr {
+ __le64 swap_data;
+ __le64 cmp_data;
+};
+
+struct sq_fr_pmr_ext_hdr {
+ __le64 pblptr;
+ __le64 va;
+};
+
+struct sq_bind_ext_hdr {
+ __le64 va;
+ __le32 length_lo;
+ __le32 length_hi;
+};
+
+struct rq_ext_hdr {
+ __le64 rsvd1;
+ __le64 rsvd2;
+};
+
+/* Helper structures end */
+
struct bnxt_qplib_srq {
struct bnxt_qplib_pd *pd;
struct bnxt_qplib_dpi *dpi;
@@ -74,6 +119,8 @@
u8 flags;
u32 start_psn;
u32 next_psn;
+ u32 slot_idx;
+ u8 slots;
struct sq_psn_search *psn_search;
struct sq_psn_search_ext *psn_ext;
};
@@ -213,6 +260,8 @@
u32 phantom_cqe_cnt;
u32 next_cq_cons;
bool flushed;
+ u32 swq_start;
+ u32 swq_last;
};
struct bnxt_qplib_qp {
@@ -224,9 +273,10 @@
u32 id;
u8 type;
u8 sig_type;
- u32 modify_flags;
+ u8 wqe_mode;
u8 state;
u8 cur_qp_state;
+ u64 modify_flags;
u32 max_inline_data;
u32 mtu;
u8 path_mtu;
@@ -300,11 +350,18 @@
(!!((hdr)->cqe_type_toggle & CQ_BASE_TOGGLE) == \
!((raw_cons) & (cp_bit)))
-static inline bool bnxt_qplib_queue_full(struct bnxt_qplib_q *qplib_q)
+static inline bool bnxt_qplib_queue_full(struct bnxt_qplib_q *que,
+ u8 slots)
{
- return HWQ_CMP((qplib_q->hwq.prod + qplib_q->q_full_delta),
- &qplib_q->hwq) == HWQ_CMP(qplib_q->hwq.cons,
- &qplib_q->hwq);
+ struct bnxt_qplib_hwq *hwq;
+ int avail;
+
+ hwq = &que->hwq;
+ /* False full is possible, retrying post-send makes sense */
+ avail = hwq->cons - hwq->prod;
+ if (hwq->cons <= hwq->prod)
+ avail += hwq->depth;
+ return avail <= slots;
}
struct bnxt_qplib_cqe {
@@ -489,4 +546,64 @@
struct bnxt_qplib_cqe *cqe,
int num_cqes);
void bnxt_qplib_flush_cqn_wq(struct bnxt_qplib_qp *qp);
+
+static inline void *bnxt_qplib_get_swqe(struct bnxt_qplib_q *que, u32 *swq_idx)
+{
+ u32 idx;
+
+ idx = que->swq_start;
+ if (swq_idx)
+ *swq_idx = idx;
+ return &que->swq[idx];
+}
+
+static inline void bnxt_qplib_swq_mod_start(struct bnxt_qplib_q *que, u32 idx)
+{
+ que->swq_start = que->swq[idx].next_idx;
+}
+
+static inline u32 bnxt_qplib_get_depth(struct bnxt_qplib_q *que)
+{
+ return (que->wqe_size * que->max_wqe) / sizeof(struct sq_sge);
+}
+
+static inline u32 bnxt_qplib_set_sq_size(struct bnxt_qplib_q *que, u8 wqe_mode)
+{
+ return (wqe_mode == BNXT_QPLIB_WQE_MODE_STATIC) ?
+ que->max_wqe : bnxt_qplib_get_depth(que);
+}
+
+static inline u32 bnxt_qplib_set_sq_max_slot(u8 wqe_mode)
+{
+ return (wqe_mode == BNXT_QPLIB_WQE_MODE_STATIC) ?
+ sizeof(struct sq_send) / sizeof(struct sq_sge) : 1;
+}
+
+static inline u32 bnxt_qplib_set_rq_max_slot(u32 wqe_size)
+{
+ return (wqe_size / sizeof(struct sq_sge));
+}
+
+static inline u16 __xlate_qfd(u16 delta, u16 wqe_bytes)
+{
+ /* For Cu/Wh delta = 128, stride = 16, wqe_bytes = 128
+ * For Gen-p5 B/C mode delta = 0, stride = 16, wqe_bytes = 128.
+ * For Gen-p5 delta = 0, stride = 16, 32 <= wqe_bytes <= 512.
+ * when 8916 is disabled.
+ */
+ return (delta * wqe_bytes) / sizeof(struct sq_sge);
+}
+
+static inline u16 bnxt_qplib_calc_ilsize(struct bnxt_qplib_swqe *wqe, u16 max)
+{
+ u16 size = 0;
+ int indx;
+
+ for (indx = 0; indx < wqe->num_sge; indx++)
+ size += wqe->sg_list[indx].size;
+ if (size > max)
+ size = max;
+
+ return size;
+}
#endif /* __BNXT_QPLIB_FP_H__ */
diff --git a/drivers/infiniband/hw/bnxt_re/qplib_res.h b/drivers/infiniband/hw/bnxt_re/qplib_res.h
index c29cbd3..9da470d 100644
--- a/drivers/infiniband/hw/bnxt_re/qplib_res.h
+++ b/drivers/infiniband/hw/bnxt_re/qplib_res.h
@@ -41,6 +41,28 @@
extern const struct bnxt_qplib_gid bnxt_qplib_gid_zero;
+#define CHIP_NUM_57508 0x1750
+#define CHIP_NUM_57504 0x1751
+#define CHIP_NUM_57502 0x1752
+
+enum bnxt_qplib_wqe_mode {
+ BNXT_QPLIB_WQE_MODE_STATIC = 0x00,
+ BNXT_QPLIB_WQE_MODE_VARIABLE = 0x01,
+ BNXT_QPLIB_WQE_MODE_INVALID = 0x02
+};
+
+struct bnxt_qplib_drv_modes {
+ u8 wqe_mode;
+ /* Other modes to follow here */
+};
+
+struct bnxt_qplib_chip_ctx {
+ u16 chip_num;
+ u8 chip_rev;
+ u8 chip_metal;
+ struct bnxt_qplib_drv_modes modes;
+};
+
#define PTR_CNT_PER_PG (PAGE_SIZE / sizeof(void *))
#define PTR_MAX_IDX_PER_PG (PTR_CNT_PER_PG - 1)
#define PTR_PG(x) (((x) & ~PTR_MAX_IDX_PER_PG) / PTR_CNT_PER_PG)
@@ -141,6 +163,9 @@
u32 cons; /* raw */
u8 cp_bit;
u8 is_user;
+ u64 *pad_pg;
+ u32 pad_stride;
+ u32 pad_pgofft;
};
struct bnxt_qplib_db_info {
@@ -148,6 +173,7 @@
void __iomem *priv_db;
struct bnxt_qplib_hwq *hwq;
u32 xid;
+ u32 max_slot;
};
/* Tables */
@@ -230,16 +256,6 @@
u64 hwrm_intf_ver;
};
-struct bnxt_qplib_chip_ctx {
- u16 chip_num;
- u8 chip_rev;
- u8 chip_metal;
-};
-
-#define CHIP_NUM_57508 0x1750
-#define CHIP_NUM_57504 0x1751
-#define CHIP_NUM_57502 0x1752
-
struct bnxt_qplib_res {
struct pci_dev *pdev;
struct bnxt_qplib_chip_ctx *cctx;
@@ -317,6 +333,14 @@
return (void *)(hwq->pbl_ptr[pg_num] + hwq->element_size * pg_idx);
}
+static inline void *bnxt_qplib_get_prod_qe(struct bnxt_qplib_hwq *hwq, u32 idx)
+{
+ idx += hwq->prod;
+ if (idx >= hwq->depth)
+ idx -= hwq->depth;
+ return bnxt_qplib_get_qe(hwq, idx, NULL);
+}
+
#define to_bnxt_qplib(ptr, type, member) \
container_of(ptr, type, member)
@@ -351,6 +375,17 @@
struct bnxt_qplib_ctx *ctx,
bool virt_fn, bool is_p5);
+static inline void bnxt_qplib_hwq_incr_prod(struct bnxt_qplib_hwq *hwq, u32 cnt)
+{
+ hwq->prod = (hwq->prod + cnt) % hwq->depth;
+}
+
+static inline void bnxt_qplib_hwq_incr_cons(struct bnxt_qplib_hwq *hwq,
+ u32 cnt)
+{
+ hwq->cons = (hwq->cons + cnt) % hwq->depth;
+}
+
static inline void bnxt_qplib_ring_db32(struct bnxt_qplib_db_info *info,
bool arm)
{
@@ -383,8 +418,7 @@
key = (info->xid & DBC_DBC_XID_MASK) | DBC_DBC_PATH_ROCE | type;
key <<= 32;
- key |= (info->hwq->prod & (info->hwq->max_elements - 1)) &
- DBC_DBC_INDEX_MASK;
+ key |= ((info->hwq->prod / info->max_slot)) & DBC_DBC_INDEX_MASK;
writeq(key, info->db);
}
diff --git a/drivers/infiniband/hw/bnxt_re/roce_hsi.h b/drivers/infiniband/hw/bnxt_re/roce_hsi.h
index 6f00f07..3e40e0d 100644
--- a/drivers/infiniband/hw/bnxt_re/roce_hsi.h
+++ b/drivers/infiniband/hw/bnxt_re/roce_hsi.h
@@ -1126,6 +1126,7 @@
#define CMDQ_CREATE_QP_QP_FLAGS_FORCE_COMPLETION 0x2UL
#define CMDQ_CREATE_QP_QP_FLAGS_RESERVED_LKEY_ENABLE 0x4UL
#define CMDQ_CREATE_QP_QP_FLAGS_FR_PMR_ENABLED 0x8UL
+ #define CMDQ_CREATE_QP_QP_FLAGS_VARIABLE_SIZED_WQE_ENABLED 0x10UL
u8 type;
#define CMDQ_CREATE_QP_TYPE_RC 0x2UL
#define CMDQ_CREATE_QP_TYPE_UD 0x4UL
diff --git a/drivers/infiniband/hw/cxgb4/iw_cxgb4.h b/drivers/infiniband/hw/cxgb4/iw_cxgb4.h
index e8e11bd..2b2b009 100644
--- a/drivers/infiniband/hw/cxgb4/iw_cxgb4.h
+++ b/drivers/infiniband/hw/cxgb4/iw_cxgb4.h
@@ -980,7 +980,7 @@
void c4iw_qp_add_ref(struct ib_qp *qp);
void c4iw_qp_rem_ref(struct ib_qp *qp);
struct ib_mr *c4iw_alloc_mr(struct ib_pd *pd, enum ib_mr_type mr_type,
- u32 max_num_sg, struct ib_udata *udata);
+ u32 max_num_sg);
int c4iw_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg, int sg_nents,
unsigned int *sg_offset);
int c4iw_dealloc_mw(struct ib_mw *mw);
@@ -1053,8 +1053,9 @@
const struct ib_recv_wr **bad_wr);
struct c4iw_wr_wait *c4iw_alloc_wr_wait(gfp_t gfp);
-typedef int c4iw_restrack_func(struct sk_buff *msg,
- struct rdma_restrack_entry *res);
-extern c4iw_restrack_func *c4iw_restrack_funcs[RDMA_RESTRACK_MAX];
+int c4iw_fill_res_mr_entry(struct sk_buff *msg, struct ib_mr *ibmr);
+int c4iw_fill_res_cq_entry(struct sk_buff *msg, struct ib_cq *ibcq);
+int c4iw_fill_res_qp_entry(struct sk_buff *msg, struct ib_qp *ibqp);
+int c4iw_fill_res_cm_id_entry(struct sk_buff *msg, struct rdma_cm_id *cm_id);
#endif
diff --git a/drivers/infiniband/hw/cxgb4/mem.c b/drivers/infiniband/hw/cxgb4/mem.c
index 962dc97..73936c3 100644
--- a/drivers/infiniband/hw/cxgb4/mem.c
+++ b/drivers/infiniband/hw/cxgb4/mem.c
@@ -399,7 +399,6 @@
mmid = stag >> 8;
mhp->ibmr.rkey = mhp->ibmr.lkey = stag;
mhp->ibmr.length = mhp->attr.len;
- mhp->ibmr.iova = mhp->attr.va_fbo;
mhp->ibmr.page_size = 1U << (mhp->attr.page_size + 12);
pr_debug("mmid 0x%x mhp %p\n", mmid, mhp);
return xa_insert_irq(&mhp->rhp->mrs, mmid, mhp, GFP_KERNEL);
@@ -691,7 +690,7 @@
}
struct ib_mr *c4iw_alloc_mr(struct ib_pd *pd, enum ib_mr_type mr_type,
- u32 max_num_sg, struct ib_udata *udata)
+ u32 max_num_sg)
{
struct c4iw_dev *rhp;
struct c4iw_pd *php;
diff --git a/drivers/infiniband/hw/cxgb4/provider.c b/drivers/infiniband/hw/cxgb4/provider.c
index ba83d94..6c579d2 100644
--- a/drivers/infiniband/hw/cxgb4/provider.c
+++ b/drivers/infiniband/hw/cxgb4/provider.c
@@ -236,14 +236,6 @@
return 0;
}
-static int c4iw_query_pkey(struct ib_device *ibdev, u8 port, u16 index,
- u16 *pkey)
-{
- pr_debug("ibdev %p\n", ibdev);
- *pkey = 0;
- return 0;
-}
-
static int c4iw_query_gid(struct ib_device *ibdev, u8 port, int index,
union ib_gid *gid)
{
@@ -317,7 +309,6 @@
IB_PORT_DEVICE_MGMT_SUP |
IB_PORT_VENDOR_CLASS_SUP | IB_PORT_BOOT_MGMT_SUP;
props->gid_tbl_len = 1;
- props->pkey_tbl_len = 1;
props->max_msg_sz = -1;
return ret;
@@ -439,7 +430,6 @@
if (err)
return err;
- immutable->pkey_tbl_len = attr.pkey_tbl_len;
immutable->gid_tbl_len = attr.gid_tbl_len;
return 0;
@@ -458,13 +448,6 @@
FW_HDR_FW_VER_BUILD_G(c4iw_dev->rdev.lldi.fw_vers));
}
-static int fill_res_entry(struct sk_buff *msg, struct rdma_restrack_entry *res)
-{
- return (res->type < ARRAY_SIZE(c4iw_restrack_funcs) &&
- c4iw_restrack_funcs[res->type]) ?
- c4iw_restrack_funcs[res->type](msg, res) : 0;
-}
-
static const struct ib_device_ops c4iw_dev_ops = {
.owner = THIS_MODULE,
.driver_id = RDMA_DRIVER_CXGB4,
@@ -485,7 +468,9 @@
.destroy_cq = c4iw_destroy_cq,
.destroy_qp = c4iw_destroy_qp,
.destroy_srq = c4iw_destroy_srq,
- .fill_res_entry = fill_res_entry,
+ .fill_res_cq_entry = c4iw_fill_res_cq_entry,
+ .fill_res_cm_id_entry = c4iw_fill_res_cm_id_entry,
+ .fill_res_mr_entry = c4iw_fill_res_mr_entry,
.get_dev_fw_str = get_dev_fw_str,
.get_dma_mr = c4iw_get_dma_mr,
.get_hw_stats = c4iw_get_mib,
@@ -508,7 +493,6 @@
.post_srq_recv = c4iw_post_srq_recv,
.query_device = c4iw_query_device,
.query_gid = c4iw_query_gid,
- .query_pkey = c4iw_query_pkey,
.query_port = c4iw_query_port,
.query_qp = c4iw_ib_query_qp,
.reg_user_mr = c4iw_reg_user_mr,
diff --git a/drivers/infiniband/hw/cxgb4/restrack.c b/drivers/infiniband/hw/cxgb4/restrack.c
index f82d46e..b32e651 100644
--- a/drivers/infiniband/hw/cxgb4/restrack.c
+++ b/drivers/infiniband/hw/cxgb4/restrack.c
@@ -134,10 +134,8 @@
return -EMSGSIZE;
}
-static int fill_res_qp_entry(struct sk_buff *msg,
- struct rdma_restrack_entry *res)
+int c4iw_fill_res_qp_entry(struct sk_buff *msg, struct ib_qp *ibqp)
{
- struct ib_qp *ibqp = container_of(res, struct ib_qp, res);
struct t4_swsqe *fsp = NULL, *lsp = NULL;
struct c4iw_qp *qhp = to_c4iw_qp(ibqp);
u16 first_sq_idx = 0, last_sq_idx = 0;
@@ -195,10 +193,9 @@
struct c4iw_ep ep;
};
-static int fill_res_ep_entry(struct sk_buff *msg,
- struct rdma_restrack_entry *res)
+int c4iw_fill_res_cm_id_entry(struct sk_buff *msg,
+ struct rdma_cm_id *cm_id)
{
- struct rdma_cm_id *cm_id = rdma_res_to_id(res);
struct nlattr *table_attr;
struct c4iw_ep_common *epcp;
struct c4iw_listen_ep *listen_ep = NULL;
@@ -372,10 +369,8 @@
return -EMSGSIZE;
}
-static int fill_res_cq_entry(struct sk_buff *msg,
- struct rdma_restrack_entry *res)
+int c4iw_fill_res_cq_entry(struct sk_buff *msg, struct ib_cq *ibcq)
{
- struct ib_cq *ibcq = container_of(res, struct ib_cq, res);
struct c4iw_cq *chp = to_c4iw_cq(ibcq);
struct nlattr *table_attr;
struct t4_cqe hwcqes[2];
@@ -433,10 +428,8 @@
return -EMSGSIZE;
}
-static int fill_res_mr_entry(struct sk_buff *msg,
- struct rdma_restrack_entry *res)
+int c4iw_fill_res_mr_entry(struct sk_buff *msg, struct ib_mr *ibmr)
{
- struct ib_mr *ibmr = container_of(res, struct ib_mr, res);
struct c4iw_mr *mhp = to_c4iw_mr(ibmr);
struct c4iw_dev *dev = mhp->rhp;
u32 stag = mhp->attr.stag;
@@ -492,10 +485,3 @@
err:
return -EMSGSIZE;
}
-
-c4iw_restrack_func *c4iw_restrack_funcs[RDMA_RESTRACK_MAX] = {
- [RDMA_RESTRACK_QP] = fill_res_qp_entry,
- [RDMA_RESTRACK_CM_ID] = fill_res_ep_entry,
- [RDMA_RESTRACK_CQ] = fill_res_cq_entry,
- [RDMA_RESTRACK_MR] = fill_res_mr_entry,
-};
diff --git a/drivers/infiniband/hw/efa/efa_admin_cmds_defs.h b/drivers/infiniband/hw/efa/efa_admin_cmds_defs.h
index bef2bd2..5484b08 100644
--- a/drivers/infiniband/hw/efa/efa_admin_cmds_defs.h
+++ b/drivers/infiniband/hw/efa/efa_admin_cmds_defs.h
@@ -606,8 +606,8 @@
/* Number of sub-CQs to be created for each CQ */
u16 sub_cqs_per_cq;
- /* MBZ */
- u16 reserved;
+ /* Minimum number of WQEs per SQ */
+ u16 min_sq_depth;
/* Maximum number of SGEs (buffers) allowed for a single send WQE */
u16 max_wr_send_sges;
@@ -632,6 +632,17 @@
/* Maximum number of SGEs for a single RDMA read WQE */
u16 max_wr_rdma_sges;
+
+ /*
+ * Maximum number of bytes that can be written to SQ between two
+ * consecutive doorbells (in units of 64B). Driver must ensure that only
+ * complete WQEs are written to queue before issuing a doorbell.
+ * Examples: max_tx_batch=16 and WQE size = 64B, means up to 16 WQEs can
+ * be written to SQ between two consecutive doorbells. max_tx_batch=11
+ * and WQE size = 128B, means up to 5 WQEs can be written to SQ between
+ * two consecutive doorbells. Zero means unlimited.
+ */
+ u16 max_tx_batch;
};
struct efa_admin_feature_aenq_desc {
diff --git a/drivers/infiniband/hw/efa/efa_com_cmd.c b/drivers/infiniband/hw/efa/efa_com_cmd.c
index fabd8df..6ac2362 100644
--- a/drivers/infiniband/hw/efa/efa_com_cmd.c
+++ b/drivers/infiniband/hw/efa/efa_com_cmd.c
@@ -480,6 +480,8 @@
result->max_llq_size = resp.u.queue_attr.max_llq_size;
result->sub_cqs_per_cq = resp.u.queue_attr.sub_cqs_per_cq;
result->max_wr_rdma_sge = resp.u.queue_attr.max_wr_rdma_sges;
+ result->max_tx_batch = resp.u.queue_attr.max_tx_batch;
+ result->min_sq_depth = resp.u.queue_attr.min_sq_depth;
err = efa_com_get_feature(edev, &resp, EFA_ADMIN_NETWORK_ATTR);
if (err) {
diff --git a/drivers/infiniband/hw/efa/efa_com_cmd.h b/drivers/infiniband/hw/efa/efa_com_cmd.h
index 41ce4a4..190bac2 100644
--- a/drivers/infiniband/hw/efa/efa_com_cmd.h
+++ b/drivers/infiniband/hw/efa/efa_com_cmd.h
@@ -127,6 +127,8 @@
u16 max_sq_sge;
u16 max_rq_sge;
u16 max_wr_rdma_sge;
+ u16 max_tx_batch;
+ u16 min_sq_depth;
u8 db_bar;
};
diff --git a/drivers/infiniband/hw/efa/efa_main.c b/drivers/infiniband/hw/efa/efa_main.c
index 82145574..92d7011 100644
--- a/drivers/infiniband/hw/efa/efa_main.c
+++ b/drivers/infiniband/hw/efa/efa_main.c
@@ -12,10 +12,12 @@
#include "efa.h"
-#define PCI_DEV_ID_EFA_VF 0xefa0
+#define PCI_DEV_ID_EFA0_VF 0xefa0
+#define PCI_DEV_ID_EFA1_VF 0xefa1
static const struct pci_device_id efa_pci_tbl[] = {
- { PCI_VDEVICE(AMAZON, PCI_DEV_ID_EFA_VF) },
+ { PCI_VDEVICE(AMAZON, PCI_DEV_ID_EFA0_VF) },
+ { PCI_VDEVICE(AMAZON, PCI_DEV_ID_EFA1_VF) },
{ }
};
diff --git a/drivers/infiniband/hw/efa/efa_verbs.c b/drivers/infiniband/hw/efa/efa_verbs.c
index 7dd0824..9e201f169 100644
--- a/drivers/infiniband/hw/efa/efa_verbs.c
+++ b/drivers/infiniband/hw/efa/efa_verbs.c
@@ -1502,11 +1502,39 @@
return efa_com_dealloc_uar(&dev->edev, ¶ms);
}
+#define EFA_CHECK_USER_COMP(_dev, _comp_mask, _attr, _mask, _attr_str) \
+ (_attr_str = (!(_dev)->dev_attr._attr || ((_comp_mask) & (_mask))) ? \
+ NULL : #_attr)
+
+static int efa_user_comp_handshake(const struct ib_ucontext *ibucontext,
+ const struct efa_ibv_alloc_ucontext_cmd *cmd)
+{
+ struct efa_dev *dev = to_edev(ibucontext->device);
+ char *attr_str;
+
+ if (EFA_CHECK_USER_COMP(dev, cmd->comp_mask, max_tx_batch,
+ EFA_ALLOC_UCONTEXT_CMD_COMP_TX_BATCH, attr_str))
+ goto err;
+
+ if (EFA_CHECK_USER_COMP(dev, cmd->comp_mask, min_sq_depth,
+ EFA_ALLOC_UCONTEXT_CMD_COMP_MIN_SQ_WR,
+ attr_str))
+ goto err;
+
+ return 0;
+
+err:
+ ibdev_dbg(&dev->ibdev, "Userspace handshake failed for %s attribute\n",
+ attr_str);
+ return -EOPNOTSUPP;
+}
+
int efa_alloc_ucontext(struct ib_ucontext *ibucontext, struct ib_udata *udata)
{
struct efa_ucontext *ucontext = to_eucontext(ibucontext);
struct efa_dev *dev = to_edev(ibucontext->device);
struct efa_ibv_alloc_ucontext_resp resp = {};
+ struct efa_ibv_alloc_ucontext_cmd cmd = {};
struct efa_com_alloc_uar_result result;
int err;
@@ -1515,6 +1543,18 @@
* we will ack input fields in our response.
*/
+ err = ib_copy_from_udata(&cmd, udata,
+ min(sizeof(cmd), udata->inlen));
+ if (err) {
+ ibdev_dbg(&dev->ibdev,
+ "Cannot copy udata for alloc_ucontext\n");
+ goto err_out;
+ }
+
+ err = efa_user_comp_handshake(ibucontext, &cmd);
+ if (err)
+ goto err_out;
+
err = efa_com_alloc_uar(&dev->edev, &result);
if (err)
goto err_out;
@@ -1526,6 +1566,8 @@
resp.sub_cqs_per_cq = dev->dev_attr.sub_cqs_per_cq;
resp.inline_buf_size = dev->dev_attr.inline_buf_size;
resp.max_llq_size = dev->dev_attr.max_llq_size;
+ resp.max_tx_batch = dev->dev_attr.max_tx_batch;
+ resp.min_sq_wr = dev->dev_attr.min_sq_depth;
if (udata && udata->outlen) {
err = ib_copy_to_udata(udata, &resp,
diff --git a/drivers/infiniband/hw/hfi1/chip.c b/drivers/infiniband/hw/hfi1/chip.c
index 15f9c63..7eaf9953 100644
--- a/drivers/infiniband/hw/hfi1/chip.c
+++ b/drivers/infiniband/hw/hfi1/chip.c
@@ -7317,11 +7317,11 @@
case 1: return OPA_LINK_WIDTH_1X;
case 2: return OPA_LINK_WIDTH_2X;
case 3: return OPA_LINK_WIDTH_3X;
+ case 4: return OPA_LINK_WIDTH_4X;
default:
dd_dev_info(dd, "%s: invalid width %d, using 4\n",
__func__, width);
- /* fall through */
- case 4: return OPA_LINK_WIDTH_4X;
+ return OPA_LINK_WIDTH_4X;
}
}
@@ -7376,12 +7376,13 @@
case 0:
dd->pport[0].link_speed_active = OPA_LINK_SPEED_12_5G;
break;
+ case 1:
+ dd->pport[0].link_speed_active = OPA_LINK_SPEED_25G;
+ break;
default:
dd_dev_err(dd,
"%s: unexpected max rate %d, using 25Gb\n",
__func__, (int)max_rate);
- /* fall through */
- case 1:
dd->pport[0].link_speed_active = OPA_LINK_SPEED_25G;
break;
}
@@ -12878,11 +12879,6 @@
static u32 chip_to_opa_lstate(struct hfi1_devdata *dd, u32 chip_lstate)
{
switch (chip_lstate) {
- default:
- dd_dev_err(dd,
- "Unknown logical state 0x%x, reporting IB_PORT_DOWN\n",
- chip_lstate);
- /* fall through */
case LSTATE_DOWN:
return IB_PORT_DOWN;
case LSTATE_INIT:
@@ -12891,6 +12887,11 @@
return IB_PORT_ARMED;
case LSTATE_ACTIVE:
return IB_PORT_ACTIVE;
+ default:
+ dd_dev_err(dd,
+ "Unknown logical state 0x%x, reporting IB_PORT_DOWN\n",
+ chip_lstate);
+ return IB_PORT_DOWN;
}
}
@@ -12898,10 +12899,6 @@
{
/* look at the HFI meta-states only */
switch (chip_pstate & 0xf0) {
- default:
- dd_dev_err(dd, "Unexpected chip physical state of 0x%x\n",
- chip_pstate);
- /* fall through */
case PLS_DISABLED:
return IB_PORTPHYSSTATE_DISABLED;
case PLS_OFFLINE:
@@ -12914,6 +12911,10 @@
return IB_PORTPHYSSTATE_LINKUP;
case PLS_PHYTEST:
return IB_PORTPHYSSTATE_PHY_TEST;
+ default:
+ dd_dev_err(dd, "Unexpected chip physical state of 0x%x\n",
+ chip_pstate);
+ return IB_PORTPHYSSTATE_DISABLED;
}
}
diff --git a/drivers/infiniband/hw/hfi1/firmware.c b/drivers/infiniband/hw/hfi1/firmware.c
index 2b57ba7..0e83d4b 100644
--- a/drivers/infiniband/hw/hfi1/firmware.c
+++ b/drivers/infiniband/hw/hfi1/firmware.c
@@ -1868,11 +1868,8 @@
2;
break;
case PLATFORM_CONFIG_RX_PRESET_TABLE:
- /* fall through */
case PLATFORM_CONFIG_TX_PRESET_TABLE:
- /* fall through */
case PLATFORM_CONFIG_QSFP_ATTEN_TABLE:
- /* fall through */
case PLATFORM_CONFIG_VARIABLE_SETTINGS_TABLE:
pcfgcache->config_tables[table_type].num_table =
table_length_dwords;
@@ -1890,15 +1887,10 @@
/* metadata table */
switch (table_type) {
case PLATFORM_CONFIG_SYSTEM_TABLE:
- /* fall through */
case PLATFORM_CONFIG_PORT_TABLE:
- /* fall through */
case PLATFORM_CONFIG_RX_PRESET_TABLE:
- /* fall through */
case PLATFORM_CONFIG_TX_PRESET_TABLE:
- /* fall through */
case PLATFORM_CONFIG_QSFP_ATTEN_TABLE:
- /* fall through */
case PLATFORM_CONFIG_VARIABLE_SETTINGS_TABLE:
break;
default:
@@ -2027,15 +2019,10 @@
switch (table) {
case PLATFORM_CONFIG_SYSTEM_TABLE:
- /* fall through */
case PLATFORM_CONFIG_PORT_TABLE:
- /* fall through */
case PLATFORM_CONFIG_RX_PRESET_TABLE:
- /* fall through */
case PLATFORM_CONFIG_TX_PRESET_TABLE:
- /* fall through */
case PLATFORM_CONFIG_QSFP_ATTEN_TABLE:
- /* fall through */
case PLATFORM_CONFIG_VARIABLE_SETTINGS_TABLE:
if (field && field < platform_config_table_limits[table])
src_ptr =
@@ -2138,11 +2125,8 @@
pcfgcache->config_tables[table_type].table;
break;
case PLATFORM_CONFIG_RX_PRESET_TABLE:
- /* fall through */
case PLATFORM_CONFIG_TX_PRESET_TABLE:
- /* fall through */
case PLATFORM_CONFIG_QSFP_ATTEN_TABLE:
- /* fall through */
case PLATFORM_CONFIG_VARIABLE_SETTINGS_TABLE:
src_ptr = pcfgcache->config_tables[table_type].table;
diff --git a/drivers/infiniband/hw/hfi1/mad.c b/drivers/infiniband/hw/hfi1/mad.c
index 7073f23..3222e3a 100644
--- a/drivers/infiniband/hw/hfi1/mad.c
+++ b/drivers/infiniband/hw/hfi1/mad.c
@@ -721,7 +721,7 @@
/* Bad mkey not a violation below level 2 */
if (ibp->rvp.mkeyprot < 2)
break;
- /* fall through */
+ fallthrough;
case IB_MGMT_METHOD_SET:
case IB_MGMT_METHOD_TRAP_REPRESS:
if (ibp->rvp.mkey_violations != 0xFFFF)
@@ -1272,7 +1272,7 @@
case IB_PORT_NOP:
if (phys_state == IB_PORTPHYSSTATE_NOP)
break;
- /* FALLTHROUGH */
+ fallthrough;
case IB_PORT_DOWN:
if (phys_state == IB_PORTPHYSSTATE_NOP) {
link_state = HLS_DN_DOWNDEF;
@@ -2300,7 +2300,6 @@
* can be changed from the default values
*/
case OPA_VLARB_PREEMPT_ELEMENTS:
- /* FALLTHROUGH */
case OPA_VLARB_PREEMPT_MATRIX:
smp->status |= IB_SMP_UNSUP_METH_ATTR;
break;
@@ -4170,7 +4169,7 @@
return IB_MAD_RESULT_SUCCESS | IB_MAD_RESULT_CONSUMED;
if (ibp->rvp.port_cap_flags & IB_PORT_SM)
return IB_MAD_RESULT_SUCCESS;
- /* FALLTHROUGH */
+ fallthrough;
default:
smp->status |= IB_SMP_UNSUP_METH_ATTR;
ret = reply((struct ib_mad_hdr *)smp);
@@ -4240,7 +4239,7 @@
return IB_MAD_RESULT_SUCCESS | IB_MAD_RESULT_CONSUMED;
if (ibp->rvp.port_cap_flags & IB_PORT_SM)
return IB_MAD_RESULT_SUCCESS;
- /* FALLTHROUGH */
+ fallthrough;
default:
smp->status |= IB_SMP_UNSUP_METH_ATTR;
ret = reply((struct ib_mad_hdr *)smp);
diff --git a/drivers/infiniband/hw/hfi1/pcie.c b/drivers/infiniband/hw/hfi1/pcie.c
index 1a6268d..18d32f0 100644
--- a/drivers/infiniband/hw/hfi1/pcie.c
+++ b/drivers/infiniband/hw/hfi1/pcie.c
@@ -306,7 +306,7 @@
ret = pcie_capability_read_dword(dd->pcidev, PCI_EXP_LNKCAP, &linkcap);
if (ret) {
dd_dev_err(dd, "Unable to read from PCI config\n");
- return ret;
+ return pcibios_err_to_errno(ret);
}
if ((linkcap & PCI_EXP_LNKCAP_SLS) != PCI_EXP_LNKCAP_SLS_8_0GB) {
@@ -334,10 +334,14 @@
return 0;
}
-/* restore command and BARs after a reset has wiped them out */
+/**
+ * Restore command and BARs after a reset has wiped them out
+ *
+ * Returns 0 on success, otherwise a negative error value
+ */
int restore_pci_variables(struct hfi1_devdata *dd)
{
- int ret = 0;
+ int ret;
ret = pci_write_config_word(dd->pcidev, PCI_COMMAND, dd->pci_command);
if (ret)
@@ -386,13 +390,17 @@
error:
dd_dev_err(dd, "Unable to write to PCI config\n");
- return ret;
+ return pcibios_err_to_errno(ret);
}
-/* Save BARs and command to rewrite after device reset */
+/**
+ * Save BARs and command to rewrite after device reset
+ *
+ * Returns 0 on success, otherwise a negative error value
+ */
int save_pci_variables(struct hfi1_devdata *dd)
{
- int ret = 0;
+ int ret;
ret = pci_read_config_dword(dd->pcidev, PCI_BASE_ADDRESS_0,
&dd->pcibar0);
@@ -441,7 +449,7 @@
error:
dd_dev_err(dd, "Unable to read from PCI config\n");
- return ret;
+ return pcibios_err_to_errno(ret);
}
/*
diff --git a/drivers/infiniband/hw/hfi1/pio.c b/drivers/infiniband/hw/hfi1/pio.c
index 79126b2..ff864f6 100644
--- a/drivers/infiniband/hw/hfi1/pio.c
+++ b/drivers/infiniband/hw/hfi1/pio.c
@@ -86,7 +86,7 @@
switch (op) {
case PSC_GLOBAL_ENABLE:
reg |= SEND_CTRL_SEND_ENABLE_SMASK;
- /* Fall through */
+ fallthrough;
case PSC_DATA_VL_ENABLE:
mask = 0;
for (i = 0; i < ARRAY_SIZE(dd->vld); i++)
diff --git a/drivers/infiniband/hw/hfi1/pio_copy.c b/drivers/infiniband/hw/hfi1/pio_copy.c
index 03024ce..b12e466 100644
--- a/drivers/infiniband/hw/hfi1/pio_copy.c
+++ b/drivers/infiniband/hw/hfi1/pio_copy.c
@@ -191,22 +191,22 @@
switch (n) {
case 7:
*dest++ = *src++;
- /* fall through */
+ fallthrough;
case 6:
*dest++ = *src++;
- /* fall through */
+ fallthrough;
case 5:
*dest++ = *src++;
- /* fall through */
+ fallthrough;
case 4:
*dest++ = *src++;
- /* fall through */
+ fallthrough;
case 3:
*dest++ = *src++;
- /* fall through */
+ fallthrough;
case 2:
*dest++ = *src++;
- /* fall through */
+ fallthrough;
case 1:
*dest++ = *src++;
/* fall through */
diff --git a/drivers/infiniband/hw/hfi1/platform.c b/drivers/infiniband/hw/hfi1/platform.c
index 36593f2..4642d6c 100644
--- a/drivers/infiniband/hw/hfi1/platform.c
+++ b/drivers/infiniband/hw/hfi1/platform.c
@@ -668,8 +668,8 @@
/* active optical cables only */
switch ((cache[QSFP_MOD_TECH_OFFS] & 0xF0) >> 4) {
- case 0x0 ... 0x9: /* fallthrough */
- case 0xC: /* fallthrough */
+ case 0x0 ... 0x9: fallthrough;
+ case 0xC: fallthrough;
case 0xE:
/* active AOC */
power_class = get_qsfp_power_class(cache[QSFP_MOD_PWR_OFFS]);
@@ -899,8 +899,8 @@
*ptr_tuning_method = OPA_PASSIVE_TUNING;
break;
- case 0x0 ... 0x9: /* fallthrough */
- case 0xC: /* fallthrough */
+ case 0x0 ... 0x9: fallthrough;
+ case 0xC: fallthrough;
case 0xE:
ret = tune_active_qsfp(ppd, ptr_tx_preset, ptr_rx_preset,
ptr_total_atten);
@@ -909,7 +909,7 @@
*ptr_tuning_method = OPA_ACTIVE_TUNING;
break;
- case 0xD: /* fallthrough */
+ case 0xD: fallthrough;
case 0xF:
default:
dd_dev_warn(ppd->dd, "%s: Unknown/unsupported cable\n",
diff --git a/drivers/infiniband/hw/hfi1/qp.c b/drivers/infiniband/hw/hfi1/qp.c
index be62284..356518e 100644
--- a/drivers/infiniband/hw/hfi1/qp.c
+++ b/drivers/infiniband/hw/hfi1/qp.c
@@ -312,7 +312,7 @@
switch (qp->ibqp.qp_type) {
case IB_QPT_RC:
hfi1_setup_tid_rdma_wqe(qp, wqe);
- /* fall through */
+ fallthrough;
case IB_QPT_UC:
if (wqe->length > 0x80000000U)
return -EINVAL;
diff --git a/drivers/infiniband/hw/hfi1/qp.h b/drivers/infiniband/hw/hfi1/qp.h
index b670321..b0d053d 100644
--- a/drivers/infiniband/hw/hfi1/qp.h
+++ b/drivers/infiniband/hw/hfi1/qp.h
@@ -113,20 +113,6 @@
}
/**
- * hfi1_create_qp - create a queue pair for a device
- * @ibpd: the protection domain who's device we create the queue pair for
- * @init_attr: the attributes of the queue pair
- * @udata: user data for libibverbs.so
- *
- * Returns the queue pair on success, otherwise returns an errno.
- *
- * Called by the ib_create_qp() core verbs function.
- */
-struct ib_qp *hfi1_create_qp(struct ib_pd *ibpd,
- struct ib_qp_init_attr *init_attr,
- struct ib_udata *udata);
-
-/**
* hfi1_qp_wakeup - wake up on the indicated event
* @qp: the QP
* @flag: flag the qp on which the qp is stalled
diff --git a/drivers/infiniband/hw/hfi1/qsfp.c b/drivers/infiniband/hw/hfi1/qsfp.c
index b596699..8386c84 100644
--- a/drivers/infiniband/hw/hfi1/qsfp.c
+++ b/drivers/infiniband/hw/hfi1/qsfp.c
@@ -231,7 +231,7 @@
break;
case 2:
offset_bytes[1] = (offset >> 8) & 0xff;
- /* fall through */
+ fallthrough;
case 1:
num_msgs = 2;
offset_bytes[0] = offset & 0xff;
@@ -279,7 +279,7 @@
break;
case 2:
offset_bytes[1] = (offset >> 8) & 0xff;
- /* fall through */
+ fallthrough;
case 1:
num_msgs = 2;
offset_bytes[0] = offset & 0xff;
diff --git a/drivers/infiniband/hw/hfi1/rc.c b/drivers/infiniband/hw/hfi1/rc.c
index f1734e5..1bb5f57 100644
--- a/drivers/infiniband/hw/hfi1/rc.c
+++ b/drivers/infiniband/hw/hfi1/rc.c
@@ -141,7 +141,7 @@
case OP(RDMA_READ_RESPONSE_ONLY):
e = &qp->s_ack_queue[qp->s_tail_ack_queue];
release_rdma_sge_mr(e);
- /* FALLTHROUGH */
+ fallthrough;
case OP(ATOMIC_ACKNOWLEDGE):
/*
* We can increment the tail pointer now that the last
@@ -160,7 +160,7 @@
qp->s_acked_ack_queue = next;
qp->s_tail_ack_queue = next;
trace_hfi1_rsp_make_rc_ack(qp, e->psn);
- /* FALLTHROUGH */
+ fallthrough;
case OP(SEND_ONLY):
case OP(ACKNOWLEDGE):
/* Check for no next entry in the queue. */
@@ -267,7 +267,7 @@
case OP(RDMA_READ_RESPONSE_FIRST):
qp->s_ack_state = OP(RDMA_READ_RESPONSE_MIDDLE);
- /* FALLTHROUGH */
+ fallthrough;
case OP(RDMA_READ_RESPONSE_MIDDLE):
ps->s_txreq->ss = &qp->s_ack_rdma_sge;
ps->s_txreq->mr = qp->s_ack_rdma_sge.sge.mr;
@@ -881,8 +881,7 @@
goto bail;
}
qp->s_num_rd_atomic++;
-
- /* FALLTHROUGH */
+ fallthrough;
case IB_WR_OPFN:
if (newreq && !(qp->s_flags & RVT_S_UNLIMITED_CREDIT))
qp->s_lsn++;
@@ -946,10 +945,10 @@
* See restart_rc().
*/
qp->s_len = restart_sge(&qp->s_sge, wqe, qp->s_psn, pmtu);
- /* FALLTHROUGH */
+ fallthrough;
case OP(SEND_FIRST):
qp->s_state = OP(SEND_MIDDLE);
- /* FALLTHROUGH */
+ fallthrough;
case OP(SEND_MIDDLE):
bth2 = mask_psn(qp->s_psn++);
ss = &qp->s_sge;
@@ -991,10 +990,10 @@
* See restart_rc().
*/
qp->s_len = restart_sge(&qp->s_sge, wqe, qp->s_psn, pmtu);
- /* FALLTHROUGH */
+ fallthrough;
case OP(RDMA_WRITE_FIRST):
qp->s_state = OP(RDMA_WRITE_MIDDLE);
- /* FALLTHROUGH */
+ fallthrough;
case OP(RDMA_WRITE_MIDDLE):
bth2 = mask_psn(qp->s_psn++);
ss = &qp->s_sge;
@@ -2901,7 +2900,7 @@
if (!ret)
goto rnr_nak;
qp->r_rcv_len = 0;
- /* FALLTHROUGH */
+ fallthrough;
case OP(SEND_MIDDLE):
case OP(RDMA_WRITE_MIDDLE):
send_middle:
@@ -2941,7 +2940,7 @@
goto no_immediate_data;
if (opcode == OP(SEND_ONLY_WITH_INVALIDATE))
goto send_last_inv;
- /* FALLTHROUGH -- for SEND_ONLY_WITH_IMMEDIATE */
+ fallthrough; /* for SEND_ONLY_WITH_IMMEDIATE */
case OP(SEND_LAST_WITH_IMMEDIATE):
send_last_imm:
wc.ex.imm_data = ohdr->u.imm_data;
@@ -2957,7 +2956,7 @@
goto send_last;
case OP(RDMA_WRITE_LAST):
copy_last = rvt_is_user_qp(qp);
- /* fall through */
+ fallthrough;
case OP(SEND_LAST):
no_immediate_data:
wc.wc_flags = 0;
@@ -3010,7 +3009,7 @@
case OP(RDMA_WRITE_ONLY):
copy_last = rvt_is_user_qp(qp);
- /* fall through */
+ fallthrough;
case OP(RDMA_WRITE_FIRST):
case OP(RDMA_WRITE_ONLY_WITH_IMMEDIATE):
if (unlikely(!(qp->qp_access_flags & IB_ACCESS_REMOTE_WRITE)))
diff --git a/drivers/infiniband/hw/hfi1/sdma.c b/drivers/infiniband/hw/hfi1/sdma.c
index c93ea02..04575c9 100644
--- a/drivers/infiniband/hw/hfi1/sdma.c
+++ b/drivers/infiniband/hw/hfi1/sdma.c
@@ -2584,7 +2584,7 @@
* 7220, e.g.
*/
ss->go_s99_running = 1;
- /* fall through -- and start dma engine */
+ fallthrough; /* and start dma engine */
case sdma_event_e10_go_hw_start:
/* This reference means the state machine is started */
sdma_get(&sde->state);
@@ -2726,7 +2726,6 @@
case sdma_event_e70_go_idle:
break;
case sdma_event_e85_link_down:
- /* fall through */
case sdma_event_e80_hw_freeze:
sdma_set_state(sde, sdma_state_s80_hw_freeze);
atomic_dec(&sde->dd->sdma_unfreeze_count);
@@ -3007,7 +3006,7 @@
case sdma_event_e60_hw_halted:
need_progress = 1;
sdma_err_progress_check_schedule(sde);
- /* fall through */
+ fallthrough;
case sdma_event_e90_sw_halted:
/*
* SW initiated halt does not perform engines
@@ -3021,7 +3020,7 @@
break;
case sdma_event_e85_link_down:
ss->go_s99_running = 0;
- /* fall through */
+ fallthrough;
case sdma_event_e80_hw_freeze:
sdma_set_state(sde, sdma_state_s80_hw_freeze);
atomic_dec(&sde->dd->sdma_unfreeze_count);
@@ -3252,7 +3251,7 @@
tx->num_desc++;
tx->descs[2].qw[0] = 0;
tx->descs[2].qw[1] = 0;
- /* FALLTHROUGH */
+ fallthrough;
case SDMA_AHG_APPLY_UPDATE2:
tx->num_desc++;
tx->descs[1].qw[0] = 0;
diff --git a/drivers/infiniband/hw/hfi1/tid_rdma.c b/drivers/infiniband/hw/hfi1/tid_rdma.c
index facff13..9af82ff 100644
--- a/drivers/infiniband/hw/hfi1/tid_rdma.c
+++ b/drivers/infiniband/hw/hfi1/tid_rdma.c
@@ -3227,7 +3227,7 @@
case IB_WR_RDMA_READ:
if (prev->wr.opcode != IB_WR_TID_RDMA_WRITE)
break;
- /* fall through */
+ fallthrough;
case IB_WR_TID_RDMA_READ:
switch (prev->wr.opcode) {
case IB_WR_RDMA_READ:
@@ -5067,7 +5067,7 @@
if (priv->s_state == TID_OP(WRITE_REQ))
hfi1_tid_rdma_restart_req(qp, wqe, &bth2);
priv->s_state = TID_OP(WRITE_DATA);
- /* fall through */
+ fallthrough;
case TID_OP(WRITE_DATA):
/*
diff --git a/drivers/infiniband/hw/hfi1/uc.c b/drivers/infiniband/hw/hfi1/uc.c
index 0c77f18..1fb9183 100644
--- a/drivers/infiniband/hw/hfi1/uc.c
+++ b/drivers/infiniband/hw/hfi1/uc.c
@@ -216,7 +216,7 @@
case OP(SEND_FIRST):
qp->s_state = OP(SEND_MIDDLE);
- /* FALLTHROUGH */
+ fallthrough;
case OP(SEND_MIDDLE):
len = qp->s_len;
if (len > pmtu) {
@@ -241,7 +241,7 @@
case OP(RDMA_WRITE_FIRST):
qp->s_state = OP(RDMA_WRITE_MIDDLE);
- /* FALLTHROUGH */
+ fallthrough;
case OP(RDMA_WRITE_MIDDLE):
len = qp->s_len;
if (len > pmtu) {
@@ -414,7 +414,7 @@
goto no_immediate_data;
else if (opcode == OP(SEND_ONLY_WITH_IMMEDIATE))
goto send_last_imm;
- /* FALLTHROUGH */
+ fallthrough;
case OP(SEND_MIDDLE):
/* Check for invalid length PMTU or posted rwqe len. */
/*
@@ -515,7 +515,7 @@
wc.ex.imm_data = ohdr->u.rc.imm_data;
goto rdma_last_imm;
}
- /* FALLTHROUGH */
+ fallthrough;
case OP(RDMA_WRITE_MIDDLE):
/* Check for invalid length PMTU or posted rwqe len. */
if (unlikely(tlen != (hdrsize + pmtu + 4)))
diff --git a/drivers/infiniband/hw/hns/hns_roce_device.h b/drivers/infiniband/hw/hns/hns_roce_device.h
index 479fa55..da9888d 100644
--- a/drivers/infiniband/hw/hns/hns_roce_device.h
+++ b/drivers/infiniband/hw/hns/hns_roce_device.h
@@ -37,9 +37,8 @@
#define DRV_NAME "hns_roce"
-/* hip08 is a pci device, it includes two version according pci version id */
-#define PCI_REVISION_ID_HIP08_A 0x20
-#define PCI_REVISION_ID_HIP08_B 0x21
+/* hip08 is a pci device */
+#define PCI_REVISION_ID_HIP08 0x21
#define HNS_ROCE_HW_VER1 ('h' << 24 | 'i' << 16 | '0' << 8 | '6')
@@ -348,20 +347,22 @@
bool mtt_only; /* only alloc buffer-required MTT memory */
};
+struct hns_roce_hem_cfg {
+ dma_addr_t root_ba; /* root BA table's address */
+ bool is_direct; /* addressing without BA table */
+ unsigned int ba_pg_shift; /* BA table page shift */
+ unsigned int buf_pg_shift; /* buffer page shift */
+ unsigned int buf_pg_count; /* buffer page count */
+ struct hns_roce_buf_region region[HNS_ROCE_MAX_BT_REGION];
+ int region_count;
+};
+
/* memory translate region */
struct hns_roce_mtr {
struct hns_roce_hem_list hem_list; /* multi-hop addressing resource */
struct ib_umem *umem; /* user space buffer */
struct hns_roce_buf *kmem; /* kernel space buffer */
- struct {
- dma_addr_t root_ba; /* root BA table's address */
- bool is_direct; /* addressing without BA table */
- unsigned int ba_pg_shift; /* BA table page shift */
- unsigned int buf_pg_shift; /* buffer page shift */
- int buf_pg_count; /* buffer page count */
- struct hns_roce_buf_region region[HNS_ROCE_MAX_BT_REGION];
- unsigned int region_count;
- } hem_cfg; /* config for hardware addressing */
+ struct hns_roce_hem_cfg hem_cfg; /* config for hardware addressing */
};
struct hns_roce_mw {
@@ -1192,7 +1193,7 @@
u64 virt_addr, int mr_access_flags, struct ib_pd *pd,
struct ib_udata *udata);
struct ib_mr *hns_roce_alloc_mr(struct ib_pd *pd, enum ib_mr_type mr_type,
- u32 max_num_sg, struct ib_udata *udata);
+ u32 max_num_sg);
int hns_roce_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg, int sg_nents,
unsigned int *sg_offset);
int hns_roce_dereg_mr(struct ib_mr *ibmr, struct ib_udata *udata);
@@ -1267,6 +1268,6 @@
int hns_roce_init(struct hns_roce_dev *hr_dev);
void hns_roce_exit(struct hns_roce_dev *hr_dev);
-int hns_roce_fill_res_entry(struct sk_buff *msg,
- struct rdma_restrack_entry *res);
+int hns_roce_fill_res_cq_entry(struct sk_buff *msg,
+ struct ib_cq *ib_cq);
#endif /* _HNS_ROCE_DEVICE_H */
diff --git a/drivers/infiniband/hw/hns/hns_roce_hw_v1.c b/drivers/infiniband/hw/hns/hns_roce_hw_v1.c
index cf39f56..07b4c85 100644
--- a/drivers/infiniband/hw/hns/hns_roce_hw_v1.c
+++ b/drivers/infiniband/hw/hns/hns_roce_hw_v1.c
@@ -2483,7 +2483,6 @@
u64 *sq_ba, u64 *rq_ba, dma_addr_t *bt_ba)
{
struct ib_device *ibdev = &hr_dev->ib_dev;
- int rq_pa_start;
int count;
count = hns_roce_mtr_find(hr_dev, &hr_qp->mtr, 0, sq_ba, 1, bt_ba);
@@ -2491,9 +2490,9 @@
ibdev_err(ibdev, "Failed to find SQ ba\n");
return -ENOBUFS;
}
- rq_pa_start = hr_qp->rq.offset >> hr_qp->mtr.hem_cfg.buf_pg_shift;
- count = hns_roce_mtr_find(hr_dev, &hr_qp->mtr, rq_pa_start, rq_ba, 1,
- NULL);
+
+ count = hns_roce_mtr_find(hr_dev, &hr_qp->mtr, hr_qp->rq.offset, rq_ba,
+ 1, NULL);
if (!count) {
ibdev_err(ibdev, "Failed to find RQ ba\n");
return -ENOBUFS;
diff --git a/drivers/infiniband/hw/hns/hns_roce_hw_v2.c b/drivers/infiniband/hw/hns/hns_roce_hw_v2.c
index 0618ced..d296859 100644
--- a/drivers/infiniband/hw/hns/hns_roce_hw_v2.c
+++ b/drivers/infiniband/hw/hns/hns_roce_hw_v2.c
@@ -91,10 +91,11 @@
}
static void set_frmr_seg(struct hns_roce_v2_rc_send_wqe *rc_sq_wqe,
- void *wqe, const struct ib_reg_wr *wr)
+ const struct ib_reg_wr *wr)
{
+ struct hns_roce_wqe_frmr_seg *fseg =
+ (void *)rc_sq_wqe + sizeof(struct hns_roce_v2_rc_send_wqe);
struct hns_roce_mr *mr = to_hr_mr(wr->mr);
- struct hns_roce_wqe_frmr_seg *fseg = wqe;
u64 pbl_ba;
/* use ib_access_flags */
@@ -128,14 +129,16 @@
V2_RC_FRMR_WQE_BYTE_40_BLK_MODE_S, 0);
}
-static void set_atomic_seg(const struct ib_send_wr *wr, void *wqe,
+static void set_atomic_seg(const struct ib_send_wr *wr,
struct hns_roce_v2_rc_send_wqe *rc_sq_wqe,
unsigned int valid_num_sge)
{
- struct hns_roce_wqe_atomic_seg *aseg;
+ struct hns_roce_v2_wqe_data_seg *dseg =
+ (void *)rc_sq_wqe + sizeof(struct hns_roce_v2_rc_send_wqe);
+ struct hns_roce_wqe_atomic_seg *aseg =
+ (void *)dseg + sizeof(struct hns_roce_v2_wqe_data_seg);
- set_data_seg_v2(wqe, wr->sg_list);
- aseg = wqe + sizeof(struct hns_roce_v2_wqe_data_seg);
+ set_data_seg_v2(dseg, wr->sg_list);
if (wr->opcode == IB_WR_ATOMIC_CMP_AND_SWP) {
aseg->fetchadd_swap_data = cpu_to_le64(atomic_wr(wr)->swap);
@@ -143,7 +146,7 @@
} else {
aseg->fetchadd_swap_data =
cpu_to_le64(atomic_wr(wr)->compare_add);
- aseg->cmp_data = 0;
+ aseg->cmp_data = 0;
}
roce_set_field(rc_sq_wqe->byte_16, V2_RC_SEND_WQE_BYTE_16_SGE_NUM_M,
@@ -176,13 +179,15 @@
static int set_rwqe_data_seg(struct ib_qp *ibqp, const struct ib_send_wr *wr,
struct hns_roce_v2_rc_send_wqe *rc_sq_wqe,
- void *wqe, unsigned int *sge_ind,
+ unsigned int *sge_ind,
unsigned int valid_num_sge)
{
struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);
- struct hns_roce_v2_wqe_data_seg *dseg = wqe;
+ struct hns_roce_v2_wqe_data_seg *dseg =
+ (void *)rc_sq_wqe + sizeof(struct hns_roce_v2_rc_send_wqe);
struct ib_device *ibdev = &hr_dev->ib_dev;
struct hns_roce_qp *qp = to_hr_qp(ibqp);
+ void *wqe = dseg;
int j = 0;
int i;
@@ -438,7 +443,6 @@
roce_set_bit(rc_sq_wqe->byte_4, V2_RC_SEND_WQE_BYTE_4_OWNER_S,
owner_bit);
- wqe += sizeof(struct hns_roce_v2_rc_send_wqe);
switch (wr->opcode) {
case IB_WR_RDMA_READ:
case IB_WR_RDMA_WRITE:
@@ -451,7 +455,7 @@
rc_sq_wqe->inv_key = cpu_to_le32(wr->ex.invalidate_rkey);
break;
case IB_WR_REG_MR:
- set_frmr_seg(rc_sq_wqe, wqe, reg_wr(wr));
+ set_frmr_seg(rc_sq_wqe, reg_wr(wr));
break;
case IB_WR_ATOMIC_CMP_AND_SWP:
case IB_WR_ATOMIC_FETCH_AND_ADD:
@@ -468,10 +472,10 @@
if (wr->opcode == IB_WR_ATOMIC_CMP_AND_SWP ||
wr->opcode == IB_WR_ATOMIC_FETCH_AND_ADD)
- set_atomic_seg(wr, wqe, rc_sq_wqe, valid_num_sge);
+ set_atomic_seg(wr, rc_sq_wqe, valid_num_sge);
else if (wr->opcode != IB_WR_REG_MR)
ret = set_rwqe_data_seg(&qp->ibqp, wr, rc_sq_wqe,
- wqe, &curr_idx, valid_num_sge);
+ &curr_idx, valid_num_sge);
*sge_idx = curr_idx;
@@ -1510,8 +1514,6 @@
req_a = (struct hns_roce_vf_res_a *)desc[0].data;
req_b = (struct hns_roce_vf_res_b *)desc[1].data;
- memset(req_a, 0, sizeof(*req_a));
- memset(req_b, 0, sizeof(*req_b));
for (i = 0; i < 2; i++) {
hns_roce_cmq_setup_basic_desc(&desc[i],
HNS_ROCE_OPC_ALLOC_VF_RES, false);
@@ -1744,27 +1746,25 @@
caps->max_srq_wrs = HNS_ROCE_V2_MAX_SRQ_WR;
caps->max_srq_sges = HNS_ROCE_V2_MAX_SRQ_SGE;
- if (hr_dev->pci_dev->revision >= PCI_REVISION_ID_HIP08_B) {
- caps->flags |= HNS_ROCE_CAP_FLAG_ATOMIC | HNS_ROCE_CAP_FLAG_MW |
- HNS_ROCE_CAP_FLAG_SRQ | HNS_ROCE_CAP_FLAG_FRMR |
- HNS_ROCE_CAP_FLAG_QP_FLOW_CTRL;
+ caps->flags |= HNS_ROCE_CAP_FLAG_ATOMIC | HNS_ROCE_CAP_FLAG_MW |
+ HNS_ROCE_CAP_FLAG_SRQ | HNS_ROCE_CAP_FLAG_FRMR |
+ HNS_ROCE_CAP_FLAG_QP_FLOW_CTRL;
- caps->num_qpc_timer = HNS_ROCE_V2_MAX_QPC_TIMER_NUM;
- caps->qpc_timer_entry_sz = HNS_ROCE_V2_QPC_TIMER_ENTRY_SZ;
- caps->qpc_timer_ba_pg_sz = 0;
- caps->qpc_timer_buf_pg_sz = 0;
- caps->qpc_timer_hop_num = HNS_ROCE_HOP_NUM_0;
- caps->num_cqc_timer = HNS_ROCE_V2_MAX_CQC_TIMER_NUM;
- caps->cqc_timer_entry_sz = HNS_ROCE_V2_CQC_TIMER_ENTRY_SZ;
- caps->cqc_timer_ba_pg_sz = 0;
- caps->cqc_timer_buf_pg_sz = 0;
- caps->cqc_timer_hop_num = HNS_ROCE_HOP_NUM_0;
+ caps->num_qpc_timer = HNS_ROCE_V2_MAX_QPC_TIMER_NUM;
+ caps->qpc_timer_entry_sz = HNS_ROCE_V2_QPC_TIMER_ENTRY_SZ;
+ caps->qpc_timer_ba_pg_sz = 0;
+ caps->qpc_timer_buf_pg_sz = 0;
+ caps->qpc_timer_hop_num = HNS_ROCE_HOP_NUM_0;
+ caps->num_cqc_timer = HNS_ROCE_V2_MAX_CQC_TIMER_NUM;
+ caps->cqc_timer_entry_sz = HNS_ROCE_V2_CQC_TIMER_ENTRY_SZ;
+ caps->cqc_timer_ba_pg_sz = 0;
+ caps->cqc_timer_buf_pg_sz = 0;
+ caps->cqc_timer_hop_num = HNS_ROCE_HOP_NUM_0;
- caps->sccc_entry_sz = HNS_ROCE_V2_SCCC_ENTRY_SZ;
- caps->sccc_ba_pg_sz = 0;
- caps->sccc_buf_pg_sz = 0;
- caps->sccc_hop_num = HNS_ROCE_SCCC_HOP_NUM;
- }
+ caps->sccc_entry_sz = HNS_ROCE_V2_SCCC_ENTRY_SZ;
+ caps->sccc_ba_pg_sz = 0;
+ caps->sccc_buf_pg_sz = 0;
+ caps->sccc_hop_num = HNS_ROCE_SCCC_HOP_NUM;
}
static void calc_pg_sz(int obj_num, int obj_size, int hop_num, int ctx_bt_num,
@@ -1995,20 +1995,18 @@
caps->srqc_bt_num, &caps->srqc_buf_pg_sz,
&caps->srqc_ba_pg_sz, HEM_TYPE_SRQC);
- if (hr_dev->pci_dev->revision >= PCI_REVISION_ID_HIP08_B) {
- caps->sccc_hop_num = ctx_hop_num;
- caps->qpc_timer_hop_num = HNS_ROCE_HOP_NUM_0;
- caps->cqc_timer_hop_num = HNS_ROCE_HOP_NUM_0;
+ caps->sccc_hop_num = ctx_hop_num;
+ caps->qpc_timer_hop_num = HNS_ROCE_HOP_NUM_0;
+ caps->cqc_timer_hop_num = HNS_ROCE_HOP_NUM_0;
- calc_pg_sz(caps->num_qps, caps->sccc_entry_sz,
- caps->sccc_hop_num, caps->sccc_bt_num,
- &caps->sccc_buf_pg_sz, &caps->sccc_ba_pg_sz,
- HEM_TYPE_SCCC);
- calc_pg_sz(caps->num_cqc_timer, caps->cqc_timer_entry_sz,
- caps->cqc_timer_hop_num, caps->cqc_timer_bt_num,
- &caps->cqc_timer_buf_pg_sz,
- &caps->cqc_timer_ba_pg_sz, HEM_TYPE_CQC_TIMER);
- }
+ calc_pg_sz(caps->num_qps, caps->sccc_entry_sz,
+ caps->sccc_hop_num, caps->sccc_bt_num,
+ &caps->sccc_buf_pg_sz, &caps->sccc_ba_pg_sz,
+ HEM_TYPE_SCCC);
+ calc_pg_sz(caps->num_cqc_timer, caps->cqc_timer_entry_sz,
+ caps->cqc_timer_hop_num, caps->cqc_timer_bt_num,
+ &caps->cqc_timer_buf_pg_sz,
+ &caps->cqc_timer_ba_pg_sz, HEM_TYPE_CQC_TIMER);
calc_pg_sz(caps->num_cqe_segs, caps->mtt_entry_sz, caps->cqe_hop_num,
1, &caps->cqe_buf_pg_sz, &caps->cqe_ba_pg_sz, HEM_TYPE_CQE);
@@ -2055,22 +2053,19 @@
return ret;
}
- if (hr_dev->pci_dev->revision >= PCI_REVISION_ID_HIP08_B) {
- ret = hns_roce_query_pf_timer_resource(hr_dev);
- if (ret) {
- dev_err(hr_dev->dev,
- "Query pf timer resource fail, ret = %d.\n",
- ret);
- return ret;
- }
+ ret = hns_roce_query_pf_timer_resource(hr_dev);
+ if (ret) {
+ dev_err(hr_dev->dev,
+ "failed to query pf timer resource, ret = %d.\n", ret);
+ return ret;
+ }
- ret = hns_roce_set_vf_switch_param(hr_dev, 0);
- if (ret) {
- dev_err(hr_dev->dev,
- "Set function switch param fail, ret = %d.\n",
- ret);
- return ret;
- }
+ ret = hns_roce_set_vf_switch_param(hr_dev, 0);
+ if (ret) {
+ dev_err(hr_dev->dev,
+ "failed to set function switch param, ret = %d.\n",
+ ret);
+ return ret;
}
hr_dev->vendor_part_id = hr_dev->pci_dev->device;
@@ -2336,8 +2331,7 @@
{
struct hns_roce_v2_priv *priv = hr_dev->priv;
- if (hr_dev->pci_dev->revision >= PCI_REVISION_ID_HIP08_B)
- hns_roce_function_clear(hr_dev);
+ hns_roce_function_clear(hr_dev);
hns_roce_free_link_table(hr_dev, &priv->tpq);
hns_roce_free_link_table(hr_dev, &priv->tsq);
@@ -3053,6 +3047,7 @@
IB_WC_RETRY_EXC_ERR },
{ HNS_ROCE_CQE_V2_RNR_RETRY_EXC_ERR, IB_WC_RNR_RETRY_EXC_ERR },
{ HNS_ROCE_CQE_V2_REMOTE_ABORT_ERR, IB_WC_REM_ABORT_ERR },
+ { HNS_ROCE_CQE_V2_GENERAL_ERR, IB_WC_GENERAL_ERR}
};
u32 cqe_status = roce_get_field(cqe->byte_4, V2_CQE_BYTE_4_STATUS_M,
@@ -3075,6 +3070,14 @@
sizeof(*cqe), false);
/*
+ * For hns ROCEE, GENERAL_ERR is an error type that is not defined in
+ * the standard protocol, the driver must ignore it and needn't to set
+ * the QP to an error state.
+ */
+ if (cqe_status == HNS_ROCE_CQE_V2_GENERAL_ERR)
+ return;
+
+ /*
* Hip08 hardware cannot flush the WQEs in SQ/RQ if the QP state gets
* into errored mode. Hence, as a workaround to this hardware
* limitation, driver needs to assist in flushing. But the flushing
@@ -3170,51 +3173,51 @@
/* SQ corresponding to CQE */
switch (roce_get_field(cqe->byte_4, V2_CQE_BYTE_4_OPCODE_M,
V2_CQE_BYTE_4_OPCODE_S) & 0x1f) {
- case HNS_ROCE_SQ_OPCODE_SEND:
+ case HNS_ROCE_V2_WQE_OP_SEND:
wc->opcode = IB_WC_SEND;
break;
- case HNS_ROCE_SQ_OPCODE_SEND_WITH_INV:
+ case HNS_ROCE_V2_WQE_OP_SEND_WITH_INV:
wc->opcode = IB_WC_SEND;
break;
- case HNS_ROCE_SQ_OPCODE_SEND_WITH_IMM:
+ case HNS_ROCE_V2_WQE_OP_SEND_WITH_IMM:
wc->opcode = IB_WC_SEND;
wc->wc_flags |= IB_WC_WITH_IMM;
break;
- case HNS_ROCE_SQ_OPCODE_RDMA_READ:
+ case HNS_ROCE_V2_WQE_OP_RDMA_READ:
wc->opcode = IB_WC_RDMA_READ;
wc->byte_len = le32_to_cpu(cqe->byte_cnt);
break;
- case HNS_ROCE_SQ_OPCODE_RDMA_WRITE:
+ case HNS_ROCE_V2_WQE_OP_RDMA_WRITE:
wc->opcode = IB_WC_RDMA_WRITE;
break;
- case HNS_ROCE_SQ_OPCODE_RDMA_WRITE_WITH_IMM:
+ case HNS_ROCE_V2_WQE_OP_RDMA_WRITE_WITH_IMM:
wc->opcode = IB_WC_RDMA_WRITE;
wc->wc_flags |= IB_WC_WITH_IMM;
break;
- case HNS_ROCE_SQ_OPCODE_LOCAL_INV:
+ case HNS_ROCE_V2_WQE_OP_LOCAL_INV:
wc->opcode = IB_WC_LOCAL_INV;
wc->wc_flags |= IB_WC_WITH_INVALIDATE;
break;
- case HNS_ROCE_SQ_OPCODE_ATOMIC_COMP_AND_SWAP:
+ case HNS_ROCE_V2_WQE_OP_ATOM_CMP_AND_SWAP:
wc->opcode = IB_WC_COMP_SWAP;
wc->byte_len = 8;
break;
- case HNS_ROCE_SQ_OPCODE_ATOMIC_FETCH_AND_ADD:
+ case HNS_ROCE_V2_WQE_OP_ATOM_FETCH_AND_ADD:
wc->opcode = IB_WC_FETCH_ADD;
wc->byte_len = 8;
break;
- case HNS_ROCE_SQ_OPCODE_ATOMIC_MASK_COMP_AND_SWAP:
+ case HNS_ROCE_V2_WQE_OP_ATOM_MSK_CMP_AND_SWAP:
wc->opcode = IB_WC_MASKED_COMP_SWAP;
wc->byte_len = 8;
break;
- case HNS_ROCE_SQ_OPCODE_ATOMIC_MASK_FETCH_AND_ADD:
+ case HNS_ROCE_V2_WQE_OP_ATOM_MSK_FETCH_AND_ADD:
wc->opcode = IB_WC_MASKED_FETCH_ADD;
wc->byte_len = 8;
break;
- case HNS_ROCE_SQ_OPCODE_FAST_REG_WR:
+ case HNS_ROCE_V2_WQE_OP_FAST_REG_PMR:
wc->opcode = IB_WC_REG_MR;
break;
- case HNS_ROCE_SQ_OPCODE_BIND_MW:
+ case HNS_ROCE_V2_WQE_OP_BIND_MW:
wc->opcode = IB_WC_REG_MR;
break;
default:
@@ -3374,11 +3377,33 @@
return op + step_idx;
}
+static int set_hem_to_hw(struct hns_roce_dev *hr_dev, int obj, u64 bt_ba,
+ u32 hem_type, int step_idx)
+{
+ struct hns_roce_cmd_mailbox *mailbox;
+ int ret;
+ int op;
+
+ op = get_op_for_set_hem(hr_dev, hem_type, step_idx);
+ if (op < 0)
+ return 0;
+
+ mailbox = hns_roce_alloc_cmd_mailbox(hr_dev);
+ if (IS_ERR(mailbox))
+ return PTR_ERR(mailbox);
+
+ ret = hns_roce_cmd_mbox(hr_dev, bt_ba, mailbox->dma, obj,
+ 0, op, HNS_ROCE_CMD_TIMEOUT_MSECS);
+
+ hns_roce_free_cmd_mailbox(hr_dev, mailbox);
+
+ return ret;
+}
+
static int hns_roce_v2_set_hem(struct hns_roce_dev *hr_dev,
struct hns_roce_hem_table *table, int obj,
int step_idx)
{
- struct hns_roce_cmd_mailbox *mailbox;
struct hns_roce_hem_iter iter;
struct hns_roce_hem_mhop mhop;
struct hns_roce_hem *hem;
@@ -3390,7 +3415,6 @@
u64 bt_ba = 0;
u32 chunk_ba_num;
u32 hop_num;
- int op;
if (!hns_roce_check_whether_mhop(hr_dev, table->type))
return 0;
@@ -3412,14 +3436,6 @@
hem_idx = i;
}
- op = get_op_for_set_hem(hr_dev, table->type, step_idx);
- if (op == -EINVAL)
- return 0;
-
- mailbox = hns_roce_alloc_cmd_mailbox(hr_dev);
- if (IS_ERR(mailbox))
- return PTR_ERR(mailbox);
-
if (table->type == HEM_TYPE_SCCC)
obj = mhop.l0_idx;
@@ -3428,11 +3444,8 @@
for (hns_roce_hem_first(hem, &iter);
!hns_roce_hem_last(&iter); hns_roce_hem_next(&iter)) {
bt_ba = hns_roce_hem_addr(&iter);
-
- /* configure the ba, tag, and op */
- ret = hns_roce_cmd_mbox(hr_dev, bt_ba, mailbox->dma,
- obj, 0, op,
- HNS_ROCE_CMD_TIMEOUT_MSECS);
+ ret = set_hem_to_hw(hr_dev, obj, bt_ba, table->type,
+ step_idx);
}
} else {
if (step_idx == 0)
@@ -3440,12 +3453,9 @@
else if (step_idx == 1 && hop_num == 2)
bt_ba = table->bt_l1_dma_addr[l1_idx];
- /* configure the ba, tag, and op */
- ret = hns_roce_cmd_mbox(hr_dev, bt_ba, mailbox->dma, obj,
- 0, op, HNS_ROCE_CMD_TIMEOUT_MSECS);
+ ret = set_hem_to_hw(hr_dev, obj, bt_ba, table->type, step_idx);
}
- hns_roce_free_cmd_mailbox(hr_dev, mailbox);
return ret;
}
@@ -3745,51 +3755,23 @@
}
}
-static bool check_wqe_rq_mtt_count(struct hns_roce_dev *hr_dev,
- struct hns_roce_qp *hr_qp, int mtt_cnt,
- u32 page_size)
-{
- struct ib_device *ibdev = &hr_dev->ib_dev;
-
- if (hr_qp->rq.wqe_cnt < 1)
- return true;
-
- if (mtt_cnt < 1) {
- ibdev_err(ibdev, "failed to find RQWQE buf ba of QP(0x%lx)\n",
- hr_qp->qpn);
- return false;
- }
-
- if (mtt_cnt < MTT_MIN_COUNT &&
- (hr_qp->rq.offset + page_size) < hr_qp->buff_size) {
- ibdev_err(ibdev,
- "failed to find next RQWQE buf ba of QP(0x%lx)\n",
- hr_qp->qpn);
- return false;
- }
-
- return true;
-}
-
static int config_qp_rq_buf(struct hns_roce_dev *hr_dev,
struct hns_roce_qp *hr_qp,
struct hns_roce_v2_qp_context *context,
struct hns_roce_v2_qp_context *qpc_mask)
{
- struct ib_qp *ibqp = &hr_qp->ibqp;
u64 mtts[MTT_MIN_COUNT] = { 0 };
u64 wqe_sge_ba;
- u32 page_size;
int count;
/* Search qp buf's mtts */
- page_size = 1 << hr_qp->mtr.hem_cfg.buf_pg_shift;
- count = hns_roce_mtr_find(hr_dev, &hr_qp->mtr,
- hr_qp->rq.offset / page_size, mtts,
+ count = hns_roce_mtr_find(hr_dev, &hr_qp->mtr, hr_qp->rq.offset, mtts,
MTT_MIN_COUNT, &wqe_sge_ba);
- if (!ibqp->srq)
- if (!check_wqe_rq_mtt_count(hr_dev, hr_qp, count, page_size))
- return -EINVAL;
+ if (hr_qp->rq.wqe_cnt && count < 1) {
+ ibdev_err(&hr_dev->ib_dev,
+ "failed to find RQ WQE, QPN = 0x%lx.\n", hr_qp->qpn);
+ return -EINVAL;
+ }
context->wqe_sge_ba = cpu_to_le32(wqe_sge_ba >> 3);
qpc_mask->wqe_sge_ba = 0;
@@ -3891,7 +3873,6 @@
struct ib_device *ibdev = &hr_dev->ib_dev;
u64 sge_cur_blk = 0;
u64 sq_cur_blk = 0;
- u32 page_size;
int count;
/* search qp buf's mtts */
@@ -3902,9 +3883,8 @@
return -EINVAL;
}
if (hr_qp->sge.sge_cnt > 0) {
- page_size = 1 << hr_qp->mtr.hem_cfg.buf_pg_shift;
count = hns_roce_mtr_find(hr_dev, &hr_qp->mtr,
- hr_qp->sge.offset / page_size,
+ hr_qp->sge.offset,
&sge_cur_blk, 1, NULL);
if (count < 1) {
ibdev_err(ibdev, "failed to find QP(0x%lx) SGE buf.\n",
@@ -4265,12 +4245,13 @@
roce_set_field(qpc_mask->byte_24_mtu_tc, V2_QPC_BYTE_24_HOP_LIMIT_M,
V2_QPC_BYTE_24_HOP_LIMIT_S, 0);
- if (hr_dev->pci_dev->revision >= PCI_REVISION_ID_HIP08_B && is_udp)
+ if (is_udp)
roce_set_field(context->byte_24_mtu_tc, V2_QPC_BYTE_24_TC_M,
V2_QPC_BYTE_24_TC_S, grh->traffic_class >> 2);
else
roce_set_field(context->byte_24_mtu_tc, V2_QPC_BYTE_24_TC_M,
V2_QPC_BYTE_24_TC_S, grh->traffic_class);
+
roce_set_field(qpc_mask->byte_24_mtu_tc, V2_QPC_BYTE_24_TC_M,
V2_QPC_BYTE_24_TC_S, 0);
roce_set_field(context->byte_28_at_fl, V2_QPC_BYTE_28_FL_M,
@@ -4301,7 +4282,9 @@
[IB_QPS_RTR] = { [IB_QPS_RESET] = true,
[IB_QPS_RTS] = true,
[IB_QPS_ERR] = true },
- [IB_QPS_RTS] = { [IB_QPS_RESET] = true, [IB_QPS_ERR] = true },
+ [IB_QPS_RTS] = { [IB_QPS_RESET] = true,
+ [IB_QPS_RTS] = true,
+ [IB_QPS_ERR] = true },
[IB_QPS_SQD] = {},
[IB_QPS_SQE] = {},
[IB_QPS_ERR] = { [IB_QPS_RESET] = true, [IB_QPS_ERR] = true }
diff --git a/drivers/infiniband/hw/hns/hns_roce_hw_v2.h b/drivers/infiniband/hw/hns/hns_roce_hw_v2.h
index e176b0a..1fb1c58 100644
--- a/drivers/infiniband/hw/hns/hns_roce_hw_v2.h
+++ b/drivers/infiniband/hw/hns/hns_roce_hw_v2.h
@@ -179,27 +179,11 @@
HNS_ROCE_V2_WQE_OP_ATOM_MSK_FETCH_AND_ADD = 0x9,
HNS_ROCE_V2_WQE_OP_FAST_REG_PMR = 0xa,
HNS_ROCE_V2_WQE_OP_LOCAL_INV = 0xb,
- HNS_ROCE_V2_WQE_OP_BIND_MW_TYPE = 0xc,
+ HNS_ROCE_V2_WQE_OP_BIND_MW = 0xc,
HNS_ROCE_V2_WQE_OP_MASK = 0x1f,
};
enum {
- HNS_ROCE_SQ_OPCODE_SEND = 0x0,
- HNS_ROCE_SQ_OPCODE_SEND_WITH_INV = 0x1,
- HNS_ROCE_SQ_OPCODE_SEND_WITH_IMM = 0x2,
- HNS_ROCE_SQ_OPCODE_RDMA_WRITE = 0x3,
- HNS_ROCE_SQ_OPCODE_RDMA_WRITE_WITH_IMM = 0x4,
- HNS_ROCE_SQ_OPCODE_RDMA_READ = 0x5,
- HNS_ROCE_SQ_OPCODE_ATOMIC_COMP_AND_SWAP = 0x6,
- HNS_ROCE_SQ_OPCODE_ATOMIC_FETCH_AND_ADD = 0x7,
- HNS_ROCE_SQ_OPCODE_ATOMIC_MASK_COMP_AND_SWAP = 0x8,
- HNS_ROCE_SQ_OPCODE_ATOMIC_MASK_FETCH_AND_ADD = 0x9,
- HNS_ROCE_SQ_OPCODE_FAST_REG_WR = 0xa,
- HNS_ROCE_SQ_OPCODE_LOCAL_INV = 0xb,
- HNS_ROCE_SQ_OPCODE_BIND_MW = 0xc,
-};
-
-enum {
/* rq operations */
HNS_ROCE_V2_OPCODE_RDMA_WRITE_IMM = 0x0,
HNS_ROCE_V2_OPCODE_SEND = 0x1,
@@ -230,6 +214,7 @@
HNS_ROCE_CQE_V2_TRANSPORT_RETRY_EXC_ERR = 0x15,
HNS_ROCE_CQE_V2_RNR_RETRY_EXC_ERR = 0x16,
HNS_ROCE_CQE_V2_REMOTE_ABORT_ERR = 0x22,
+ HNS_ROCE_CQE_V2_GENERAL_ERR = 0x23,
HNS_ROCE_V2_CQE_STATUS_MASK = 0xff,
};
diff --git a/drivers/infiniband/hw/hns/hns_roce_main.c b/drivers/infiniband/hw/hns/hns_roce_main.c
index 50763cf..5907cfd 100644
--- a/drivers/infiniband/hw/hns/hns_roce_main.c
+++ b/drivers/infiniband/hw/hns/hns_roce_main.c
@@ -428,7 +428,7 @@
.destroy_ah = hns_roce_destroy_ah,
.destroy_cq = hns_roce_destroy_cq,
.disassociate_ucontext = hns_roce_disassociate_ucontext,
- .fill_res_entry = hns_roce_fill_res_entry,
+ .fill_res_cq_entry = hns_roce_fill_res_cq_entry,
.get_dma_mr = hns_roce_get_dma_mr,
.get_link_layer = hns_roce_get_link_layer,
.get_port_immutable = hns_roce_port_immutable,
diff --git a/drivers/infiniband/hw/hns/hns_roce_mr.c b/drivers/infiniband/hw/hns/hns_roce_mr.c
index 6b226a5..e5df388 100644
--- a/drivers/infiniband/hw/hns/hns_roce_mr.c
+++ b/drivers/infiniband/hw/hns/hns_roce_mr.c
@@ -415,7 +415,7 @@
}
struct ib_mr *hns_roce_alloc_mr(struct ib_pd *pd, enum ib_mr_type mr_type,
- u32 max_num_sg, struct ib_udata *udata)
+ u32 max_num_sg)
{
struct hns_roce_dev *hr_dev = to_hr_dev(pd->device);
struct device *dev = hr_dev->dev;
@@ -871,6 +871,15 @@
int err;
int i;
+ /*
+ * Only use the first page address as root ba when hopnum is 0, this
+ * is because the addresses of all pages are consecutive in this case.
+ */
+ if (mtr->hem_cfg.is_direct) {
+ mtr->hem_cfg.root_ba = pages[0];
+ return 0;
+ }
+
for (i = 0; i < mtr->hem_cfg.region_count; i++) {
r = &mtr->hem_cfg.region[i];
if (r->offset + r->count > page_cnt) {
@@ -896,6 +905,8 @@
int hns_roce_mtr_find(struct hns_roce_dev *hr_dev, struct hns_roce_mtr *mtr,
int offset, u64 *mtt_buf, int mtt_max, u64 *base_addr)
{
+ struct hns_roce_hem_cfg *cfg = &mtr->hem_cfg;
+ int start_index;
int mtt_count;
int total = 0;
__le64 *mtts;
@@ -907,26 +918,32 @@
goto done;
/* no mtt memory in direct mode, so just return the buffer address */
- if (mtr->hem_cfg.is_direct) {
- npage = offset;
- for (total = 0; total < mtt_max; total++, npage++) {
- addr = mtr->hem_cfg.root_ba +
- (npage << mtr->hem_cfg.buf_pg_shift);
+ if (cfg->is_direct) {
+ start_index = offset >> HNS_HW_PAGE_SHIFT;
+ for (mtt_count = 0; mtt_count < cfg->region_count &&
+ total < mtt_max; mtt_count++) {
+ npage = cfg->region[mtt_count].offset;
+ if (npage < start_index)
+ continue;
+ addr = cfg->root_ba + (npage << HNS_HW_PAGE_SHIFT);
if (hr_dev->hw_rev == HNS_ROCE_HW_VER1)
mtt_buf[total] = to_hr_hw_page_addr(addr);
else
mtt_buf[total] = addr;
+
+ total++;
}
goto done;
}
+ start_index = offset >> cfg->buf_pg_shift;
left = mtt_max;
while (left > 0) {
mtt_count = 0;
mtts = hns_roce_hem_list_find_mtt(hr_dev, &mtr->hem_list,
- offset + total,
+ start_index + total,
&mtt_count, NULL);
if (!mtts || !mtt_count)
goto done;
@@ -939,104 +956,136 @@
done:
if (base_addr)
- *base_addr = mtr->hem_cfg.root_ba;
+ *base_addr = cfg->root_ba;
return total;
}
-/* convert buffer size to page index and page count */
-static unsigned int mtr_init_region(struct hns_roce_buf_attr *attr,
- int page_cnt,
- struct hns_roce_buf_region *regions,
- int region_cnt, unsigned int page_shift)
+static int mtr_init_buf_cfg(struct hns_roce_dev *hr_dev,
+ struct hns_roce_buf_attr *attr,
+ struct hns_roce_hem_cfg *cfg,
+ unsigned int *buf_page_shift)
{
- unsigned int page_size = 1 << page_shift;
- int max_region = attr->region_count;
struct hns_roce_buf_region *r;
- unsigned int i = 0;
- int page_idx = 0;
+ unsigned int page_shift = 0;
+ int page_cnt = 0;
+ size_t buf_size;
+ int region_cnt;
- for (; i < region_cnt && i < max_region && page_idx < page_cnt; i++) {
- r = ®ions[i];
- r->hopnum = attr->region[i].hopnum == HNS_ROCE_HOP_NUM_0 ?
- 0 : attr->region[i].hopnum;
- r->offset = page_idx;
- r->count = DIV_ROUND_UP(attr->region[i].size, page_size);
- page_idx += r->count;
+ if (cfg->is_direct) {
+ buf_size = cfg->buf_pg_count << cfg->buf_pg_shift;
+ page_cnt = DIV_ROUND_UP(buf_size, HNS_HW_PAGE_SIZE);
+ /*
+ * When HEM buffer use level-0 addressing, the page size equals
+ * the buffer size, and the the page size = 4K * 2^N.
+ */
+ cfg->buf_pg_shift = HNS_HW_PAGE_SHIFT + order_base_2(page_cnt);
+ if (attr->region_count > 1) {
+ cfg->buf_pg_count = page_cnt;
+ page_shift = HNS_HW_PAGE_SHIFT;
+ } else {
+ cfg->buf_pg_count = 1;
+ page_shift = cfg->buf_pg_shift;
+ if (buf_size != 1 << page_shift) {
+ ibdev_err(&hr_dev->ib_dev,
+ "failed to check direct size %zu shift %d.\n",
+ buf_size, page_shift);
+ return -EINVAL;
+ }
+ }
+ } else {
+ page_shift = cfg->buf_pg_shift;
}
- return i;
+ /* convert buffer size to page index and page count */
+ for (page_cnt = 0, region_cnt = 0; page_cnt < cfg->buf_pg_count &&
+ region_cnt < attr->region_count &&
+ region_cnt < ARRAY_SIZE(cfg->region); region_cnt++) {
+ r = &cfg->region[region_cnt];
+ r->offset = page_cnt;
+ buf_size = hr_hw_page_align(attr->region[region_cnt].size);
+ r->count = DIV_ROUND_UP(buf_size, 1 << page_shift);
+ page_cnt += r->count;
+ r->hopnum = to_hr_hem_hopnum(attr->region[region_cnt].hopnum,
+ r->count);
+ }
+
+ if (region_cnt < 1) {
+ ibdev_err(&hr_dev->ib_dev,
+ "failed to check mtr region count, pages = %d.\n",
+ cfg->buf_pg_count);
+ return -ENOBUFS;
+ }
+
+ cfg->region_count = region_cnt;
+ *buf_page_shift = page_shift;
+
+ return page_cnt;
}
/**
* hns_roce_mtr_create - Create hns memory translate region.
*
* @mtr: memory translate region
- * @init_attr: init attribute for creating mtr
- * @page_shift: page shift for multi-hop base address table
+ * @buf_attr: buffer attribute for creating mtr
+ * @ba_page_shift: page shift for multi-hop base address table
* @udata: user space context, if it's NULL, means kernel space
* @user_addr: userspace virtual address to start at
- * @buf_alloced: mtr has private buffer, true means need to alloc
*/
int hns_roce_mtr_create(struct hns_roce_dev *hr_dev, struct hns_roce_mtr *mtr,
struct hns_roce_buf_attr *buf_attr,
- unsigned int page_shift, struct ib_udata *udata,
+ unsigned int ba_page_shift, struct ib_udata *udata,
unsigned long user_addr)
{
+ struct hns_roce_hem_cfg *cfg = &mtr->hem_cfg;
struct ib_device *ibdev = &hr_dev->ib_dev;
+ unsigned int buf_page_shift = 0;
dma_addr_t *pages = NULL;
- int region_cnt = 0;
int all_pg_cnt;
int get_pg_cnt;
- bool has_mtt;
- int err = 0;
+ int ret = 0;
- has_mtt = mtr_has_mtt(buf_attr);
+ /* if disable mtt, all pages must in a continuous address range */
+ cfg->is_direct = !mtr_has_mtt(buf_attr);
+
/* if buffer only need mtt, just init the hem cfg */
if (buf_attr->mtt_only) {
- mtr->hem_cfg.buf_pg_shift = buf_attr->page_shift;
- mtr->hem_cfg.buf_pg_count = mtr_bufs_size(buf_attr) >>
- buf_attr->page_shift;
+ cfg->buf_pg_shift = buf_attr->page_shift;
+ cfg->buf_pg_count = mtr_bufs_size(buf_attr) >>
+ buf_attr->page_shift;
mtr->umem = NULL;
mtr->kmem = NULL;
} else {
- err = mtr_alloc_bufs(hr_dev, mtr, buf_attr, !has_mtt, udata,
- user_addr);
- if (err) {
- ibdev_err(ibdev, "Failed to alloc mtr bufs, err %d\n",
- err);
- return err;
+ ret = mtr_alloc_bufs(hr_dev, mtr, buf_attr, cfg->is_direct,
+ udata, user_addr);
+ if (ret) {
+ ibdev_err(ibdev,
+ "failed to alloc mtr bufs, ret = %d.\n", ret);
+ return ret;
}
}
- /* alloc mtt memory */
- all_pg_cnt = mtr->hem_cfg.buf_pg_count;
- hns_roce_hem_list_init(&mtr->hem_list);
- mtr->hem_cfg.is_direct = !has_mtt;
- mtr->hem_cfg.ba_pg_shift = page_shift;
- mtr->hem_cfg.region_count = 0;
- region_cnt = mtr_init_region(buf_attr, all_pg_cnt,
- mtr->hem_cfg.region,
- ARRAY_SIZE(mtr->hem_cfg.region),
- mtr->hem_cfg.buf_pg_shift);
- if (region_cnt < 1) {
- err = -ENOBUFS;
- ibdev_err(ibdev, "failed to init mtr region %d\n", region_cnt);
+ all_pg_cnt = mtr_init_buf_cfg(hr_dev, buf_attr, cfg, &buf_page_shift);
+ if (all_pg_cnt < 1) {
+ ret = -ENOBUFS;
+ ibdev_err(ibdev, "failed to init mtr buf cfg.\n");
goto err_alloc_bufs;
}
- mtr->hem_cfg.region_count = region_cnt;
-
- if (has_mtt) {
- err = hns_roce_hem_list_request(hr_dev, &mtr->hem_list,
- mtr->hem_cfg.region, region_cnt,
- page_shift);
- if (err) {
- ibdev_err(ibdev, "Failed to request mtr hem, err %d\n",
- err);
+ hns_roce_hem_list_init(&mtr->hem_list);
+ if (!cfg->is_direct) {
+ ret = hns_roce_hem_list_request(hr_dev, &mtr->hem_list,
+ cfg->region, cfg->region_count,
+ ba_page_shift);
+ if (ret) {
+ ibdev_err(ibdev, "failed to request mtr hem, ret = %d.\n",
+ ret);
goto err_alloc_bufs;
}
- mtr->hem_cfg.root_ba = mtr->hem_list.root_ba;
+ cfg->root_ba = mtr->hem_list.root_ba;
+ cfg->ba_pg_shift = ba_page_shift;
+ } else {
+ cfg->ba_pg_shift = cfg->buf_pg_shift;
}
/* no buffer to map */
@@ -1046,31 +1095,26 @@
/* alloc a tmp array to store buffer's dma address */
pages = kvcalloc(all_pg_cnt, sizeof(dma_addr_t), GFP_KERNEL);
if (!pages) {
- err = -ENOMEM;
- ibdev_err(ibdev, "Failed to alloc mtr page list %d\n",
+ ret = -ENOMEM;
+ ibdev_err(ibdev, "failed to alloc mtr page list %d.\n",
all_pg_cnt);
goto err_alloc_hem_list;
}
get_pg_cnt = mtr_get_pages(hr_dev, mtr, pages, all_pg_cnt,
- mtr->hem_cfg.buf_pg_shift);
+ buf_page_shift);
if (get_pg_cnt != all_pg_cnt) {
- ibdev_err(ibdev, "Failed to get mtr page %d != %d\n",
+ ibdev_err(ibdev, "failed to get mtr page %d != %d.\n",
get_pg_cnt, all_pg_cnt);
- err = -ENOBUFS;
+ ret = -ENOBUFS;
goto err_alloc_page_list;
}
- if (!has_mtt) {
- mtr->hem_cfg.root_ba = pages[0];
- } else {
- /* write buffer's dma address to BA table */
- err = hns_roce_mtr_map(hr_dev, mtr, pages, all_pg_cnt);
- if (err) {
- ibdev_err(ibdev, "Failed to map mtr pages, err %d\n",
- err);
- goto err_alloc_page_list;
- }
+ /* write buffer's dma address to BA table */
+ ret = hns_roce_mtr_map(hr_dev, mtr, pages, all_pg_cnt);
+ if (ret) {
+ ibdev_err(ibdev, "failed to map mtr pages, ret = %d.\n", ret);
+ goto err_alloc_page_list;
}
/* drop tmp array */
@@ -1082,7 +1126,7 @@
hns_roce_hem_list_release(hr_dev, &mtr->hem_list);
err_alloc_bufs:
mtr_free_bufs(hr_dev, mtr);
- return err;
+ return ret;
}
void hns_roce_mtr_destroy(struct hns_roce_dev *hr_dev, struct hns_roce_mtr *mtr)
diff --git a/drivers/infiniband/hw/hns/hns_roce_qp.c b/drivers/infiniband/hw/hns/hns_roce_qp.c
index a0a47bd..e94ca13 100644
--- a/drivers/infiniband/hw/hns/hns_roce_qp.c
+++ b/drivers/infiniband/hw/hns/hns_roce_qp.c
@@ -411,7 +411,6 @@
struct hns_roce_qp *hr_qp,
struct ib_qp_cap *cap)
{
- struct ib_device *ibdev = &hr_dev->ib_dev;
u32 cnt;
cnt = max(1U, cap->max_send_sge);
@@ -431,15 +430,6 @@
} else if (hr_qp->sq.max_gs > HNS_ROCE_SGE_IN_WQE) {
cnt = roundup_pow_of_two(sq_wqe_cnt *
(hr_qp->sq.max_gs - HNS_ROCE_SGE_IN_WQE));
-
- if (hr_dev->pci_dev->revision == PCI_REVISION_ID_HIP08_A) {
- if (cnt > hr_dev->caps.max_extend_sg) {
- ibdev_err(ibdev,
- "failed to check exSGE num, exSGE num = %d.\n",
- cnt);
- return -EINVAL;
- }
- }
} else {
cnt = 0;
}
diff --git a/drivers/infiniband/hw/hns/hns_roce_restrack.c b/drivers/infiniband/hw/hns/hns_roce_restrack.c
index 0687173..259444c 100644
--- a/drivers/infiniband/hw/hns/hns_roce_restrack.c
+++ b/drivers/infiniband/hw/hns/hns_roce_restrack.c
@@ -76,10 +76,9 @@
return -EMSGSIZE;
}
-static int hns_roce_fill_res_cq_entry(struct sk_buff *msg,
- struct rdma_restrack_entry *res)
+int hns_roce_fill_res_cq_entry(struct sk_buff *msg,
+ struct ib_cq *ib_cq)
{
- struct ib_cq *ib_cq = container_of(res, struct ib_cq, res);
struct hns_roce_dev *hr_dev = to_hr_dev(ib_cq->device);
struct hns_roce_cq *hr_cq = to_hr_cq(ib_cq);
struct hns_roce_v2_cq_context *context;
@@ -119,12 +118,3 @@
kfree(context);
return ret;
}
-
-int hns_roce_fill_res_entry(struct sk_buff *msg,
- struct rdma_restrack_entry *res)
-{
- if (res->type == RDMA_RESTRACK_CQ)
- return hns_roce_fill_res_cq_entry(msg, res);
-
- return 0;
-}
diff --git a/drivers/infiniband/hw/i40iw/i40iw_verbs.c b/drivers/infiniband/hw/i40iw/i40iw_verbs.c
index 19af29a..6957e4f 100644
--- a/drivers/infiniband/hw/i40iw/i40iw_verbs.c
+++ b/drivers/infiniband/hw/i40iw/i40iw_verbs.c
@@ -101,7 +101,6 @@
props->port_cap_flags = IB_PORT_CM_SUP | IB_PORT_REINIT_SUP |
IB_PORT_VENDOR_CLASS_SUP | IB_PORT_BOOT_MGMT_SUP;
props->gid_tbl_len = 1;
- props->pkey_tbl_len = 1;
props->active_width = IB_WIDTH_4X;
props->active_speed = 1;
props->max_msg_sz = I40IW_MAX_OUTBOUND_MESSAGE_SIZE;
@@ -1543,10 +1542,9 @@
* @pd: ibpd pointer
* @mr_type: memory for stag registrion
* @max_num_sg: man number of pages
- * @udata: user data or NULL for kernel objects
*/
static struct ib_mr *i40iw_alloc_mr(struct ib_pd *pd, enum ib_mr_type mr_type,
- u32 max_num_sg, struct ib_udata *udata)
+ u32 max_num_sg)
{
struct i40iw_pd *iwpd = to_iwpd(pd);
struct i40iw_device *iwdev = to_iwdev(pd->device);
@@ -2460,7 +2458,6 @@
if (err)
return err;
- immutable->pkey_tbl_len = attr.pkey_tbl_len;
immutable->gid_tbl_len = attr.gid_tbl_len;
return 0;
@@ -2616,22 +2613,6 @@
return 0;
}
-/**
- * i40iw_query_pkey - Query partition key
- * @ibdev: device pointer from stack
- * @port: port number
- * @index: index of pkey
- * @pkey: pointer to store the pkey
- */
-static int i40iw_query_pkey(struct ib_device *ibdev,
- u8 port,
- u16 index,
- u16 *pkey)
-{
- *pkey = 0;
- return 0;
-}
-
static const struct ib_device_ops i40iw_dev_ops = {
.owner = THIS_MODULE,
.driver_id = RDMA_DRIVER_I40IW,
@@ -2671,7 +2652,6 @@
.post_send = i40iw_post_send,
.query_device = i40iw_query_device,
.query_gid = i40iw_query_gid,
- .query_pkey = i40iw_query_pkey,
.query_port = i40iw_query_port,
.query_qp = i40iw_query_qp,
.reg_user_mr = i40iw_reg_user_mr,
diff --git a/drivers/infiniband/hw/mlx4/main.c b/drivers/infiniband/hw/mlx4/main.c
index 816d288..5e7910a 100644
--- a/drivers/infiniband/hw/mlx4/main.c
+++ b/drivers/infiniband/hw/mlx4/main.c
@@ -1219,56 +1219,47 @@
mlx4_pd_free(to_mdev(pd->device)->dev, to_mpd(pd)->pdn);
}
-static struct ib_xrcd *mlx4_ib_alloc_xrcd(struct ib_device *ibdev,
- struct ib_udata *udata)
+static int mlx4_ib_alloc_xrcd(struct ib_xrcd *ibxrcd, struct ib_udata *udata)
{
- struct mlx4_ib_xrcd *xrcd;
+ struct mlx4_ib_dev *dev = to_mdev(ibxrcd->device);
+ struct mlx4_ib_xrcd *xrcd = to_mxrcd(ibxrcd);
struct ib_cq_init_attr cq_attr = {};
int err;
- if (!(to_mdev(ibdev)->dev->caps.flags & MLX4_DEV_CAP_FLAG_XRC))
- return ERR_PTR(-ENOSYS);
+ if (!(dev->dev->caps.flags & MLX4_DEV_CAP_FLAG_XRC))
+ return -EOPNOTSUPP;
- xrcd = kmalloc(sizeof *xrcd, GFP_KERNEL);
- if (!xrcd)
- return ERR_PTR(-ENOMEM);
-
- err = mlx4_xrcd_alloc(to_mdev(ibdev)->dev, &xrcd->xrcdn);
+ err = mlx4_xrcd_alloc(dev->dev, &xrcd->xrcdn);
if (err)
- goto err1;
+ return err;
- xrcd->pd = ib_alloc_pd(ibdev, 0);
+ xrcd->pd = ib_alloc_pd(ibxrcd->device, 0);
if (IS_ERR(xrcd->pd)) {
err = PTR_ERR(xrcd->pd);
goto err2;
}
cq_attr.cqe = 1;
- xrcd->cq = ib_create_cq(ibdev, NULL, NULL, xrcd, &cq_attr);
+ xrcd->cq = ib_create_cq(ibxrcd->device, NULL, NULL, xrcd, &cq_attr);
if (IS_ERR(xrcd->cq)) {
err = PTR_ERR(xrcd->cq);
goto err3;
}
- return &xrcd->ibxrcd;
+ return 0;
err3:
ib_dealloc_pd(xrcd->pd);
err2:
- mlx4_xrcd_free(to_mdev(ibdev)->dev, xrcd->xrcdn);
-err1:
- kfree(xrcd);
- return ERR_PTR(err);
+ mlx4_xrcd_free(dev->dev, xrcd->xrcdn);
+ return err;
}
-static int mlx4_ib_dealloc_xrcd(struct ib_xrcd *xrcd, struct ib_udata *udata)
+static void mlx4_ib_dealloc_xrcd(struct ib_xrcd *xrcd, struct ib_udata *udata)
{
ib_destroy_cq(to_mxrcd(xrcd)->cq);
ib_dealloc_pd(to_mxrcd(xrcd)->pd);
mlx4_xrcd_free(to_mdev(xrcd->device)->dev, to_mxrcd(xrcd)->xrcdn);
- kfree(xrcd);
-
- return 0;
}
static int add_gid_entry(struct ib_qp *ibqp, union ib_gid *gid)
@@ -2607,6 +2598,8 @@
static const struct ib_device_ops mlx4_ib_dev_xrc_ops = {
.alloc_xrcd = mlx4_ib_alloc_xrcd,
.dealloc_xrcd = mlx4_ib_dealloc_xrcd,
+
+ INIT_RDMA_OBJ_SIZE(ib_xrcd, mlx4_ib_xrcd, ibxrcd),
};
static const struct ib_device_ops mlx4_ib_dev_fs_ops = {
diff --git a/drivers/infiniband/hw/mlx4/mlx4_ib.h b/drivers/infiniband/hw/mlx4/mlx4_ib.h
index 6f4ea10..38e87a7 100644
--- a/drivers/infiniband/hw/mlx4/mlx4_ib.h
+++ b/drivers/infiniband/hw/mlx4/mlx4_ib.h
@@ -729,7 +729,7 @@
struct ib_udata *udata);
int mlx4_ib_dealloc_mw(struct ib_mw *mw);
struct ib_mr *mlx4_ib_alloc_mr(struct ib_pd *pd, enum ib_mr_type mr_type,
- u32 max_num_sg, struct ib_udata *udata);
+ u32 max_num_sg);
int mlx4_ib_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg, int sg_nents,
unsigned int *sg_offset);
int mlx4_ib_modify_cq(struct ib_cq *cq, u16 cq_count, u16 cq_period);
diff --git a/drivers/infiniband/hw/mlx4/mr.c b/drivers/infiniband/hw/mlx4/mr.c
index 7e0b205..1d5ef0d 100644
--- a/drivers/infiniband/hw/mlx4/mr.c
+++ b/drivers/infiniband/hw/mlx4/mr.c
@@ -439,7 +439,6 @@
mr->ibmr.rkey = mr->ibmr.lkey = mr->mmr.key;
mr->ibmr.length = length;
- mr->ibmr.iova = virt_addr;
mr->ibmr.page_size = 1U << shift;
return &mr->ibmr;
@@ -655,7 +654,7 @@
}
struct ib_mr *mlx4_ib_alloc_mr(struct ib_pd *pd, enum ib_mr_type mr_type,
- u32 max_num_sg, struct ib_udata *udata)
+ u32 max_num_sg)
{
struct mlx4_ib_dev *dev = to_mdev(pd->device);
struct mlx4_ib_mr *mr;
diff --git a/drivers/infiniband/hw/mlx5/Makefile b/drivers/infiniband/hw/mlx5/Makefile
index 8cca61c..b4c009b 100644
--- a/drivers/infiniband/hw/mlx5/Makefile
+++ b/drivers/infiniband/hw/mlx5/Makefile
@@ -4,6 +4,7 @@
mlx5_ib-y := ah.o \
cmd.o \
cong.o \
+ counters.o \
cq.o \
doorbell.o \
gsi.o \
@@ -22,5 +23,6 @@
mlx5_ib-$(CONFIG_INFINIBAND_ON_DEMAND_PAGING) += odp.o
mlx5_ib-$(CONFIG_MLX5_ESWITCH) += ib_rep.o
mlx5_ib-$(CONFIG_INFINIBAND_USER_ACCESS) += devx.o \
- flow.o \
- qos.o
+ fs.o \
+ qos.o \
+ std_types.o
diff --git a/drivers/infiniband/hw/mlx5/cmd.c b/drivers/infiniband/hw/mlx5/cmd.c
index cc24c71..ebb2f10 100644
--- a/drivers/infiniband/hw/mlx5/cmd.c
+++ b/drivers/infiniband/hw/mlx5/cmd.c
@@ -148,18 +148,6 @@
spin_unlock(&dm->lock);
}
-int mlx5_cmd_query_ext_ppcnt_counters(struct mlx5_core_dev *dev, void *out)
-{
- u32 in[MLX5_ST_SZ_DW(ppcnt_reg)] = {};
- int sz = MLX5_ST_SZ_BYTES(ppcnt_reg);
-
- MLX5_SET(ppcnt_reg, in, local_port, 1);
-
- MLX5_SET(ppcnt_reg, in, grp, MLX5_ETHERNET_EXTENDED_COUNTERS_GROUP);
- return mlx5_core_access_reg(dev, in, sz, out, sz, MLX5_REG_PPCNT,
- 0, 0);
-}
-
void mlx5_cmd_destroy_tir(struct mlx5_core_dev *dev, u32 tirn, u16 uid)
{
u32 in[MLX5_ST_SZ_DW(destroy_tir_in)] = {};
diff --git a/drivers/infiniband/hw/mlx5/cmd.h b/drivers/infiniband/hw/mlx5/cmd.h
index f4d8558..1d192a8 100644
--- a/drivers/infiniband/hw/mlx5/cmd.h
+++ b/drivers/infiniband/hw/mlx5/cmd.h
@@ -41,7 +41,6 @@
int mlx5_cmd_null_mkey(struct mlx5_core_dev *dev, u32 *null_mkey);
int mlx5_cmd_query_cong_params(struct mlx5_core_dev *dev, int cong_point,
void *out);
-int mlx5_cmd_query_ext_ppcnt_counters(struct mlx5_core_dev *dev, void *out);
int mlx5_cmd_alloc_memic(struct mlx5_dm *dm, phys_addr_t *addr,
u64 length, u32 alignment);
void mlx5_cmd_dealloc_memic(struct mlx5_dm *dm, phys_addr_t addr, u64 length);
diff --git a/drivers/infiniband/hw/mlx5/counters.c b/drivers/infiniband/hw/mlx5/counters.c
new file mode 100644
index 0000000..145f3cb
--- /dev/null
+++ b/drivers/infiniband/hw/mlx5/counters.c
@@ -0,0 +1,709 @@
+// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
+/*
+ * Copyright (c) 2013-2020, Mellanox Technologies inc. All rights reserved.
+ */
+
+#include "mlx5_ib.h"
+#include <linux/mlx5/eswitch.h>
+#include "counters.h"
+#include "ib_rep.h"
+#include "qp.h"
+
+struct mlx5_ib_counter {
+ const char *name;
+ size_t offset;
+};
+
+#define INIT_Q_COUNTER(_name) \
+ { .name = #_name, .offset = MLX5_BYTE_OFF(query_q_counter_out, _name)}
+
+static const struct mlx5_ib_counter basic_q_cnts[] = {
+ INIT_Q_COUNTER(rx_write_requests),
+ INIT_Q_COUNTER(rx_read_requests),
+ INIT_Q_COUNTER(rx_atomic_requests),
+ INIT_Q_COUNTER(out_of_buffer),
+};
+
+static const struct mlx5_ib_counter out_of_seq_q_cnts[] = {
+ INIT_Q_COUNTER(out_of_sequence),
+};
+
+static const struct mlx5_ib_counter retrans_q_cnts[] = {
+ INIT_Q_COUNTER(duplicate_request),
+ INIT_Q_COUNTER(rnr_nak_retry_err),
+ INIT_Q_COUNTER(packet_seq_err),
+ INIT_Q_COUNTER(implied_nak_seq_err),
+ INIT_Q_COUNTER(local_ack_timeout_err),
+};
+
+#define INIT_CONG_COUNTER(_name) \
+ { .name = #_name, .offset = \
+ MLX5_BYTE_OFF(query_cong_statistics_out, _name ## _high)}
+
+static const struct mlx5_ib_counter cong_cnts[] = {
+ INIT_CONG_COUNTER(rp_cnp_ignored),
+ INIT_CONG_COUNTER(rp_cnp_handled),
+ INIT_CONG_COUNTER(np_ecn_marked_roce_packets),
+ INIT_CONG_COUNTER(np_cnp_sent),
+};
+
+static const struct mlx5_ib_counter extended_err_cnts[] = {
+ INIT_Q_COUNTER(resp_local_length_error),
+ INIT_Q_COUNTER(resp_cqe_error),
+ INIT_Q_COUNTER(req_cqe_error),
+ INIT_Q_COUNTER(req_remote_invalid_request),
+ INIT_Q_COUNTER(req_remote_access_errors),
+ INIT_Q_COUNTER(resp_remote_access_errors),
+ INIT_Q_COUNTER(resp_cqe_flush_error),
+ INIT_Q_COUNTER(req_cqe_flush_error),
+};
+
+static const struct mlx5_ib_counter roce_accl_cnts[] = {
+ INIT_Q_COUNTER(roce_adp_retrans),
+ INIT_Q_COUNTER(roce_adp_retrans_to),
+ INIT_Q_COUNTER(roce_slow_restart),
+ INIT_Q_COUNTER(roce_slow_restart_cnps),
+ INIT_Q_COUNTER(roce_slow_restart_trans),
+};
+
+#define INIT_EXT_PPCNT_COUNTER(_name) \
+ { .name = #_name, .offset = \
+ MLX5_BYTE_OFF(ppcnt_reg, \
+ counter_set.eth_extended_cntrs_grp_data_layout._name##_high)}
+
+static const struct mlx5_ib_counter ext_ppcnt_cnts[] = {
+ INIT_EXT_PPCNT_COUNTER(rx_icrc_encapsulated),
+};
+
+static int mlx5_ib_read_counters(struct ib_counters *counters,
+ struct ib_counters_read_attr *read_attr,
+ struct uverbs_attr_bundle *attrs)
+{
+ struct mlx5_ib_mcounters *mcounters = to_mcounters(counters);
+ struct mlx5_read_counters_attr mread_attr = {};
+ struct mlx5_ib_flow_counters_desc *desc;
+ int ret, i;
+
+ mutex_lock(&mcounters->mcntrs_mutex);
+ if (mcounters->cntrs_max_index > read_attr->ncounters) {
+ ret = -EINVAL;
+ goto err_bound;
+ }
+
+ mread_attr.out = kcalloc(mcounters->counters_num, sizeof(u64),
+ GFP_KERNEL);
+ if (!mread_attr.out) {
+ ret = -ENOMEM;
+ goto err_bound;
+ }
+
+ mread_attr.hw_cntrs_hndl = mcounters->hw_cntrs_hndl;
+ mread_attr.flags = read_attr->flags;
+ ret = mcounters->read_counters(counters->device, &mread_attr);
+ if (ret)
+ goto err_read;
+
+ /* do the pass over the counters data array to assign according to the
+ * descriptions and indexing pairs
+ */
+ desc = mcounters->counters_data;
+ for (i = 0; i < mcounters->ncounters; i++)
+ read_attr->counters_buff[desc[i].index] += mread_attr.out[desc[i].description];
+
+err_read:
+ kfree(mread_attr.out);
+err_bound:
+ mutex_unlock(&mcounters->mcntrs_mutex);
+ return ret;
+}
+
+static void mlx5_ib_destroy_counters(struct ib_counters *counters)
+{
+ struct mlx5_ib_mcounters *mcounters = to_mcounters(counters);
+
+ mlx5_ib_counters_clear_description(counters);
+ if (mcounters->hw_cntrs_hndl)
+ mlx5_fc_destroy(to_mdev(counters->device)->mdev,
+ mcounters->hw_cntrs_hndl);
+}
+
+static int mlx5_ib_create_counters(struct ib_counters *counters,
+ struct uverbs_attr_bundle *attrs)
+{
+ struct mlx5_ib_mcounters *mcounters = to_mcounters(counters);
+
+ mutex_init(&mcounters->mcntrs_mutex);
+ return 0;
+}
+
+
+static bool is_mdev_switchdev_mode(const struct mlx5_core_dev *mdev)
+{
+ return MLX5_ESWITCH_MANAGER(mdev) &&
+ mlx5_ib_eswitch_mode(mdev->priv.eswitch) ==
+ MLX5_ESWITCH_OFFLOADS;
+}
+
+static const struct mlx5_ib_counters *get_counters(struct mlx5_ib_dev *dev,
+ u8 port_num)
+{
+ return is_mdev_switchdev_mode(dev->mdev) ? &dev->port[0].cnts :
+ &dev->port[port_num].cnts;
+}
+
+/**
+ * mlx5_ib_get_counters_id - Returns counters id to use for device+port
+ * @dev: Pointer to mlx5 IB device
+ * @port_num: Zero based port number
+ *
+ * mlx5_ib_get_counters_id() Returns counters set id to use for given
+ * device port combination in switchdev and non switchdev mode of the
+ * parent device.
+ */
+u16 mlx5_ib_get_counters_id(struct mlx5_ib_dev *dev, u8 port_num)
+{
+ const struct mlx5_ib_counters *cnts = get_counters(dev, port_num);
+
+ return cnts->set_id;
+}
+
+static struct rdma_hw_stats *mlx5_ib_alloc_hw_stats(struct ib_device *ibdev,
+ u8 port_num)
+{
+ struct mlx5_ib_dev *dev = to_mdev(ibdev);
+ const struct mlx5_ib_counters *cnts;
+ bool is_switchdev = is_mdev_switchdev_mode(dev->mdev);
+
+ if ((is_switchdev && port_num) || (!is_switchdev && !port_num))
+ return NULL;
+
+ cnts = get_counters(dev, port_num - 1);
+
+ return rdma_alloc_hw_stats_struct(cnts->names,
+ cnts->num_q_counters +
+ cnts->num_cong_counters +
+ cnts->num_ext_ppcnt_counters,
+ RDMA_HW_STATS_DEFAULT_LIFESPAN);
+}
+
+static int mlx5_ib_query_q_counters(struct mlx5_core_dev *mdev,
+ const struct mlx5_ib_counters *cnts,
+ struct rdma_hw_stats *stats,
+ u16 set_id)
+{
+ u32 out[MLX5_ST_SZ_DW(query_q_counter_out)] = {};
+ u32 in[MLX5_ST_SZ_DW(query_q_counter_in)] = {};
+ __be32 val;
+ int ret, i;
+
+ MLX5_SET(query_q_counter_in, in, opcode, MLX5_CMD_OP_QUERY_Q_COUNTER);
+ MLX5_SET(query_q_counter_in, in, counter_set_id, set_id);
+ ret = mlx5_cmd_exec_inout(mdev, query_q_counter, in, out);
+ if (ret)
+ return ret;
+
+ for (i = 0; i < cnts->num_q_counters; i++) {
+ val = *(__be32 *)((void *)out + cnts->offsets[i]);
+ stats->value[i] = (u64)be32_to_cpu(val);
+ }
+
+ return 0;
+}
+
+static int mlx5_ib_query_ext_ppcnt_counters(struct mlx5_ib_dev *dev,
+ const struct mlx5_ib_counters *cnts,
+ struct rdma_hw_stats *stats)
+{
+ int offset = cnts->num_q_counters + cnts->num_cong_counters;
+ u32 in[MLX5_ST_SZ_DW(ppcnt_reg)] = {};
+ int sz = MLX5_ST_SZ_BYTES(ppcnt_reg);
+ int ret, i;
+ void *out;
+
+ out = kvzalloc(sz, GFP_KERNEL);
+ if (!out)
+ return -ENOMEM;
+
+ MLX5_SET(ppcnt_reg, in, local_port, 1);
+ MLX5_SET(ppcnt_reg, in, grp, MLX5_ETHERNET_EXTENDED_COUNTERS_GROUP);
+ ret = mlx5_core_access_reg(dev->mdev, in, sz, out, sz, MLX5_REG_PPCNT,
+ 0, 0);
+ if (ret)
+ goto free;
+
+ for (i = 0; i < cnts->num_ext_ppcnt_counters; i++)
+ stats->value[i + offset] =
+ be64_to_cpup((__be64 *)(out +
+ cnts->offsets[i + offset]));
+free:
+ kvfree(out);
+ return ret;
+}
+
+static int mlx5_ib_get_hw_stats(struct ib_device *ibdev,
+ struct rdma_hw_stats *stats,
+ u8 port_num, int index)
+{
+ struct mlx5_ib_dev *dev = to_mdev(ibdev);
+ const struct mlx5_ib_counters *cnts = get_counters(dev, port_num - 1);
+ struct mlx5_core_dev *mdev;
+ int ret, num_counters;
+ u8 mdev_port_num;
+
+ if (!stats)
+ return -EINVAL;
+
+ num_counters = cnts->num_q_counters +
+ cnts->num_cong_counters +
+ cnts->num_ext_ppcnt_counters;
+
+ /* q_counters are per IB device, query the master mdev */
+ ret = mlx5_ib_query_q_counters(dev->mdev, cnts, stats, cnts->set_id);
+ if (ret)
+ return ret;
+
+ if (MLX5_CAP_PCAM_FEATURE(dev->mdev, rx_icrc_encapsulated_counter)) {
+ ret = mlx5_ib_query_ext_ppcnt_counters(dev, cnts, stats);
+ if (ret)
+ return ret;
+ }
+
+ if (MLX5_CAP_GEN(dev->mdev, cc_query_allowed)) {
+ mdev = mlx5_ib_get_native_port_mdev(dev, port_num,
+ &mdev_port_num);
+ if (!mdev) {
+ /* If port is not affiliated yet, its in down state
+ * which doesn't have any counters yet, so it would be
+ * zero. So no need to read from the HCA.
+ */
+ goto done;
+ }
+ ret = mlx5_lag_query_cong_counters(dev->mdev,
+ stats->value +
+ cnts->num_q_counters,
+ cnts->num_cong_counters,
+ cnts->offsets +
+ cnts->num_q_counters);
+
+ mlx5_ib_put_native_port_mdev(dev, port_num);
+ if (ret)
+ return ret;
+ }
+
+done:
+ return num_counters;
+}
+
+static struct rdma_hw_stats *
+mlx5_ib_counter_alloc_stats(struct rdma_counter *counter)
+{
+ struct mlx5_ib_dev *dev = to_mdev(counter->device);
+ const struct mlx5_ib_counters *cnts =
+ get_counters(dev, counter->port - 1);
+
+ return rdma_alloc_hw_stats_struct(cnts->names,
+ cnts->num_q_counters +
+ cnts->num_cong_counters +
+ cnts->num_ext_ppcnt_counters,
+ RDMA_HW_STATS_DEFAULT_LIFESPAN);
+}
+
+static int mlx5_ib_counter_update_stats(struct rdma_counter *counter)
+{
+ struct mlx5_ib_dev *dev = to_mdev(counter->device);
+ const struct mlx5_ib_counters *cnts =
+ get_counters(dev, counter->port - 1);
+
+ return mlx5_ib_query_q_counters(dev->mdev, cnts,
+ counter->stats, counter->id);
+}
+
+static int mlx5_ib_counter_dealloc(struct rdma_counter *counter)
+{
+ struct mlx5_ib_dev *dev = to_mdev(counter->device);
+ u32 in[MLX5_ST_SZ_DW(dealloc_q_counter_in)] = {};
+
+ if (!counter->id)
+ return 0;
+
+ MLX5_SET(dealloc_q_counter_in, in, opcode,
+ MLX5_CMD_OP_DEALLOC_Q_COUNTER);
+ MLX5_SET(dealloc_q_counter_in, in, counter_set_id, counter->id);
+ return mlx5_cmd_exec_in(dev->mdev, dealloc_q_counter, in);
+}
+
+static int mlx5_ib_counter_bind_qp(struct rdma_counter *counter,
+ struct ib_qp *qp)
+{
+ struct mlx5_ib_dev *dev = to_mdev(qp->device);
+ int err;
+
+ if (!counter->id) {
+ u32 out[MLX5_ST_SZ_DW(alloc_q_counter_out)] = {};
+ u32 in[MLX5_ST_SZ_DW(alloc_q_counter_in)] = {};
+
+ MLX5_SET(alloc_q_counter_in, in, opcode,
+ MLX5_CMD_OP_ALLOC_Q_COUNTER);
+ MLX5_SET(alloc_q_counter_in, in, uid, MLX5_SHARED_RESOURCE_UID);
+ err = mlx5_cmd_exec_inout(dev->mdev, alloc_q_counter, in, out);
+ if (err)
+ return err;
+ counter->id =
+ MLX5_GET(alloc_q_counter_out, out, counter_set_id);
+ }
+
+ err = mlx5_ib_qp_set_counter(qp, counter);
+ if (err)
+ goto fail_set_counter;
+
+ return 0;
+
+fail_set_counter:
+ mlx5_ib_counter_dealloc(counter);
+ counter->id = 0;
+
+ return err;
+}
+
+static int mlx5_ib_counter_unbind_qp(struct ib_qp *qp)
+{
+ return mlx5_ib_qp_set_counter(qp, NULL);
+}
+
+
+static void mlx5_ib_fill_counters(struct mlx5_ib_dev *dev,
+ const char **names,
+ size_t *offsets)
+{
+ int i;
+ int j = 0;
+
+ for (i = 0; i < ARRAY_SIZE(basic_q_cnts); i++, j++) {
+ names[j] = basic_q_cnts[i].name;
+ offsets[j] = basic_q_cnts[i].offset;
+ }
+
+ if (MLX5_CAP_GEN(dev->mdev, out_of_seq_cnt)) {
+ for (i = 0; i < ARRAY_SIZE(out_of_seq_q_cnts); i++, j++) {
+ names[j] = out_of_seq_q_cnts[i].name;
+ offsets[j] = out_of_seq_q_cnts[i].offset;
+ }
+ }
+
+ if (MLX5_CAP_GEN(dev->mdev, retransmission_q_counters)) {
+ for (i = 0; i < ARRAY_SIZE(retrans_q_cnts); i++, j++) {
+ names[j] = retrans_q_cnts[i].name;
+ offsets[j] = retrans_q_cnts[i].offset;
+ }
+ }
+
+ if (MLX5_CAP_GEN(dev->mdev, enhanced_error_q_counters)) {
+ for (i = 0; i < ARRAY_SIZE(extended_err_cnts); i++, j++) {
+ names[j] = extended_err_cnts[i].name;
+ offsets[j] = extended_err_cnts[i].offset;
+ }
+ }
+
+ if (MLX5_CAP_GEN(dev->mdev, roce_accl)) {
+ for (i = 0; i < ARRAY_SIZE(roce_accl_cnts); i++, j++) {
+ names[j] = roce_accl_cnts[i].name;
+ offsets[j] = roce_accl_cnts[i].offset;
+ }
+ }
+
+ if (MLX5_CAP_GEN(dev->mdev, cc_query_allowed)) {
+ for (i = 0; i < ARRAY_SIZE(cong_cnts); i++, j++) {
+ names[j] = cong_cnts[i].name;
+ offsets[j] = cong_cnts[i].offset;
+ }
+ }
+
+ if (MLX5_CAP_PCAM_FEATURE(dev->mdev, rx_icrc_encapsulated_counter)) {
+ for (i = 0; i < ARRAY_SIZE(ext_ppcnt_cnts); i++, j++) {
+ names[j] = ext_ppcnt_cnts[i].name;
+ offsets[j] = ext_ppcnt_cnts[i].offset;
+ }
+ }
+}
+
+
+static int __mlx5_ib_alloc_counters(struct mlx5_ib_dev *dev,
+ struct mlx5_ib_counters *cnts)
+{
+ u32 num_counters;
+
+ num_counters = ARRAY_SIZE(basic_q_cnts);
+
+ if (MLX5_CAP_GEN(dev->mdev, out_of_seq_cnt))
+ num_counters += ARRAY_SIZE(out_of_seq_q_cnts);
+
+ if (MLX5_CAP_GEN(dev->mdev, retransmission_q_counters))
+ num_counters += ARRAY_SIZE(retrans_q_cnts);
+
+ if (MLX5_CAP_GEN(dev->mdev, enhanced_error_q_counters))
+ num_counters += ARRAY_SIZE(extended_err_cnts);
+
+ if (MLX5_CAP_GEN(dev->mdev, roce_accl))
+ num_counters += ARRAY_SIZE(roce_accl_cnts);
+
+ cnts->num_q_counters = num_counters;
+
+ if (MLX5_CAP_GEN(dev->mdev, cc_query_allowed)) {
+ cnts->num_cong_counters = ARRAY_SIZE(cong_cnts);
+ num_counters += ARRAY_SIZE(cong_cnts);
+ }
+ if (MLX5_CAP_PCAM_FEATURE(dev->mdev, rx_icrc_encapsulated_counter)) {
+ cnts->num_ext_ppcnt_counters = ARRAY_SIZE(ext_ppcnt_cnts);
+ num_counters += ARRAY_SIZE(ext_ppcnt_cnts);
+ }
+ cnts->names = kcalloc(num_counters, sizeof(cnts->names), GFP_KERNEL);
+ if (!cnts->names)
+ return -ENOMEM;
+
+ cnts->offsets = kcalloc(num_counters,
+ sizeof(cnts->offsets), GFP_KERNEL);
+ if (!cnts->offsets)
+ goto err_names;
+
+ return 0;
+
+err_names:
+ kfree(cnts->names);
+ cnts->names = NULL;
+ return -ENOMEM;
+}
+
+static void mlx5_ib_dealloc_counters(struct mlx5_ib_dev *dev)
+{
+ u32 in[MLX5_ST_SZ_DW(dealloc_q_counter_in)] = {};
+ int num_cnt_ports;
+ int i;
+
+ num_cnt_ports = is_mdev_switchdev_mode(dev->mdev) ? 1 : dev->num_ports;
+
+ MLX5_SET(dealloc_q_counter_in, in, opcode,
+ MLX5_CMD_OP_DEALLOC_Q_COUNTER);
+
+ for (i = 0; i < num_cnt_ports; i++) {
+ if (dev->port[i].cnts.set_id) {
+ MLX5_SET(dealloc_q_counter_in, in, counter_set_id,
+ dev->port[i].cnts.set_id);
+ mlx5_cmd_exec_in(dev->mdev, dealloc_q_counter, in);
+ }
+ kfree(dev->port[i].cnts.names);
+ kfree(dev->port[i].cnts.offsets);
+ }
+}
+
+static int mlx5_ib_alloc_counters(struct mlx5_ib_dev *dev)
+{
+ u32 out[MLX5_ST_SZ_DW(alloc_q_counter_out)] = {};
+ u32 in[MLX5_ST_SZ_DW(alloc_q_counter_in)] = {};
+ int num_cnt_ports;
+ int err = 0;
+ int i;
+ bool is_shared;
+
+ MLX5_SET(alloc_q_counter_in, in, opcode, MLX5_CMD_OP_ALLOC_Q_COUNTER);
+ is_shared = MLX5_CAP_GEN(dev->mdev, log_max_uctx) != 0;
+ num_cnt_ports = is_mdev_switchdev_mode(dev->mdev) ? 1 : dev->num_ports;
+
+ for (i = 0; i < num_cnt_ports; i++) {
+ err = __mlx5_ib_alloc_counters(dev, &dev->port[i].cnts);
+ if (err)
+ goto err_alloc;
+
+ mlx5_ib_fill_counters(dev, dev->port[i].cnts.names,
+ dev->port[i].cnts.offsets);
+
+ MLX5_SET(alloc_q_counter_in, in, uid,
+ is_shared ? MLX5_SHARED_RESOURCE_UID : 0);
+
+ err = mlx5_cmd_exec_inout(dev->mdev, alloc_q_counter, in, out);
+ if (err) {
+ mlx5_ib_warn(dev,
+ "couldn't allocate queue counter for port %d, err %d\n",
+ i + 1, err);
+ goto err_alloc;
+ }
+
+ dev->port[i].cnts.set_id =
+ MLX5_GET(alloc_q_counter_out, out, counter_set_id);
+ }
+ return 0;
+
+err_alloc:
+ mlx5_ib_dealloc_counters(dev);
+ return err;
+}
+
+static int read_flow_counters(struct ib_device *ibdev,
+ struct mlx5_read_counters_attr *read_attr)
+{
+ struct mlx5_fc *fc = read_attr->hw_cntrs_hndl;
+ struct mlx5_ib_dev *dev = to_mdev(ibdev);
+
+ return mlx5_fc_query(dev->mdev, fc,
+ &read_attr->out[IB_COUNTER_PACKETS],
+ &read_attr->out[IB_COUNTER_BYTES]);
+}
+
+/* flow counters currently expose two counters packets and bytes */
+#define FLOW_COUNTERS_NUM 2
+static int counters_set_description(
+ struct ib_counters *counters, enum mlx5_ib_counters_type counters_type,
+ struct mlx5_ib_flow_counters_desc *desc_data, u32 ncounters)
+{
+ struct mlx5_ib_mcounters *mcounters = to_mcounters(counters);
+ u32 cntrs_max_index = 0;
+ int i;
+
+ if (counters_type != MLX5_IB_COUNTERS_FLOW)
+ return -EINVAL;
+
+ /* init the fields for the object */
+ mcounters->type = counters_type;
+ mcounters->read_counters = read_flow_counters;
+ mcounters->counters_num = FLOW_COUNTERS_NUM;
+ mcounters->ncounters = ncounters;
+ /* each counter entry have both description and index pair */
+ for (i = 0; i < ncounters; i++) {
+ if (desc_data[i].description > IB_COUNTER_BYTES)
+ return -EINVAL;
+
+ if (cntrs_max_index <= desc_data[i].index)
+ cntrs_max_index = desc_data[i].index + 1;
+ }
+
+ mutex_lock(&mcounters->mcntrs_mutex);
+ mcounters->counters_data = desc_data;
+ mcounters->cntrs_max_index = cntrs_max_index;
+ mutex_unlock(&mcounters->mcntrs_mutex);
+
+ return 0;
+}
+
+#define MAX_COUNTERS_NUM (USHRT_MAX / (sizeof(u32) * 2))
+int mlx5_ib_flow_counters_set_data(struct ib_counters *ibcounters,
+ struct mlx5_ib_create_flow *ucmd)
+{
+ struct mlx5_ib_mcounters *mcounters = to_mcounters(ibcounters);
+ struct mlx5_ib_flow_counters_data *cntrs_data = NULL;
+ struct mlx5_ib_flow_counters_desc *desc_data = NULL;
+ bool hw_hndl = false;
+ int ret = 0;
+
+ if (ucmd && ucmd->ncounters_data != 0) {
+ cntrs_data = ucmd->data;
+ if (cntrs_data->ncounters > MAX_COUNTERS_NUM)
+ return -EINVAL;
+
+ desc_data = kcalloc(cntrs_data->ncounters,
+ sizeof(*desc_data),
+ GFP_KERNEL);
+ if (!desc_data)
+ return -ENOMEM;
+
+ if (copy_from_user(desc_data,
+ u64_to_user_ptr(cntrs_data->counters_data),
+ sizeof(*desc_data) * cntrs_data->ncounters)) {
+ ret = -EFAULT;
+ goto free;
+ }
+ }
+
+ if (!mcounters->hw_cntrs_hndl) {
+ mcounters->hw_cntrs_hndl = mlx5_fc_create(
+ to_mdev(ibcounters->device)->mdev, false);
+ if (IS_ERR(mcounters->hw_cntrs_hndl)) {
+ ret = PTR_ERR(mcounters->hw_cntrs_hndl);
+ goto free;
+ }
+ hw_hndl = true;
+ }
+
+ if (desc_data) {
+ /* counters already bound to at least one flow */
+ if (mcounters->cntrs_max_index) {
+ ret = -EINVAL;
+ goto free_hndl;
+ }
+
+ ret = counters_set_description(ibcounters,
+ MLX5_IB_COUNTERS_FLOW,
+ desc_data,
+ cntrs_data->ncounters);
+ if (ret)
+ goto free_hndl;
+
+ } else if (!mcounters->cntrs_max_index) {
+ /* counters not bound yet, must have udata passed */
+ ret = -EINVAL;
+ goto free_hndl;
+ }
+
+ return 0;
+
+free_hndl:
+ if (hw_hndl) {
+ mlx5_fc_destroy(to_mdev(ibcounters->device)->mdev,
+ mcounters->hw_cntrs_hndl);
+ mcounters->hw_cntrs_hndl = NULL;
+ }
+free:
+ kfree(desc_data);
+ return ret;
+}
+
+void mlx5_ib_counters_clear_description(struct ib_counters *counters)
+{
+ struct mlx5_ib_mcounters *mcounters;
+
+ if (!counters || atomic_read(&counters->usecnt) != 1)
+ return;
+
+ mcounters = to_mcounters(counters);
+
+ mutex_lock(&mcounters->mcntrs_mutex);
+ kfree(mcounters->counters_data);
+ mcounters->counters_data = NULL;
+ mcounters->cntrs_max_index = 0;
+ mutex_unlock(&mcounters->mcntrs_mutex);
+}
+
+static const struct ib_device_ops hw_stats_ops = {
+ .alloc_hw_stats = mlx5_ib_alloc_hw_stats,
+ .get_hw_stats = mlx5_ib_get_hw_stats,
+ .counter_bind_qp = mlx5_ib_counter_bind_qp,
+ .counter_unbind_qp = mlx5_ib_counter_unbind_qp,
+ .counter_dealloc = mlx5_ib_counter_dealloc,
+ .counter_alloc_stats = mlx5_ib_counter_alloc_stats,
+ .counter_update_stats = mlx5_ib_counter_update_stats,
+};
+
+static const struct ib_device_ops counters_ops = {
+ .create_counters = mlx5_ib_create_counters,
+ .destroy_counters = mlx5_ib_destroy_counters,
+ .read_counters = mlx5_ib_read_counters,
+
+ INIT_RDMA_OBJ_SIZE(ib_counters, mlx5_ib_mcounters, ibcntrs),
+};
+
+int mlx5_ib_counters_init(struct mlx5_ib_dev *dev)
+{
+ ib_set_device_ops(&dev->ib_dev, &counters_ops);
+
+ if (!MLX5_CAP_GEN(dev->mdev, max_qp_cnt))
+ return 0;
+
+ ib_set_device_ops(&dev->ib_dev, &hw_stats_ops);
+ return mlx5_ib_alloc_counters(dev);
+}
+
+void mlx5_ib_counters_cleanup(struct mlx5_ib_dev *dev)
+{
+ if (!MLX5_CAP_GEN(dev->mdev, max_qp_cnt))
+ return;
+
+ mlx5_ib_dealloc_counters(dev);
+}
diff --git a/drivers/infiniband/hw/mlx5/counters.h b/drivers/infiniband/hw/mlx5/counters.h
new file mode 100644
index 0000000..1aa30c2
--- /dev/null
+++ b/drivers/infiniband/hw/mlx5/counters.h
@@ -0,0 +1,17 @@
+/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */
+/*
+ * Copyright (c) 2013-2020, Mellanox Technologies inc. All rights reserved.
+ */
+
+#ifndef _MLX5_IB_COUNTERS_H
+#define _MLX5_IB_COUNTERS_H
+
+#include "mlx5_ib.h"
+
+int mlx5_ib_counters_init(struct mlx5_ib_dev *dev);
+void mlx5_ib_counters_cleanup(struct mlx5_ib_dev *dev);
+void mlx5_ib_counters_clear_description(struct ib_counters *counters);
+int mlx5_ib_flow_counters_set_data(struct ib_counters *ibcounters,
+ struct mlx5_ib_create_flow *ucmd);
+u16 mlx5_ib_get_counters_id(struct mlx5_ib_dev *dev, u8 port_num);
+#endif /* _MLX5_IB_COUNTERS_H */
diff --git a/drivers/infiniband/hw/mlx5/devx.c b/drivers/infiniband/hw/mlx5/devx.c
index 655ea9c..9e3d8b82 100644
--- a/drivers/infiniband/hw/mlx5/devx.c
+++ b/drivers/infiniband/hw/mlx5/devx.c
@@ -14,6 +14,7 @@
#include <linux/mlx5/driver.h>
#include <linux/mlx5/fs.h>
#include "mlx5_ib.h"
+#include "devx.h"
#include "qp.h"
#include <linux/xarray.h>
@@ -89,22 +90,6 @@
u8 is_destroyed:1;
};
-#define MLX5_MAX_DESTROY_INBOX_SIZE_DW MLX5_ST_SZ_DW(delete_fte_in)
-struct devx_obj {
- struct mlx5_ib_dev *ib_dev;
- u64 obj_id;
- u32 dinlen; /* destroy inbox length */
- u32 dinbox[MLX5_MAX_DESTROY_INBOX_SIZE_DW];
- u32 flags;
- union {
- struct mlx5_ib_devx_mr devx_mr;
- struct mlx5_core_dct core_dct;
- struct mlx5_core_cq core_cq;
- u32 flow_counter_bulk_size;
- };
- struct list_head event_sub; /* holds devx_event_subscription entries */
-};
-
struct devx_umem {
struct mlx5_core_dev *mdev;
struct ib_umem *umem;
@@ -171,48 +156,6 @@
mlx5_cmd_exec(dev->mdev, in, sizeof(in), out, sizeof(out));
}
-bool mlx5_ib_devx_is_flow_dest(void *obj, int *dest_id, int *dest_type)
-{
- struct devx_obj *devx_obj = obj;
- u16 opcode = MLX5_GET(general_obj_in_cmd_hdr, devx_obj->dinbox, opcode);
-
- switch (opcode) {
- case MLX5_CMD_OP_DESTROY_TIR:
- *dest_type = MLX5_FLOW_DESTINATION_TYPE_TIR;
- *dest_id = MLX5_GET(general_obj_in_cmd_hdr, devx_obj->dinbox,
- obj_id);
- return true;
-
- case MLX5_CMD_OP_DESTROY_FLOW_TABLE:
- *dest_type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
- *dest_id = MLX5_GET(destroy_flow_table_in, devx_obj->dinbox,
- table_id);
- return true;
- default:
- return false;
- }
-}
-
-bool mlx5_ib_devx_is_flow_counter(void *obj, u32 offset, u32 *counter_id)
-{
- struct devx_obj *devx_obj = obj;
- u16 opcode = MLX5_GET(general_obj_in_cmd_hdr, devx_obj->dinbox, opcode);
-
- if (opcode == MLX5_CMD_OP_DEALLOC_FLOW_COUNTER) {
-
- if (offset && offset >= devx_obj->flow_counter_bulk_size)
- return false;
-
- *counter_id = MLX5_GET(dealloc_flow_counter_in,
- devx_obj->dinbox,
- flow_counter_id);
- *counter_id += offset;
- return true;
- }
-
- return false;
-}
-
static bool is_legacy_unaffiliated_event_num(u16 event_num)
{
switch (event_num) {
@@ -2419,17 +2362,24 @@
return NOTIFY_OK;
}
-void mlx5_ib_devx_init_event_table(struct mlx5_ib_dev *dev)
+int mlx5_ib_devx_init(struct mlx5_ib_dev *dev)
{
struct mlx5_devx_event_table *table = &dev->devx_event_table;
+ int uid;
- xa_init(&table->event_xa);
- mutex_init(&table->event_xa_lock);
- MLX5_NB_INIT(&table->devx_nb, devx_event_notifier, NOTIFY_ANY);
- mlx5_eq_notifier_register(dev->mdev, &table->devx_nb);
+ uid = mlx5_ib_devx_create(dev, false);
+ if (uid > 0) {
+ dev->devx_whitelist_uid = uid;
+ xa_init(&table->event_xa);
+ mutex_init(&table->event_xa_lock);
+ MLX5_NB_INIT(&table->devx_nb, devx_event_notifier, NOTIFY_ANY);
+ mlx5_eq_notifier_register(dev->mdev, &table->devx_nb);
+ }
+
+ return 0;
}
-void mlx5_ib_devx_cleanup_event_table(struct mlx5_ib_dev *dev)
+void mlx5_ib_devx_cleanup(struct mlx5_ib_dev *dev)
{
struct mlx5_devx_event_table *table = &dev->devx_event_table;
struct devx_event_subscription *sub, *tmp;
@@ -2437,17 +2387,21 @@
void *entry;
unsigned long id;
- mlx5_eq_notifier_unregister(dev->mdev, &table->devx_nb);
- mutex_lock(&dev->devx_event_table.event_xa_lock);
- xa_for_each(&table->event_xa, id, entry) {
- event = entry;
- list_for_each_entry_safe(sub, tmp, &event->unaffiliated_list,
- xa_list)
- devx_cleanup_subscription(dev, sub);
- kfree(entry);
+ if (dev->devx_whitelist_uid) {
+ mlx5_eq_notifier_unregister(dev->mdev, &table->devx_nb);
+ mutex_lock(&dev->devx_event_table.event_xa_lock);
+ xa_for_each(&table->event_xa, id, entry) {
+ event = entry;
+ list_for_each_entry_safe(
+ sub, tmp, &event->unaffiliated_list, xa_list)
+ devx_cleanup_subscription(dev, sub);
+ kfree(entry);
+ }
+ mutex_unlock(&dev->devx_event_table.event_xa_lock);
+ xa_destroy(&table->event_xa);
+
+ mlx5_ib_devx_destroy(dev, dev->devx_whitelist_uid);
}
- mutex_unlock(&dev->devx_event_table.event_xa_lock);
- xa_destroy(&table->event_xa);
}
static ssize_t devx_async_cmd_event_read(struct file *filp, char __user *buf,
diff --git a/drivers/infiniband/hw/mlx5/devx.h b/drivers/infiniband/hw/mlx5/devx.h
new file mode 100644
index 0000000..1f69866
--- /dev/null
+++ b/drivers/infiniband/hw/mlx5/devx.h
@@ -0,0 +1,45 @@
+/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */
+/*
+ * Copyright (c) 2019-2020, Mellanox Technologies inc. All rights reserved.
+ */
+
+#ifndef _MLX5_IB_DEVX_H
+#define _MLX5_IB_DEVX_H
+
+#include "mlx5_ib.h"
+
+#define MLX5_MAX_DESTROY_INBOX_SIZE_DW MLX5_ST_SZ_DW(delete_fte_in)
+struct devx_obj {
+ struct mlx5_ib_dev *ib_dev;
+ u64 obj_id;
+ u32 dinlen; /* destroy inbox length */
+ u32 dinbox[MLX5_MAX_DESTROY_INBOX_SIZE_DW];
+ u32 flags;
+ union {
+ struct mlx5_ib_devx_mr devx_mr;
+ struct mlx5_core_dct core_dct;
+ struct mlx5_core_cq core_cq;
+ u32 flow_counter_bulk_size;
+ };
+ struct list_head event_sub; /* holds devx_event_subscription entries */
+};
+#if IS_ENABLED(CONFIG_INFINIBAND_USER_ACCESS)
+int mlx5_ib_devx_create(struct mlx5_ib_dev *dev, bool is_user);
+void mlx5_ib_devx_destroy(struct mlx5_ib_dev *dev, u16 uid);
+int mlx5_ib_devx_init(struct mlx5_ib_dev *dev);
+void mlx5_ib_devx_cleanup(struct mlx5_ib_dev *dev);
+#else
+static inline int mlx5_ib_devx_create(struct mlx5_ib_dev *dev, bool is_user)
+{
+ return -EOPNOTSUPP;
+}
+static inline void mlx5_ib_devx_destroy(struct mlx5_ib_dev *dev, u16 uid) {}
+static inline int mlx5_ib_devx_init(struct mlx5_ib_dev *dev)
+{
+ return 0;
+}
+static inline void mlx5_ib_devx_cleanup(struct mlx5_ib_dev *dev)
+{
+}
+#endif
+#endif /* _MLX5_IB_DEVX_H */
diff --git a/drivers/infiniband/hw/mlx5/flow.c b/drivers/infiniband/hw/mlx5/flow.c
deleted file mode 100644
index 216a110..0000000
--- a/drivers/infiniband/hw/mlx5/flow.c
+++ /dev/null
@@ -1,765 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
-/*
- * Copyright (c) 2018, Mellanox Technologies inc. All rights reserved.
- */
-
-#include <rdma/ib_user_verbs.h>
-#include <rdma/ib_verbs.h>
-#include <rdma/uverbs_types.h>
-#include <rdma/uverbs_ioctl.h>
-#include <rdma/uverbs_std_types.h>
-#include <rdma/mlx5_user_ioctl_cmds.h>
-#include <rdma/mlx5_user_ioctl_verbs.h>
-#include <rdma/ib_umem.h>
-#include <linux/mlx5/driver.h>
-#include <linux/mlx5/fs.h>
-#include "mlx5_ib.h"
-
-#define UVERBS_MODULE_NAME mlx5_ib
-#include <rdma/uverbs_named_ioctl.h>
-
-static int
-mlx5_ib_ft_type_to_namespace(enum mlx5_ib_uapi_flow_table_type table_type,
- enum mlx5_flow_namespace_type *namespace)
-{
- switch (table_type) {
- case MLX5_IB_UAPI_FLOW_TABLE_TYPE_NIC_RX:
- *namespace = MLX5_FLOW_NAMESPACE_BYPASS;
- break;
- case MLX5_IB_UAPI_FLOW_TABLE_TYPE_NIC_TX:
- *namespace = MLX5_FLOW_NAMESPACE_EGRESS;
- break;
- case MLX5_IB_UAPI_FLOW_TABLE_TYPE_FDB:
- *namespace = MLX5_FLOW_NAMESPACE_FDB;
- break;
- case MLX5_IB_UAPI_FLOW_TABLE_TYPE_RDMA_RX:
- *namespace = MLX5_FLOW_NAMESPACE_RDMA_RX;
- break;
- case MLX5_IB_UAPI_FLOW_TABLE_TYPE_RDMA_TX:
- *namespace = MLX5_FLOW_NAMESPACE_RDMA_TX;
- break;
- default:
- return -EINVAL;
- }
-
- return 0;
-}
-
-static const struct uverbs_attr_spec mlx5_ib_flow_type[] = {
- [MLX5_IB_FLOW_TYPE_NORMAL] = {
- .type = UVERBS_ATTR_TYPE_PTR_IN,
- .u.ptr = {
- .len = sizeof(u16), /* data is priority */
- .min_len = sizeof(u16),
- }
- },
- [MLX5_IB_FLOW_TYPE_SNIFFER] = {
- .type = UVERBS_ATTR_TYPE_PTR_IN,
- UVERBS_ATTR_NO_DATA(),
- },
- [MLX5_IB_FLOW_TYPE_ALL_DEFAULT] = {
- .type = UVERBS_ATTR_TYPE_PTR_IN,
- UVERBS_ATTR_NO_DATA(),
- },
- [MLX5_IB_FLOW_TYPE_MC_DEFAULT] = {
- .type = UVERBS_ATTR_TYPE_PTR_IN,
- UVERBS_ATTR_NO_DATA(),
- },
-};
-
-static int get_dests(struct uverbs_attr_bundle *attrs,
- struct mlx5_ib_flow_matcher *fs_matcher, int *dest_id,
- int *dest_type, struct ib_qp **qp, u32 *flags)
-{
- bool dest_devx, dest_qp;
- void *devx_obj;
- int err;
-
- dest_devx = uverbs_attr_is_valid(attrs,
- MLX5_IB_ATTR_CREATE_FLOW_DEST_DEVX);
- dest_qp = uverbs_attr_is_valid(attrs,
- MLX5_IB_ATTR_CREATE_FLOW_DEST_QP);
-
- *flags = 0;
- err = uverbs_get_flags32(flags, attrs, MLX5_IB_ATTR_CREATE_FLOW_FLAGS,
- MLX5_IB_ATTR_CREATE_FLOW_FLAGS_DEFAULT_MISS |
- MLX5_IB_ATTR_CREATE_FLOW_FLAGS_DROP);
- if (err)
- return err;
-
- /* Both flags are not allowed */
- if (*flags & MLX5_IB_ATTR_CREATE_FLOW_FLAGS_DEFAULT_MISS &&
- *flags & MLX5_IB_ATTR_CREATE_FLOW_FLAGS_DROP)
- return -EINVAL;
-
- if (fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_BYPASS) {
- if (dest_devx && (dest_qp || *flags))
- return -EINVAL;
- else if (dest_qp && *flags)
- return -EINVAL;
- }
-
- /* Allow only DEVX object, drop as dest for FDB */
- if (fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_FDB && !(dest_devx ||
- (*flags & MLX5_IB_ATTR_CREATE_FLOW_FLAGS_DROP)))
- return -EINVAL;
-
- /* Allow only DEVX object or QP as dest when inserting to RDMA_RX */
- if ((fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_RDMA_RX) &&
- ((!dest_devx && !dest_qp) || (dest_devx && dest_qp)))
- return -EINVAL;
-
- *qp = NULL;
- if (dest_devx) {
- devx_obj =
- uverbs_attr_get_obj(attrs,
- MLX5_IB_ATTR_CREATE_FLOW_DEST_DEVX);
-
- /* Verify that the given DEVX object is a flow
- * steering destination.
- */
- if (!mlx5_ib_devx_is_flow_dest(devx_obj, dest_id, dest_type))
- return -EINVAL;
- /* Allow only flow table as dest when inserting to FDB or RDMA_RX */
- if ((fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_FDB ||
- fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_RDMA_RX) &&
- *dest_type != MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE)
- return -EINVAL;
- } else if (dest_qp) {
- struct mlx5_ib_qp *mqp;
-
- *qp = uverbs_attr_get_obj(attrs,
- MLX5_IB_ATTR_CREATE_FLOW_DEST_QP);
- if (IS_ERR(*qp))
- return PTR_ERR(*qp);
-
- if ((*qp)->qp_type != IB_QPT_RAW_PACKET)
- return -EINVAL;
-
- mqp = to_mqp(*qp);
- if (mqp->is_rss)
- *dest_id = mqp->rss_qp.tirn;
- else
- *dest_id = mqp->raw_packet_qp.rq.tirn;
- *dest_type = MLX5_FLOW_DESTINATION_TYPE_TIR;
- } else if (fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_EGRESS) {
- *dest_type = MLX5_FLOW_DESTINATION_TYPE_PORT;
- }
-
- if (*dest_type == MLX5_FLOW_DESTINATION_TYPE_TIR &&
- fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_EGRESS)
- return -EINVAL;
-
- return 0;
-}
-
-#define MLX5_IB_CREATE_FLOW_MAX_FLOW_ACTIONS 2
-static int UVERBS_HANDLER(MLX5_IB_METHOD_CREATE_FLOW)(
- struct uverbs_attr_bundle *attrs)
-{
- struct mlx5_flow_context flow_context = {.flow_tag =
- MLX5_FS_DEFAULT_FLOW_TAG};
- u32 *offset_attr, offset = 0, counter_id = 0;
- int dest_id, dest_type, inlen, len, ret, i;
- struct mlx5_ib_flow_handler *flow_handler;
- struct mlx5_ib_flow_matcher *fs_matcher;
- struct ib_uobject **arr_flow_actions;
- struct ib_uflow_resources *uflow_res;
- struct mlx5_flow_act flow_act = {};
- struct ib_qp *qp = NULL;
- void *devx_obj, *cmd_in;
- struct ib_uobject *uobj;
- struct mlx5_ib_dev *dev;
- u32 flags;
-
- if (!capable(CAP_NET_RAW))
- return -EPERM;
-
- fs_matcher = uverbs_attr_get_obj(attrs,
- MLX5_IB_ATTR_CREATE_FLOW_MATCHER);
- uobj = uverbs_attr_get_uobject(attrs, MLX5_IB_ATTR_CREATE_FLOW_HANDLE);
- dev = mlx5_udata_to_mdev(&attrs->driver_udata);
-
- if (get_dests(attrs, fs_matcher, &dest_id, &dest_type, &qp, &flags))
- return -EINVAL;
-
- if (flags & MLX5_IB_ATTR_CREATE_FLOW_FLAGS_DEFAULT_MISS)
- flow_act.action |= MLX5_FLOW_CONTEXT_ACTION_FWD_NEXT_NS;
-
- if (flags & MLX5_IB_ATTR_CREATE_FLOW_FLAGS_DROP)
- flow_act.action |= MLX5_FLOW_CONTEXT_ACTION_DROP;
-
- len = uverbs_attr_get_uobjs_arr(attrs,
- MLX5_IB_ATTR_CREATE_FLOW_ARR_COUNTERS_DEVX, &arr_flow_actions);
- if (len) {
- devx_obj = arr_flow_actions[0]->object;
-
- if (uverbs_attr_is_valid(attrs,
- MLX5_IB_ATTR_CREATE_FLOW_ARR_COUNTERS_DEVX_OFFSET)) {
-
- int num_offsets = uverbs_attr_ptr_get_array_size(
- attrs,
- MLX5_IB_ATTR_CREATE_FLOW_ARR_COUNTERS_DEVX_OFFSET,
- sizeof(u32));
-
- if (num_offsets != 1)
- return -EINVAL;
-
- offset_attr = uverbs_attr_get_alloced_ptr(
- attrs,
- MLX5_IB_ATTR_CREATE_FLOW_ARR_COUNTERS_DEVX_OFFSET);
- offset = *offset_attr;
- }
-
- if (!mlx5_ib_devx_is_flow_counter(devx_obj, offset,
- &counter_id))
- return -EINVAL;
-
- flow_act.action |= MLX5_FLOW_CONTEXT_ACTION_COUNT;
- }
-
- cmd_in = uverbs_attr_get_alloced_ptr(
- attrs, MLX5_IB_ATTR_CREATE_FLOW_MATCH_VALUE);
- inlen = uverbs_attr_get_len(attrs,
- MLX5_IB_ATTR_CREATE_FLOW_MATCH_VALUE);
-
- uflow_res = flow_resources_alloc(MLX5_IB_CREATE_FLOW_MAX_FLOW_ACTIONS);
- if (!uflow_res)
- return -ENOMEM;
-
- len = uverbs_attr_get_uobjs_arr(attrs,
- MLX5_IB_ATTR_CREATE_FLOW_ARR_FLOW_ACTIONS, &arr_flow_actions);
- for (i = 0; i < len; i++) {
- struct mlx5_ib_flow_action *maction =
- to_mflow_act(arr_flow_actions[i]->object);
-
- ret = parse_flow_flow_action(maction, false, &flow_act);
- if (ret)
- goto err_out;
- flow_resources_add(uflow_res, IB_FLOW_SPEC_ACTION_HANDLE,
- arr_flow_actions[i]->object);
- }
-
- ret = uverbs_copy_from(&flow_context.flow_tag, attrs,
- MLX5_IB_ATTR_CREATE_FLOW_TAG);
- if (!ret) {
- if (flow_context.flow_tag >= BIT(24)) {
- ret = -EINVAL;
- goto err_out;
- }
- flow_context.flags |= FLOW_CONTEXT_HAS_TAG;
- }
-
- flow_handler = mlx5_ib_raw_fs_rule_add(dev, fs_matcher,
- &flow_context,
- &flow_act,
- counter_id,
- cmd_in, inlen,
- dest_id, dest_type);
- if (IS_ERR(flow_handler)) {
- ret = PTR_ERR(flow_handler);
- goto err_out;
- }
-
- ib_set_flow(uobj, &flow_handler->ibflow, qp, &dev->ib_dev, uflow_res);
-
- return 0;
-err_out:
- ib_uverbs_flow_resources_free(uflow_res);
- return ret;
-}
-
-static int flow_matcher_cleanup(struct ib_uobject *uobject,
- enum rdma_remove_reason why,
- struct uverbs_attr_bundle *attrs)
-{
- struct mlx5_ib_flow_matcher *obj = uobject->object;
- int ret;
-
- ret = ib_destroy_usecnt(&obj->usecnt, why, uobject);
- if (ret)
- return ret;
-
- kfree(obj);
- return 0;
-}
-
-static int mlx5_ib_matcher_ns(struct uverbs_attr_bundle *attrs,
- struct mlx5_ib_flow_matcher *obj)
-{
- enum mlx5_ib_uapi_flow_table_type ft_type =
- MLX5_IB_UAPI_FLOW_TABLE_TYPE_NIC_RX;
- u32 flags;
- int err;
-
- /* New users should use MLX5_IB_ATTR_FLOW_MATCHER_FT_TYPE and older
- * users should switch to it. We leave this to not break userspace
- */
- if (uverbs_attr_is_valid(attrs, MLX5_IB_ATTR_FLOW_MATCHER_FT_TYPE) &&
- uverbs_attr_is_valid(attrs, MLX5_IB_ATTR_FLOW_MATCHER_FLOW_FLAGS))
- return -EINVAL;
-
- if (uverbs_attr_is_valid(attrs, MLX5_IB_ATTR_FLOW_MATCHER_FT_TYPE)) {
- err = uverbs_get_const(&ft_type, attrs,
- MLX5_IB_ATTR_FLOW_MATCHER_FT_TYPE);
- if (err)
- return err;
-
- err = mlx5_ib_ft_type_to_namespace(ft_type, &obj->ns_type);
- if (err)
- return err;
-
- return 0;
- }
-
- if (uverbs_attr_is_valid(attrs, MLX5_IB_ATTR_FLOW_MATCHER_FLOW_FLAGS)) {
- err = uverbs_get_flags32(&flags, attrs,
- MLX5_IB_ATTR_FLOW_MATCHER_FLOW_FLAGS,
- IB_FLOW_ATTR_FLAGS_EGRESS);
- if (err)
- return err;
-
- if (flags) {
- mlx5_ib_ft_type_to_namespace(
- MLX5_IB_UAPI_FLOW_TABLE_TYPE_NIC_TX,
- &obj->ns_type);
- return 0;
- }
- }
-
- obj->ns_type = MLX5_FLOW_NAMESPACE_BYPASS;
-
- return 0;
-}
-
-static int UVERBS_HANDLER(MLX5_IB_METHOD_FLOW_MATCHER_CREATE)(
- struct uverbs_attr_bundle *attrs)
-{
- struct ib_uobject *uobj = uverbs_attr_get_uobject(
- attrs, MLX5_IB_ATTR_FLOW_MATCHER_CREATE_HANDLE);
- struct mlx5_ib_dev *dev = mlx5_udata_to_mdev(&attrs->driver_udata);
- struct mlx5_ib_flow_matcher *obj;
- int err;
-
- obj = kzalloc(sizeof(struct mlx5_ib_flow_matcher), GFP_KERNEL);
- if (!obj)
- return -ENOMEM;
-
- obj->mask_len = uverbs_attr_get_len(
- attrs, MLX5_IB_ATTR_FLOW_MATCHER_MATCH_MASK);
- err = uverbs_copy_from(&obj->matcher_mask,
- attrs,
- MLX5_IB_ATTR_FLOW_MATCHER_MATCH_MASK);
- if (err)
- goto end;
-
- obj->flow_type = uverbs_attr_get_enum_id(
- attrs, MLX5_IB_ATTR_FLOW_MATCHER_FLOW_TYPE);
-
- if (obj->flow_type == MLX5_IB_FLOW_TYPE_NORMAL) {
- err = uverbs_copy_from(&obj->priority,
- attrs,
- MLX5_IB_ATTR_FLOW_MATCHER_FLOW_TYPE);
- if (err)
- goto end;
- }
-
- err = uverbs_copy_from(&obj->match_criteria_enable,
- attrs,
- MLX5_IB_ATTR_FLOW_MATCHER_MATCH_CRITERIA);
- if (err)
- goto end;
-
- err = mlx5_ib_matcher_ns(attrs, obj);
- if (err)
- goto end;
-
- uobj->object = obj;
- obj->mdev = dev->mdev;
- atomic_set(&obj->usecnt, 0);
- return 0;
-
-end:
- kfree(obj);
- return err;
-}
-
-void mlx5_ib_destroy_flow_action_raw(struct mlx5_ib_flow_action *maction)
-{
- switch (maction->flow_action_raw.sub_type) {
- case MLX5_IB_FLOW_ACTION_MODIFY_HEADER:
- mlx5_modify_header_dealloc(maction->flow_action_raw.dev->mdev,
- maction->flow_action_raw.modify_hdr);
- break;
- case MLX5_IB_FLOW_ACTION_PACKET_REFORMAT:
- mlx5_packet_reformat_dealloc(maction->flow_action_raw.dev->mdev,
- maction->flow_action_raw.pkt_reformat);
- break;
- case MLX5_IB_FLOW_ACTION_DECAP:
- break;
- default:
- break;
- }
-}
-
-static struct ib_flow_action *
-mlx5_ib_create_modify_header(struct mlx5_ib_dev *dev,
- enum mlx5_ib_uapi_flow_table_type ft_type,
- u8 num_actions, void *in)
-{
- enum mlx5_flow_namespace_type namespace;
- struct mlx5_ib_flow_action *maction;
- int ret;
-
- ret = mlx5_ib_ft_type_to_namespace(ft_type, &namespace);
- if (ret)
- return ERR_PTR(-EINVAL);
-
- maction = kzalloc(sizeof(*maction), GFP_KERNEL);
- if (!maction)
- return ERR_PTR(-ENOMEM);
-
- maction->flow_action_raw.modify_hdr =
- mlx5_modify_header_alloc(dev->mdev, namespace, num_actions, in);
-
- if (IS_ERR(maction->flow_action_raw.modify_hdr)) {
- ret = PTR_ERR(maction->flow_action_raw.modify_hdr);
- kfree(maction);
- return ERR_PTR(ret);
- }
- maction->flow_action_raw.sub_type =
- MLX5_IB_FLOW_ACTION_MODIFY_HEADER;
- maction->flow_action_raw.dev = dev;
-
- return &maction->ib_action;
-}
-
-static bool mlx5_ib_modify_header_supported(struct mlx5_ib_dev *dev)
-{
- return MLX5_CAP_FLOWTABLE_NIC_RX(dev->mdev,
- max_modify_header_actions) ||
- MLX5_CAP_FLOWTABLE_NIC_TX(dev->mdev,
- max_modify_header_actions) ||
- MLX5_CAP_FLOWTABLE_RDMA_TX(dev->mdev,
- max_modify_header_actions);
-}
-
-static int UVERBS_HANDLER(MLX5_IB_METHOD_FLOW_ACTION_CREATE_MODIFY_HEADER)(
- struct uverbs_attr_bundle *attrs)
-{
- struct ib_uobject *uobj = uverbs_attr_get_uobject(
- attrs, MLX5_IB_ATTR_CREATE_MODIFY_HEADER_HANDLE);
- struct mlx5_ib_dev *mdev = mlx5_udata_to_mdev(&attrs->driver_udata);
- enum mlx5_ib_uapi_flow_table_type ft_type;
- struct ib_flow_action *action;
- int num_actions;
- void *in;
- int ret;
-
- if (!mlx5_ib_modify_header_supported(mdev))
- return -EOPNOTSUPP;
-
- in = uverbs_attr_get_alloced_ptr(attrs,
- MLX5_IB_ATTR_CREATE_MODIFY_HEADER_ACTIONS_PRM);
-
- num_actions = uverbs_attr_ptr_get_array_size(
- attrs, MLX5_IB_ATTR_CREATE_MODIFY_HEADER_ACTIONS_PRM,
- MLX5_UN_SZ_BYTES(set_add_copy_action_in_auto));
- if (num_actions < 0)
- return num_actions;
-
- ret = uverbs_get_const(&ft_type, attrs,
- MLX5_IB_ATTR_CREATE_MODIFY_HEADER_FT_TYPE);
- if (ret)
- return ret;
- action = mlx5_ib_create_modify_header(mdev, ft_type, num_actions, in);
- if (IS_ERR(action))
- return PTR_ERR(action);
-
- uverbs_flow_action_fill_action(action, uobj, &mdev->ib_dev,
- IB_FLOW_ACTION_UNSPECIFIED);
-
- return 0;
-}
-
-static bool mlx5_ib_flow_action_packet_reformat_valid(struct mlx5_ib_dev *ibdev,
- u8 packet_reformat_type,
- u8 ft_type)
-{
- switch (packet_reformat_type) {
- case MLX5_IB_UAPI_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TO_L2_TUNNEL:
- if (ft_type == MLX5_IB_UAPI_FLOW_TABLE_TYPE_NIC_TX)
- return MLX5_CAP_FLOWTABLE(ibdev->mdev,
- encap_general_header);
- break;
- case MLX5_IB_UAPI_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TO_L3_TUNNEL:
- if (ft_type == MLX5_IB_UAPI_FLOW_TABLE_TYPE_NIC_TX)
- return MLX5_CAP_FLOWTABLE_NIC_TX(ibdev->mdev,
- reformat_l2_to_l3_tunnel);
- break;
- case MLX5_IB_UAPI_FLOW_ACTION_PACKET_REFORMAT_TYPE_L3_TUNNEL_TO_L2:
- if (ft_type == MLX5_IB_UAPI_FLOW_TABLE_TYPE_NIC_RX)
- return MLX5_CAP_FLOWTABLE_NIC_RX(ibdev->mdev,
- reformat_l3_tunnel_to_l2);
- break;
- case MLX5_IB_UAPI_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TUNNEL_TO_L2:
- if (ft_type == MLX5_IB_UAPI_FLOW_TABLE_TYPE_NIC_RX)
- return MLX5_CAP_FLOWTABLE_NIC_RX(ibdev->mdev, decap);
- break;
- default:
- break;
- }
-
- return false;
-}
-
-static int mlx5_ib_dv_to_prm_packet_reforamt_type(u8 dv_prt, u8 *prm_prt)
-{
- switch (dv_prt) {
- case MLX5_IB_UAPI_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TO_L2_TUNNEL:
- *prm_prt = MLX5_REFORMAT_TYPE_L2_TO_L2_TUNNEL;
- break;
- case MLX5_IB_UAPI_FLOW_ACTION_PACKET_REFORMAT_TYPE_L3_TUNNEL_TO_L2:
- *prm_prt = MLX5_REFORMAT_TYPE_L3_TUNNEL_TO_L2;
- break;
- case MLX5_IB_UAPI_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TO_L3_TUNNEL:
- *prm_prt = MLX5_REFORMAT_TYPE_L2_TO_L3_TUNNEL;
- break;
- default:
- return -EINVAL;
- }
-
- return 0;
-}
-
-static int mlx5_ib_flow_action_create_packet_reformat_ctx(
- struct mlx5_ib_dev *dev,
- struct mlx5_ib_flow_action *maction,
- u8 ft_type, u8 dv_prt,
- void *in, size_t len)
-{
- enum mlx5_flow_namespace_type namespace;
- u8 prm_prt;
- int ret;
-
- ret = mlx5_ib_ft_type_to_namespace(ft_type, &namespace);
- if (ret)
- return ret;
-
- ret = mlx5_ib_dv_to_prm_packet_reforamt_type(dv_prt, &prm_prt);
- if (ret)
- return ret;
-
- maction->flow_action_raw.pkt_reformat =
- mlx5_packet_reformat_alloc(dev->mdev, prm_prt, len,
- in, namespace);
- if (IS_ERR(maction->flow_action_raw.pkt_reformat)) {
- ret = PTR_ERR(maction->flow_action_raw.pkt_reformat);
- return ret;
- }
-
- maction->flow_action_raw.sub_type =
- MLX5_IB_FLOW_ACTION_PACKET_REFORMAT;
- maction->flow_action_raw.dev = dev;
-
- return 0;
-}
-
-static int UVERBS_HANDLER(MLX5_IB_METHOD_FLOW_ACTION_CREATE_PACKET_REFORMAT)(
- struct uverbs_attr_bundle *attrs)
-{
- struct ib_uobject *uobj = uverbs_attr_get_uobject(attrs,
- MLX5_IB_ATTR_CREATE_PACKET_REFORMAT_HANDLE);
- struct mlx5_ib_dev *mdev = mlx5_udata_to_mdev(&attrs->driver_udata);
- enum mlx5_ib_uapi_flow_action_packet_reformat_type dv_prt;
- enum mlx5_ib_uapi_flow_table_type ft_type;
- struct mlx5_ib_flow_action *maction;
- int ret;
-
- ret = uverbs_get_const(&ft_type, attrs,
- MLX5_IB_ATTR_CREATE_PACKET_REFORMAT_FT_TYPE);
- if (ret)
- return ret;
-
- ret = uverbs_get_const(&dv_prt, attrs,
- MLX5_IB_ATTR_CREATE_PACKET_REFORMAT_TYPE);
- if (ret)
- return ret;
-
- if (!mlx5_ib_flow_action_packet_reformat_valid(mdev, dv_prt, ft_type))
- return -EOPNOTSUPP;
-
- maction = kzalloc(sizeof(*maction), GFP_KERNEL);
- if (!maction)
- return -ENOMEM;
-
- if (dv_prt ==
- MLX5_IB_UAPI_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TUNNEL_TO_L2) {
- maction->flow_action_raw.sub_type =
- MLX5_IB_FLOW_ACTION_DECAP;
- maction->flow_action_raw.dev = mdev;
- } else {
- void *in;
- int len;
-
- in = uverbs_attr_get_alloced_ptr(attrs,
- MLX5_IB_ATTR_CREATE_PACKET_REFORMAT_DATA_BUF);
- if (IS_ERR(in)) {
- ret = PTR_ERR(in);
- goto free_maction;
- }
-
- len = uverbs_attr_get_len(attrs,
- MLX5_IB_ATTR_CREATE_PACKET_REFORMAT_DATA_BUF);
-
- ret = mlx5_ib_flow_action_create_packet_reformat_ctx(mdev,
- maction, ft_type, dv_prt, in, len);
- if (ret)
- goto free_maction;
- }
-
- uverbs_flow_action_fill_action(&maction->ib_action, uobj, &mdev->ib_dev,
- IB_FLOW_ACTION_UNSPECIFIED);
- return 0;
-
-free_maction:
- kfree(maction);
- return ret;
-}
-
-DECLARE_UVERBS_NAMED_METHOD(
- MLX5_IB_METHOD_CREATE_FLOW,
- UVERBS_ATTR_IDR(MLX5_IB_ATTR_CREATE_FLOW_HANDLE,
- UVERBS_OBJECT_FLOW,
- UVERBS_ACCESS_NEW,
- UA_MANDATORY),
- UVERBS_ATTR_PTR_IN(
- MLX5_IB_ATTR_CREATE_FLOW_MATCH_VALUE,
- UVERBS_ATTR_SIZE(1, sizeof(struct mlx5_ib_match_params)),
- UA_MANDATORY,
- UA_ALLOC_AND_COPY),
- UVERBS_ATTR_IDR(MLX5_IB_ATTR_CREATE_FLOW_MATCHER,
- MLX5_IB_OBJECT_FLOW_MATCHER,
- UVERBS_ACCESS_READ,
- UA_MANDATORY),
- UVERBS_ATTR_IDR(MLX5_IB_ATTR_CREATE_FLOW_DEST_QP,
- UVERBS_OBJECT_QP,
- UVERBS_ACCESS_READ),
- UVERBS_ATTR_IDR(MLX5_IB_ATTR_CREATE_FLOW_DEST_DEVX,
- MLX5_IB_OBJECT_DEVX_OBJ,
- UVERBS_ACCESS_READ),
- UVERBS_ATTR_IDRS_ARR(MLX5_IB_ATTR_CREATE_FLOW_ARR_FLOW_ACTIONS,
- UVERBS_OBJECT_FLOW_ACTION,
- UVERBS_ACCESS_READ, 1,
- MLX5_IB_CREATE_FLOW_MAX_FLOW_ACTIONS,
- UA_OPTIONAL),
- UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_CREATE_FLOW_TAG,
- UVERBS_ATTR_TYPE(u32),
- UA_OPTIONAL),
- UVERBS_ATTR_IDRS_ARR(MLX5_IB_ATTR_CREATE_FLOW_ARR_COUNTERS_DEVX,
- MLX5_IB_OBJECT_DEVX_OBJ,
- UVERBS_ACCESS_READ, 1, 1,
- UA_OPTIONAL),
- UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_CREATE_FLOW_ARR_COUNTERS_DEVX_OFFSET,
- UVERBS_ATTR_MIN_SIZE(sizeof(u32)),
- UA_OPTIONAL,
- UA_ALLOC_AND_COPY),
- UVERBS_ATTR_FLAGS_IN(MLX5_IB_ATTR_CREATE_FLOW_FLAGS,
- enum mlx5_ib_create_flow_flags,
- UA_OPTIONAL));
-
-DECLARE_UVERBS_NAMED_METHOD_DESTROY(
- MLX5_IB_METHOD_DESTROY_FLOW,
- UVERBS_ATTR_IDR(MLX5_IB_ATTR_CREATE_FLOW_HANDLE,
- UVERBS_OBJECT_FLOW,
- UVERBS_ACCESS_DESTROY,
- UA_MANDATORY));
-
-ADD_UVERBS_METHODS(mlx5_ib_fs,
- UVERBS_OBJECT_FLOW,
- &UVERBS_METHOD(MLX5_IB_METHOD_CREATE_FLOW),
- &UVERBS_METHOD(MLX5_IB_METHOD_DESTROY_FLOW));
-
-DECLARE_UVERBS_NAMED_METHOD(
- MLX5_IB_METHOD_FLOW_ACTION_CREATE_MODIFY_HEADER,
- UVERBS_ATTR_IDR(MLX5_IB_ATTR_CREATE_MODIFY_HEADER_HANDLE,
- UVERBS_OBJECT_FLOW_ACTION,
- UVERBS_ACCESS_NEW,
- UA_MANDATORY),
- UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_CREATE_MODIFY_HEADER_ACTIONS_PRM,
- UVERBS_ATTR_MIN_SIZE(MLX5_UN_SZ_BYTES(
- set_add_copy_action_in_auto)),
- UA_MANDATORY,
- UA_ALLOC_AND_COPY),
- UVERBS_ATTR_CONST_IN(MLX5_IB_ATTR_CREATE_MODIFY_HEADER_FT_TYPE,
- enum mlx5_ib_uapi_flow_table_type,
- UA_MANDATORY));
-
-DECLARE_UVERBS_NAMED_METHOD(
- MLX5_IB_METHOD_FLOW_ACTION_CREATE_PACKET_REFORMAT,
- UVERBS_ATTR_IDR(MLX5_IB_ATTR_CREATE_PACKET_REFORMAT_HANDLE,
- UVERBS_OBJECT_FLOW_ACTION,
- UVERBS_ACCESS_NEW,
- UA_MANDATORY),
- UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_CREATE_PACKET_REFORMAT_DATA_BUF,
- UVERBS_ATTR_MIN_SIZE(1),
- UA_ALLOC_AND_COPY,
- UA_OPTIONAL),
- UVERBS_ATTR_CONST_IN(MLX5_IB_ATTR_CREATE_PACKET_REFORMAT_TYPE,
- enum mlx5_ib_uapi_flow_action_packet_reformat_type,
- UA_MANDATORY),
- UVERBS_ATTR_CONST_IN(MLX5_IB_ATTR_CREATE_PACKET_REFORMAT_FT_TYPE,
- enum mlx5_ib_uapi_flow_table_type,
- UA_MANDATORY));
-
-ADD_UVERBS_METHODS(
- mlx5_ib_flow_actions,
- UVERBS_OBJECT_FLOW_ACTION,
- &UVERBS_METHOD(MLX5_IB_METHOD_FLOW_ACTION_CREATE_MODIFY_HEADER),
- &UVERBS_METHOD(MLX5_IB_METHOD_FLOW_ACTION_CREATE_PACKET_REFORMAT));
-
-DECLARE_UVERBS_NAMED_METHOD(
- MLX5_IB_METHOD_FLOW_MATCHER_CREATE,
- UVERBS_ATTR_IDR(MLX5_IB_ATTR_FLOW_MATCHER_CREATE_HANDLE,
- MLX5_IB_OBJECT_FLOW_MATCHER,
- UVERBS_ACCESS_NEW,
- UA_MANDATORY),
- UVERBS_ATTR_PTR_IN(
- MLX5_IB_ATTR_FLOW_MATCHER_MATCH_MASK,
- UVERBS_ATTR_SIZE(1, sizeof(struct mlx5_ib_match_params)),
- UA_MANDATORY),
- UVERBS_ATTR_ENUM_IN(MLX5_IB_ATTR_FLOW_MATCHER_FLOW_TYPE,
- mlx5_ib_flow_type,
- UA_MANDATORY),
- UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_FLOW_MATCHER_MATCH_CRITERIA,
- UVERBS_ATTR_TYPE(u8),
- UA_MANDATORY),
- UVERBS_ATTR_FLAGS_IN(MLX5_IB_ATTR_FLOW_MATCHER_FLOW_FLAGS,
- enum ib_flow_flags,
- UA_OPTIONAL),
- UVERBS_ATTR_CONST_IN(MLX5_IB_ATTR_FLOW_MATCHER_FT_TYPE,
- enum mlx5_ib_uapi_flow_table_type,
- UA_OPTIONAL));
-
-DECLARE_UVERBS_NAMED_METHOD_DESTROY(
- MLX5_IB_METHOD_FLOW_MATCHER_DESTROY,
- UVERBS_ATTR_IDR(MLX5_IB_ATTR_FLOW_MATCHER_DESTROY_HANDLE,
- MLX5_IB_OBJECT_FLOW_MATCHER,
- UVERBS_ACCESS_DESTROY,
- UA_MANDATORY));
-
-DECLARE_UVERBS_NAMED_OBJECT(MLX5_IB_OBJECT_FLOW_MATCHER,
- UVERBS_TYPE_ALLOC_IDR(flow_matcher_cleanup),
- &UVERBS_METHOD(MLX5_IB_METHOD_FLOW_MATCHER_CREATE),
- &UVERBS_METHOD(MLX5_IB_METHOD_FLOW_MATCHER_DESTROY));
-
-const struct uapi_definition mlx5_ib_flow_defs[] = {
- UAPI_DEF_CHAIN_OBJ_TREE_NAMED(
- MLX5_IB_OBJECT_FLOW_MATCHER),
- UAPI_DEF_CHAIN_OBJ_TREE(
- UVERBS_OBJECT_FLOW,
- &mlx5_ib_fs),
- UAPI_DEF_CHAIN_OBJ_TREE(UVERBS_OBJECT_FLOW_ACTION,
- &mlx5_ib_flow_actions),
- {},
-};
diff --git a/drivers/infiniband/hw/mlx5/fs.c b/drivers/infiniband/hw/mlx5/fs.c
new file mode 100644
index 0000000..e9cfb9a
--- /dev/null
+++ b/drivers/infiniband/hw/mlx5/fs.c
@@ -0,0 +1,2516 @@
+// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
+/*
+ * Copyright (c) 2018, Mellanox Technologies inc. All rights reserved.
+ */
+
+#include <rdma/ib_user_verbs.h>
+#include <rdma/ib_verbs.h>
+#include <rdma/uverbs_types.h>
+#include <rdma/uverbs_ioctl.h>
+#include <rdma/uverbs_std_types.h>
+#include <rdma/mlx5_user_ioctl_cmds.h>
+#include <rdma/mlx5_user_ioctl_verbs.h>
+#include <rdma/ib_umem.h>
+#include <linux/mlx5/driver.h>
+#include <linux/mlx5/fs.h>
+#include <linux/mlx5/fs_helpers.h>
+#include <linux/mlx5/accel.h>
+#include <linux/mlx5/eswitch.h>
+#include "mlx5_ib.h"
+#include "counters.h"
+#include "devx.h"
+#include "fs.h"
+
+#define UVERBS_MODULE_NAME mlx5_ib
+#include <rdma/uverbs_named_ioctl.h>
+
+enum {
+ MATCH_CRITERIA_ENABLE_OUTER_BIT,
+ MATCH_CRITERIA_ENABLE_MISC_BIT,
+ MATCH_CRITERIA_ENABLE_INNER_BIT,
+ MATCH_CRITERIA_ENABLE_MISC2_BIT
+};
+
+#define HEADER_IS_ZERO(match_criteria, headers) \
+ !(memchr_inv(MLX5_ADDR_OF(fte_match_param, match_criteria, headers), \
+ 0, MLX5_FLD_SZ_BYTES(fte_match_param, headers))) \
+
+static u8 get_match_criteria_enable(u32 *match_criteria)
+{
+ u8 match_criteria_enable;
+
+ match_criteria_enable =
+ (!HEADER_IS_ZERO(match_criteria, outer_headers)) <<
+ MATCH_CRITERIA_ENABLE_OUTER_BIT;
+ match_criteria_enable |=
+ (!HEADER_IS_ZERO(match_criteria, misc_parameters)) <<
+ MATCH_CRITERIA_ENABLE_MISC_BIT;
+ match_criteria_enable |=
+ (!HEADER_IS_ZERO(match_criteria, inner_headers)) <<
+ MATCH_CRITERIA_ENABLE_INNER_BIT;
+ match_criteria_enable |=
+ (!HEADER_IS_ZERO(match_criteria, misc_parameters_2)) <<
+ MATCH_CRITERIA_ENABLE_MISC2_BIT;
+
+ return match_criteria_enable;
+}
+
+static int set_proto(void *outer_c, void *outer_v, u8 mask, u8 val)
+{
+ u8 entry_mask;
+ u8 entry_val;
+ int err = 0;
+
+ if (!mask)
+ goto out;
+
+ entry_mask = MLX5_GET(fte_match_set_lyr_2_4, outer_c,
+ ip_protocol);
+ entry_val = MLX5_GET(fte_match_set_lyr_2_4, outer_v,
+ ip_protocol);
+ if (!entry_mask) {
+ MLX5_SET(fte_match_set_lyr_2_4, outer_c, ip_protocol, mask);
+ MLX5_SET(fte_match_set_lyr_2_4, outer_v, ip_protocol, val);
+ goto out;
+ }
+ /* Don't override existing ip protocol */
+ if (mask != entry_mask || val != entry_val)
+ err = -EINVAL;
+out:
+ return err;
+}
+
+static void set_flow_label(void *misc_c, void *misc_v, u32 mask, u32 val,
+ bool inner)
+{
+ if (inner) {
+ MLX5_SET(fte_match_set_misc,
+ misc_c, inner_ipv6_flow_label, mask);
+ MLX5_SET(fte_match_set_misc,
+ misc_v, inner_ipv6_flow_label, val);
+ } else {
+ MLX5_SET(fte_match_set_misc,
+ misc_c, outer_ipv6_flow_label, mask);
+ MLX5_SET(fte_match_set_misc,
+ misc_v, outer_ipv6_flow_label, val);
+ }
+}
+
+static void set_tos(void *outer_c, void *outer_v, u8 mask, u8 val)
+{
+ MLX5_SET(fte_match_set_lyr_2_4, outer_c, ip_ecn, mask);
+ MLX5_SET(fte_match_set_lyr_2_4, outer_v, ip_ecn, val);
+ MLX5_SET(fte_match_set_lyr_2_4, outer_c, ip_dscp, mask >> 2);
+ MLX5_SET(fte_match_set_lyr_2_4, outer_v, ip_dscp, val >> 2);
+}
+
+static int check_mpls_supp_fields(u32 field_support, const __be32 *set_mask)
+{
+ if (MLX5_GET(fte_match_mpls, set_mask, mpls_label) &&
+ !(field_support & MLX5_FIELD_SUPPORT_MPLS_LABEL))
+ return -EOPNOTSUPP;
+
+ if (MLX5_GET(fte_match_mpls, set_mask, mpls_exp) &&
+ !(field_support & MLX5_FIELD_SUPPORT_MPLS_EXP))
+ return -EOPNOTSUPP;
+
+ if (MLX5_GET(fte_match_mpls, set_mask, mpls_s_bos) &&
+ !(field_support & MLX5_FIELD_SUPPORT_MPLS_S_BOS))
+ return -EOPNOTSUPP;
+
+ if (MLX5_GET(fte_match_mpls, set_mask, mpls_ttl) &&
+ !(field_support & MLX5_FIELD_SUPPORT_MPLS_TTL))
+ return -EOPNOTSUPP;
+
+ return 0;
+}
+
+#define LAST_ETH_FIELD vlan_tag
+#define LAST_IB_FIELD sl
+#define LAST_IPV4_FIELD tos
+#define LAST_IPV6_FIELD traffic_class
+#define LAST_TCP_UDP_FIELD src_port
+#define LAST_TUNNEL_FIELD tunnel_id
+#define LAST_FLOW_TAG_FIELD tag_id
+#define LAST_DROP_FIELD size
+#define LAST_COUNTERS_FIELD counters
+
+/* Field is the last supported field */
+#define FIELDS_NOT_SUPPORTED(filter, field)\
+ memchr_inv((void *)&filter.field +\
+ sizeof(filter.field), 0,\
+ sizeof(filter) -\
+ offsetof(typeof(filter), field) -\
+ sizeof(filter.field))
+
+int parse_flow_flow_action(struct mlx5_ib_flow_action *maction,
+ bool is_egress,
+ struct mlx5_flow_act *action)
+{
+
+ switch (maction->ib_action.type) {
+ case IB_FLOW_ACTION_ESP:
+ if (action->action & (MLX5_FLOW_CONTEXT_ACTION_ENCRYPT |
+ MLX5_FLOW_CONTEXT_ACTION_DECRYPT))
+ return -EINVAL;
+ /* Currently only AES_GCM keymat is supported by the driver */
+ action->esp_id = (uintptr_t)maction->esp_aes_gcm.ctx;
+ action->action |= is_egress ?
+ MLX5_FLOW_CONTEXT_ACTION_ENCRYPT :
+ MLX5_FLOW_CONTEXT_ACTION_DECRYPT;
+ return 0;
+ case IB_FLOW_ACTION_UNSPECIFIED:
+ if (maction->flow_action_raw.sub_type ==
+ MLX5_IB_FLOW_ACTION_MODIFY_HEADER) {
+ if (action->action & MLX5_FLOW_CONTEXT_ACTION_MOD_HDR)
+ return -EINVAL;
+ action->action |= MLX5_FLOW_CONTEXT_ACTION_MOD_HDR;
+ action->modify_hdr =
+ maction->flow_action_raw.modify_hdr;
+ return 0;
+ }
+ if (maction->flow_action_raw.sub_type ==
+ MLX5_IB_FLOW_ACTION_DECAP) {
+ if (action->action & MLX5_FLOW_CONTEXT_ACTION_DECAP)
+ return -EINVAL;
+ action->action |= MLX5_FLOW_CONTEXT_ACTION_DECAP;
+ return 0;
+ }
+ if (maction->flow_action_raw.sub_type ==
+ MLX5_IB_FLOW_ACTION_PACKET_REFORMAT) {
+ if (action->action &
+ MLX5_FLOW_CONTEXT_ACTION_PACKET_REFORMAT)
+ return -EINVAL;
+ action->action |=
+ MLX5_FLOW_CONTEXT_ACTION_PACKET_REFORMAT;
+ action->pkt_reformat =
+ maction->flow_action_raw.pkt_reformat;
+ return 0;
+ }
+ fallthrough;
+ default:
+ return -EOPNOTSUPP;
+ }
+}
+
+static int parse_flow_attr(struct mlx5_core_dev *mdev,
+ struct mlx5_flow_spec *spec,
+ const union ib_flow_spec *ib_spec,
+ const struct ib_flow_attr *flow_attr,
+ struct mlx5_flow_act *action, u32 prev_type)
+{
+ struct mlx5_flow_context *flow_context = &spec->flow_context;
+ u32 *match_c = spec->match_criteria;
+ u32 *match_v = spec->match_value;
+ void *misc_params_c = MLX5_ADDR_OF(fte_match_param, match_c,
+ misc_parameters);
+ void *misc_params_v = MLX5_ADDR_OF(fte_match_param, match_v,
+ misc_parameters);
+ void *misc_params2_c = MLX5_ADDR_OF(fte_match_param, match_c,
+ misc_parameters_2);
+ void *misc_params2_v = MLX5_ADDR_OF(fte_match_param, match_v,
+ misc_parameters_2);
+ void *headers_c;
+ void *headers_v;
+ int match_ipv;
+ int ret;
+
+ if (ib_spec->type & IB_FLOW_SPEC_INNER) {
+ headers_c = MLX5_ADDR_OF(fte_match_param, match_c,
+ inner_headers);
+ headers_v = MLX5_ADDR_OF(fte_match_param, match_v,
+ inner_headers);
+ match_ipv = MLX5_CAP_FLOWTABLE_NIC_RX(mdev,
+ ft_field_support.inner_ip_version);
+ } else {
+ headers_c = MLX5_ADDR_OF(fte_match_param, match_c,
+ outer_headers);
+ headers_v = MLX5_ADDR_OF(fte_match_param, match_v,
+ outer_headers);
+ match_ipv = MLX5_CAP_FLOWTABLE_NIC_RX(mdev,
+ ft_field_support.outer_ip_version);
+ }
+
+ switch (ib_spec->type & ~IB_FLOW_SPEC_INNER) {
+ case IB_FLOW_SPEC_ETH:
+ if (FIELDS_NOT_SUPPORTED(ib_spec->eth.mask, LAST_ETH_FIELD))
+ return -EOPNOTSUPP;
+
+ ether_addr_copy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_c,
+ dmac_47_16),
+ ib_spec->eth.mask.dst_mac);
+ ether_addr_copy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
+ dmac_47_16),
+ ib_spec->eth.val.dst_mac);
+
+ ether_addr_copy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_c,
+ smac_47_16),
+ ib_spec->eth.mask.src_mac);
+ ether_addr_copy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
+ smac_47_16),
+ ib_spec->eth.val.src_mac);
+
+ if (ib_spec->eth.mask.vlan_tag) {
+ MLX5_SET(fte_match_set_lyr_2_4, headers_c,
+ cvlan_tag, 1);
+ MLX5_SET(fte_match_set_lyr_2_4, headers_v,
+ cvlan_tag, 1);
+
+ MLX5_SET(fte_match_set_lyr_2_4, headers_c,
+ first_vid, ntohs(ib_spec->eth.mask.vlan_tag));
+ MLX5_SET(fte_match_set_lyr_2_4, headers_v,
+ first_vid, ntohs(ib_spec->eth.val.vlan_tag));
+
+ MLX5_SET(fte_match_set_lyr_2_4, headers_c,
+ first_cfi,
+ ntohs(ib_spec->eth.mask.vlan_tag) >> 12);
+ MLX5_SET(fte_match_set_lyr_2_4, headers_v,
+ first_cfi,
+ ntohs(ib_spec->eth.val.vlan_tag) >> 12);
+
+ MLX5_SET(fte_match_set_lyr_2_4, headers_c,
+ first_prio,
+ ntohs(ib_spec->eth.mask.vlan_tag) >> 13);
+ MLX5_SET(fte_match_set_lyr_2_4, headers_v,
+ first_prio,
+ ntohs(ib_spec->eth.val.vlan_tag) >> 13);
+ }
+ MLX5_SET(fte_match_set_lyr_2_4, headers_c,
+ ethertype, ntohs(ib_spec->eth.mask.ether_type));
+ MLX5_SET(fte_match_set_lyr_2_4, headers_v,
+ ethertype, ntohs(ib_spec->eth.val.ether_type));
+ break;
+ case IB_FLOW_SPEC_IPV4:
+ if (FIELDS_NOT_SUPPORTED(ib_spec->ipv4.mask, LAST_IPV4_FIELD))
+ return -EOPNOTSUPP;
+
+ if (match_ipv) {
+ MLX5_SET(fte_match_set_lyr_2_4, headers_c,
+ ip_version, 0xf);
+ MLX5_SET(fte_match_set_lyr_2_4, headers_v,
+ ip_version, MLX5_FS_IPV4_VERSION);
+ } else {
+ MLX5_SET(fte_match_set_lyr_2_4, headers_c,
+ ethertype, 0xffff);
+ MLX5_SET(fte_match_set_lyr_2_4, headers_v,
+ ethertype, ETH_P_IP);
+ }
+
+ memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_c,
+ src_ipv4_src_ipv6.ipv4_layout.ipv4),
+ &ib_spec->ipv4.mask.src_ip,
+ sizeof(ib_spec->ipv4.mask.src_ip));
+ memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
+ src_ipv4_src_ipv6.ipv4_layout.ipv4),
+ &ib_spec->ipv4.val.src_ip,
+ sizeof(ib_spec->ipv4.val.src_ip));
+ memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_c,
+ dst_ipv4_dst_ipv6.ipv4_layout.ipv4),
+ &ib_spec->ipv4.mask.dst_ip,
+ sizeof(ib_spec->ipv4.mask.dst_ip));
+ memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
+ dst_ipv4_dst_ipv6.ipv4_layout.ipv4),
+ &ib_spec->ipv4.val.dst_ip,
+ sizeof(ib_spec->ipv4.val.dst_ip));
+
+ set_tos(headers_c, headers_v,
+ ib_spec->ipv4.mask.tos, ib_spec->ipv4.val.tos);
+
+ if (set_proto(headers_c, headers_v,
+ ib_spec->ipv4.mask.proto,
+ ib_spec->ipv4.val.proto))
+ return -EINVAL;
+ break;
+ case IB_FLOW_SPEC_IPV6:
+ if (FIELDS_NOT_SUPPORTED(ib_spec->ipv6.mask, LAST_IPV6_FIELD))
+ return -EOPNOTSUPP;
+
+ if (match_ipv) {
+ MLX5_SET(fte_match_set_lyr_2_4, headers_c,
+ ip_version, 0xf);
+ MLX5_SET(fte_match_set_lyr_2_4, headers_v,
+ ip_version, MLX5_FS_IPV6_VERSION);
+ } else {
+ MLX5_SET(fte_match_set_lyr_2_4, headers_c,
+ ethertype, 0xffff);
+ MLX5_SET(fte_match_set_lyr_2_4, headers_v,
+ ethertype, ETH_P_IPV6);
+ }
+
+ memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_c,
+ src_ipv4_src_ipv6.ipv6_layout.ipv6),
+ &ib_spec->ipv6.mask.src_ip,
+ sizeof(ib_spec->ipv6.mask.src_ip));
+ memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
+ src_ipv4_src_ipv6.ipv6_layout.ipv6),
+ &ib_spec->ipv6.val.src_ip,
+ sizeof(ib_spec->ipv6.val.src_ip));
+ memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_c,
+ dst_ipv4_dst_ipv6.ipv6_layout.ipv6),
+ &ib_spec->ipv6.mask.dst_ip,
+ sizeof(ib_spec->ipv6.mask.dst_ip));
+ memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
+ dst_ipv4_dst_ipv6.ipv6_layout.ipv6),
+ &ib_spec->ipv6.val.dst_ip,
+ sizeof(ib_spec->ipv6.val.dst_ip));
+
+ set_tos(headers_c, headers_v,
+ ib_spec->ipv6.mask.traffic_class,
+ ib_spec->ipv6.val.traffic_class);
+
+ if (set_proto(headers_c, headers_v,
+ ib_spec->ipv6.mask.next_hdr,
+ ib_spec->ipv6.val.next_hdr))
+ return -EINVAL;
+
+ set_flow_label(misc_params_c, misc_params_v,
+ ntohl(ib_spec->ipv6.mask.flow_label),
+ ntohl(ib_spec->ipv6.val.flow_label),
+ ib_spec->type & IB_FLOW_SPEC_INNER);
+ break;
+ case IB_FLOW_SPEC_ESP:
+ if (ib_spec->esp.mask.seq)
+ return -EOPNOTSUPP;
+
+ MLX5_SET(fte_match_set_misc, misc_params_c, outer_esp_spi,
+ ntohl(ib_spec->esp.mask.spi));
+ MLX5_SET(fte_match_set_misc, misc_params_v, outer_esp_spi,
+ ntohl(ib_spec->esp.val.spi));
+ break;
+ case IB_FLOW_SPEC_TCP:
+ if (FIELDS_NOT_SUPPORTED(ib_spec->tcp_udp.mask,
+ LAST_TCP_UDP_FIELD))
+ return -EOPNOTSUPP;
+
+ if (set_proto(headers_c, headers_v, 0xff, IPPROTO_TCP))
+ return -EINVAL;
+
+ MLX5_SET(fte_match_set_lyr_2_4, headers_c, tcp_sport,
+ ntohs(ib_spec->tcp_udp.mask.src_port));
+ MLX5_SET(fte_match_set_lyr_2_4, headers_v, tcp_sport,
+ ntohs(ib_spec->tcp_udp.val.src_port));
+
+ MLX5_SET(fte_match_set_lyr_2_4, headers_c, tcp_dport,
+ ntohs(ib_spec->tcp_udp.mask.dst_port));
+ MLX5_SET(fte_match_set_lyr_2_4, headers_v, tcp_dport,
+ ntohs(ib_spec->tcp_udp.val.dst_port));
+ break;
+ case IB_FLOW_SPEC_UDP:
+ if (FIELDS_NOT_SUPPORTED(ib_spec->tcp_udp.mask,
+ LAST_TCP_UDP_FIELD))
+ return -EOPNOTSUPP;
+
+ if (set_proto(headers_c, headers_v, 0xff, IPPROTO_UDP))
+ return -EINVAL;
+
+ MLX5_SET(fte_match_set_lyr_2_4, headers_c, udp_sport,
+ ntohs(ib_spec->tcp_udp.mask.src_port));
+ MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_sport,
+ ntohs(ib_spec->tcp_udp.val.src_port));
+
+ MLX5_SET(fte_match_set_lyr_2_4, headers_c, udp_dport,
+ ntohs(ib_spec->tcp_udp.mask.dst_port));
+ MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport,
+ ntohs(ib_spec->tcp_udp.val.dst_port));
+ break;
+ case IB_FLOW_SPEC_GRE:
+ if (ib_spec->gre.mask.c_ks_res0_ver)
+ return -EOPNOTSUPP;
+
+ if (set_proto(headers_c, headers_v, 0xff, IPPROTO_GRE))
+ return -EINVAL;
+
+ MLX5_SET(fte_match_set_lyr_2_4, headers_c, ip_protocol,
+ 0xff);
+ MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol,
+ IPPROTO_GRE);
+
+ MLX5_SET(fte_match_set_misc, misc_params_c, gre_protocol,
+ ntohs(ib_spec->gre.mask.protocol));
+ MLX5_SET(fte_match_set_misc, misc_params_v, gre_protocol,
+ ntohs(ib_spec->gre.val.protocol));
+
+ memcpy(MLX5_ADDR_OF(fte_match_set_misc, misc_params_c,
+ gre_key.nvgre.hi),
+ &ib_spec->gre.mask.key,
+ sizeof(ib_spec->gre.mask.key));
+ memcpy(MLX5_ADDR_OF(fte_match_set_misc, misc_params_v,
+ gre_key.nvgre.hi),
+ &ib_spec->gre.val.key,
+ sizeof(ib_spec->gre.val.key));
+ break;
+ case IB_FLOW_SPEC_MPLS:
+ switch (prev_type) {
+ case IB_FLOW_SPEC_UDP:
+ if (check_mpls_supp_fields(MLX5_CAP_FLOWTABLE_NIC_RX(mdev,
+ ft_field_support.outer_first_mpls_over_udp),
+ &ib_spec->mpls.mask.tag))
+ return -EOPNOTSUPP;
+
+ memcpy(MLX5_ADDR_OF(fte_match_set_misc2, misc_params2_v,
+ outer_first_mpls_over_udp),
+ &ib_spec->mpls.val.tag,
+ sizeof(ib_spec->mpls.val.tag));
+ memcpy(MLX5_ADDR_OF(fte_match_set_misc2, misc_params2_c,
+ outer_first_mpls_over_udp),
+ &ib_spec->mpls.mask.tag,
+ sizeof(ib_spec->mpls.mask.tag));
+ break;
+ case IB_FLOW_SPEC_GRE:
+ if (check_mpls_supp_fields(MLX5_CAP_FLOWTABLE_NIC_RX(mdev,
+ ft_field_support.outer_first_mpls_over_gre),
+ &ib_spec->mpls.mask.tag))
+ return -EOPNOTSUPP;
+
+ memcpy(MLX5_ADDR_OF(fte_match_set_misc2, misc_params2_v,
+ outer_first_mpls_over_gre),
+ &ib_spec->mpls.val.tag,
+ sizeof(ib_spec->mpls.val.tag));
+ memcpy(MLX5_ADDR_OF(fte_match_set_misc2, misc_params2_c,
+ outer_first_mpls_over_gre),
+ &ib_spec->mpls.mask.tag,
+ sizeof(ib_spec->mpls.mask.tag));
+ break;
+ default:
+ if (ib_spec->type & IB_FLOW_SPEC_INNER) {
+ if (check_mpls_supp_fields(MLX5_CAP_FLOWTABLE_NIC_RX(mdev,
+ ft_field_support.inner_first_mpls),
+ &ib_spec->mpls.mask.tag))
+ return -EOPNOTSUPP;
+
+ memcpy(MLX5_ADDR_OF(fte_match_set_misc2, misc_params2_v,
+ inner_first_mpls),
+ &ib_spec->mpls.val.tag,
+ sizeof(ib_spec->mpls.val.tag));
+ memcpy(MLX5_ADDR_OF(fte_match_set_misc2, misc_params2_c,
+ inner_first_mpls),
+ &ib_spec->mpls.mask.tag,
+ sizeof(ib_spec->mpls.mask.tag));
+ } else {
+ if (check_mpls_supp_fields(MLX5_CAP_FLOWTABLE_NIC_RX(mdev,
+ ft_field_support.outer_first_mpls),
+ &ib_spec->mpls.mask.tag))
+ return -EOPNOTSUPP;
+
+ memcpy(MLX5_ADDR_OF(fte_match_set_misc2, misc_params2_v,
+ outer_first_mpls),
+ &ib_spec->mpls.val.tag,
+ sizeof(ib_spec->mpls.val.tag));
+ memcpy(MLX5_ADDR_OF(fte_match_set_misc2, misc_params2_c,
+ outer_first_mpls),
+ &ib_spec->mpls.mask.tag,
+ sizeof(ib_spec->mpls.mask.tag));
+ }
+ }
+ break;
+ case IB_FLOW_SPEC_VXLAN_TUNNEL:
+ if (FIELDS_NOT_SUPPORTED(ib_spec->tunnel.mask,
+ LAST_TUNNEL_FIELD))
+ return -EOPNOTSUPP;
+
+ MLX5_SET(fte_match_set_misc, misc_params_c, vxlan_vni,
+ ntohl(ib_spec->tunnel.mask.tunnel_id));
+ MLX5_SET(fte_match_set_misc, misc_params_v, vxlan_vni,
+ ntohl(ib_spec->tunnel.val.tunnel_id));
+ break;
+ case IB_FLOW_SPEC_ACTION_TAG:
+ if (FIELDS_NOT_SUPPORTED(ib_spec->flow_tag,
+ LAST_FLOW_TAG_FIELD))
+ return -EOPNOTSUPP;
+ if (ib_spec->flow_tag.tag_id >= BIT(24))
+ return -EINVAL;
+
+ flow_context->flow_tag = ib_spec->flow_tag.tag_id;
+ flow_context->flags |= FLOW_CONTEXT_HAS_TAG;
+ break;
+ case IB_FLOW_SPEC_ACTION_DROP:
+ if (FIELDS_NOT_SUPPORTED(ib_spec->drop,
+ LAST_DROP_FIELD))
+ return -EOPNOTSUPP;
+ action->action |= MLX5_FLOW_CONTEXT_ACTION_DROP;
+ break;
+ case IB_FLOW_SPEC_ACTION_HANDLE:
+ ret = parse_flow_flow_action(to_mflow_act(ib_spec->action.act),
+ flow_attr->flags & IB_FLOW_ATTR_FLAGS_EGRESS, action);
+ if (ret)
+ return ret;
+ break;
+ case IB_FLOW_SPEC_ACTION_COUNT:
+ if (FIELDS_NOT_SUPPORTED(ib_spec->flow_count,
+ LAST_COUNTERS_FIELD))
+ return -EOPNOTSUPP;
+
+ /* for now support only one counters spec per flow */
+ if (action->action & MLX5_FLOW_CONTEXT_ACTION_COUNT)
+ return -EINVAL;
+
+ action->counters = ib_spec->flow_count.counters;
+ action->action |= MLX5_FLOW_CONTEXT_ACTION_COUNT;
+ break;
+ default:
+ return -EINVAL;
+ }
+
+ return 0;
+}
+
+/* If a flow could catch both multicast and unicast packets,
+ * it won't fall into the multicast flow steering table and this rule
+ * could steal other multicast packets.
+ */
+static bool flow_is_multicast_only(const struct ib_flow_attr *ib_attr)
+{
+ union ib_flow_spec *flow_spec;
+
+ if (ib_attr->type != IB_FLOW_ATTR_NORMAL ||
+ ib_attr->num_of_specs < 1)
+ return false;
+
+ flow_spec = (union ib_flow_spec *)(ib_attr + 1);
+ if (flow_spec->type == IB_FLOW_SPEC_IPV4) {
+ struct ib_flow_spec_ipv4 *ipv4_spec;
+
+ ipv4_spec = (struct ib_flow_spec_ipv4 *)flow_spec;
+ if (ipv4_is_multicast(ipv4_spec->val.dst_ip))
+ return true;
+
+ return false;
+ }
+
+ if (flow_spec->type == IB_FLOW_SPEC_ETH) {
+ struct ib_flow_spec_eth *eth_spec;
+
+ eth_spec = (struct ib_flow_spec_eth *)flow_spec;
+ return is_multicast_ether_addr(eth_spec->mask.dst_mac) &&
+ is_multicast_ether_addr(eth_spec->val.dst_mac);
+ }
+
+ return false;
+}
+
+enum valid_spec {
+ VALID_SPEC_INVALID,
+ VALID_SPEC_VALID,
+ VALID_SPEC_NA,
+};
+
+static enum valid_spec
+is_valid_esp_aes_gcm(struct mlx5_core_dev *mdev,
+ const struct mlx5_flow_spec *spec,
+ const struct mlx5_flow_act *flow_act,
+ bool egress)
+{
+ const u32 *match_c = spec->match_criteria;
+ bool is_crypto =
+ (flow_act->action & (MLX5_FLOW_CONTEXT_ACTION_ENCRYPT |
+ MLX5_FLOW_CONTEXT_ACTION_DECRYPT));
+ bool is_ipsec = mlx5_fs_is_ipsec_flow(match_c);
+ bool is_drop = flow_act->action & MLX5_FLOW_CONTEXT_ACTION_DROP;
+
+ /*
+ * Currently only crypto is supported in egress, when regular egress
+ * rules would be supported, always return VALID_SPEC_NA.
+ */
+ if (!is_crypto)
+ return VALID_SPEC_NA;
+
+ return is_crypto && is_ipsec &&
+ (!egress || (!is_drop &&
+ !(spec->flow_context.flags & FLOW_CONTEXT_HAS_TAG))) ?
+ VALID_SPEC_VALID : VALID_SPEC_INVALID;
+}
+
+static bool is_valid_spec(struct mlx5_core_dev *mdev,
+ const struct mlx5_flow_spec *spec,
+ const struct mlx5_flow_act *flow_act,
+ bool egress)
+{
+ /* We curretly only support ipsec egress flow */
+ return is_valid_esp_aes_gcm(mdev, spec, flow_act, egress) != VALID_SPEC_INVALID;
+}
+
+static bool is_valid_ethertype(struct mlx5_core_dev *mdev,
+ const struct ib_flow_attr *flow_attr,
+ bool check_inner)
+{
+ union ib_flow_spec *ib_spec = (union ib_flow_spec *)(flow_attr + 1);
+ int match_ipv = check_inner ?
+ MLX5_CAP_FLOWTABLE_NIC_RX(mdev,
+ ft_field_support.inner_ip_version) :
+ MLX5_CAP_FLOWTABLE_NIC_RX(mdev,
+ ft_field_support.outer_ip_version);
+ int inner_bit = check_inner ? IB_FLOW_SPEC_INNER : 0;
+ bool ipv4_spec_valid, ipv6_spec_valid;
+ unsigned int ip_spec_type = 0;
+ bool has_ethertype = false;
+ unsigned int spec_index;
+ bool mask_valid = true;
+ u16 eth_type = 0;
+ bool type_valid;
+
+ /* Validate that ethertype is correct */
+ for (spec_index = 0; spec_index < flow_attr->num_of_specs; spec_index++) {
+ if ((ib_spec->type == (IB_FLOW_SPEC_ETH | inner_bit)) &&
+ ib_spec->eth.mask.ether_type) {
+ mask_valid = (ib_spec->eth.mask.ether_type ==
+ htons(0xffff));
+ has_ethertype = true;
+ eth_type = ntohs(ib_spec->eth.val.ether_type);
+ } else if ((ib_spec->type == (IB_FLOW_SPEC_IPV4 | inner_bit)) ||
+ (ib_spec->type == (IB_FLOW_SPEC_IPV6 | inner_bit))) {
+ ip_spec_type = ib_spec->type;
+ }
+ ib_spec = (void *)ib_spec + ib_spec->size;
+ }
+
+ type_valid = (!has_ethertype) || (!ip_spec_type);
+ if (!type_valid && mask_valid) {
+ ipv4_spec_valid = (eth_type == ETH_P_IP) &&
+ (ip_spec_type == (IB_FLOW_SPEC_IPV4 | inner_bit));
+ ipv6_spec_valid = (eth_type == ETH_P_IPV6) &&
+ (ip_spec_type == (IB_FLOW_SPEC_IPV6 | inner_bit));
+
+ type_valid = (ipv4_spec_valid) || (ipv6_spec_valid) ||
+ (((eth_type == ETH_P_MPLS_UC) ||
+ (eth_type == ETH_P_MPLS_MC)) && match_ipv);
+ }
+
+ return type_valid;
+}
+
+static bool is_valid_attr(struct mlx5_core_dev *mdev,
+ const struct ib_flow_attr *flow_attr)
+{
+ return is_valid_ethertype(mdev, flow_attr, false) &&
+ is_valid_ethertype(mdev, flow_attr, true);
+}
+
+static void put_flow_table(struct mlx5_ib_dev *dev,
+ struct mlx5_ib_flow_prio *prio, bool ft_added)
+{
+ prio->refcount -= !!ft_added;
+ if (!prio->refcount) {
+ mlx5_destroy_flow_table(prio->flow_table);
+ prio->flow_table = NULL;
+ }
+}
+
+static int mlx5_ib_destroy_flow(struct ib_flow *flow_id)
+{
+ struct mlx5_ib_flow_handler *handler = container_of(flow_id,
+ struct mlx5_ib_flow_handler,
+ ibflow);
+ struct mlx5_ib_flow_handler *iter, *tmp;
+ struct mlx5_ib_dev *dev = handler->dev;
+
+ mutex_lock(&dev->flow_db->lock);
+
+ list_for_each_entry_safe(iter, tmp, &handler->list, list) {
+ mlx5_del_flow_rules(iter->rule);
+ put_flow_table(dev, iter->prio, true);
+ list_del(&iter->list);
+ kfree(iter);
+ }
+
+ mlx5_del_flow_rules(handler->rule);
+ put_flow_table(dev, handler->prio, true);
+ mlx5_ib_counters_clear_description(handler->ibcounters);
+ mutex_unlock(&dev->flow_db->lock);
+ if (handler->flow_matcher)
+ atomic_dec(&handler->flow_matcher->usecnt);
+ kfree(handler);
+
+ return 0;
+}
+
+static int ib_prio_to_core_prio(unsigned int priority, bool dont_trap)
+{
+ priority *= 2;
+ if (!dont_trap)
+ priority++;
+ return priority;
+}
+
+enum flow_table_type {
+ MLX5_IB_FT_RX,
+ MLX5_IB_FT_TX
+};
+
+#define MLX5_FS_MAX_TYPES 6
+#define MLX5_FS_MAX_ENTRIES BIT(16)
+
+static struct mlx5_ib_flow_prio *_get_prio(struct mlx5_flow_namespace *ns,
+ struct mlx5_ib_flow_prio *prio,
+ int priority,
+ int num_entries, int num_groups,
+ u32 flags)
+{
+ struct mlx5_flow_table_attr ft_attr = {};
+ struct mlx5_flow_table *ft;
+
+ ft_attr.prio = priority;
+ ft_attr.max_fte = num_entries;
+ ft_attr.flags = flags;
+ ft_attr.autogroup.max_num_groups = num_groups;
+ ft = mlx5_create_auto_grouped_flow_table(ns, &ft_attr);
+ if (IS_ERR(ft))
+ return ERR_CAST(ft);
+
+ prio->flow_table = ft;
+ prio->refcount = 0;
+ return prio;
+}
+
+static struct mlx5_ib_flow_prio *get_flow_table(struct mlx5_ib_dev *dev,
+ struct ib_flow_attr *flow_attr,
+ enum flow_table_type ft_type)
+{
+ bool dont_trap = flow_attr->flags & IB_FLOW_ATTR_FLAGS_DONT_TRAP;
+ struct mlx5_flow_namespace *ns = NULL;
+ struct mlx5_ib_flow_prio *prio;
+ struct mlx5_flow_table *ft;
+ int max_table_size;
+ int num_entries;
+ int num_groups;
+ bool esw_encap;
+ u32 flags = 0;
+ int priority;
+
+ max_table_size = BIT(MLX5_CAP_FLOWTABLE_NIC_RX(dev->mdev,
+ log_max_ft_size));
+ esw_encap = mlx5_eswitch_get_encap_mode(dev->mdev) !=
+ DEVLINK_ESWITCH_ENCAP_MODE_NONE;
+ if (flow_attr->type == IB_FLOW_ATTR_NORMAL) {
+ enum mlx5_flow_namespace_type fn_type;
+
+ if (flow_is_multicast_only(flow_attr) &&
+ !dont_trap)
+ priority = MLX5_IB_FLOW_MCAST_PRIO;
+ else
+ priority = ib_prio_to_core_prio(flow_attr->priority,
+ dont_trap);
+ if (ft_type == MLX5_IB_FT_RX) {
+ fn_type = MLX5_FLOW_NAMESPACE_BYPASS;
+ prio = &dev->flow_db->prios[priority];
+ if (!dev->is_rep && !esw_encap &&
+ MLX5_CAP_FLOWTABLE_NIC_RX(dev->mdev, decap))
+ flags |= MLX5_FLOW_TABLE_TUNNEL_EN_DECAP;
+ if (!dev->is_rep && !esw_encap &&
+ MLX5_CAP_FLOWTABLE_NIC_RX(dev->mdev,
+ reformat_l3_tunnel_to_l2))
+ flags |= MLX5_FLOW_TABLE_TUNNEL_EN_REFORMAT;
+ } else {
+ max_table_size =
+ BIT(MLX5_CAP_FLOWTABLE_NIC_TX(dev->mdev,
+ log_max_ft_size));
+ fn_type = MLX5_FLOW_NAMESPACE_EGRESS;
+ prio = &dev->flow_db->egress_prios[priority];
+ if (!dev->is_rep && !esw_encap &&
+ MLX5_CAP_FLOWTABLE_NIC_TX(dev->mdev, reformat))
+ flags |= MLX5_FLOW_TABLE_TUNNEL_EN_REFORMAT;
+ }
+ ns = mlx5_get_flow_namespace(dev->mdev, fn_type);
+ num_entries = MLX5_FS_MAX_ENTRIES;
+ num_groups = MLX5_FS_MAX_TYPES;
+ } else if (flow_attr->type == IB_FLOW_ATTR_ALL_DEFAULT ||
+ flow_attr->type == IB_FLOW_ATTR_MC_DEFAULT) {
+ ns = mlx5_get_flow_namespace(dev->mdev,
+ MLX5_FLOW_NAMESPACE_LEFTOVERS);
+ build_leftovers_ft_param(&priority,
+ &num_entries,
+ &num_groups);
+ prio = &dev->flow_db->prios[MLX5_IB_FLOW_LEFTOVERS_PRIO];
+ } else if (flow_attr->type == IB_FLOW_ATTR_SNIFFER) {
+ if (!MLX5_CAP_FLOWTABLE(dev->mdev,
+ allow_sniffer_and_nic_rx_shared_tir))
+ return ERR_PTR(-EOPNOTSUPP);
+
+ ns = mlx5_get_flow_namespace(dev->mdev, ft_type == MLX5_IB_FT_RX ?
+ MLX5_FLOW_NAMESPACE_SNIFFER_RX :
+ MLX5_FLOW_NAMESPACE_SNIFFER_TX);
+
+ prio = &dev->flow_db->sniffer[ft_type];
+ priority = 0;
+ num_entries = 1;
+ num_groups = 1;
+ }
+
+ if (!ns)
+ return ERR_PTR(-EOPNOTSUPP);
+
+ max_table_size = min_t(int, num_entries, max_table_size);
+
+ ft = prio->flow_table;
+ if (!ft)
+ return _get_prio(ns, prio, priority, max_table_size, num_groups,
+ flags);
+
+ return prio;
+}
+
+static void set_underlay_qp(struct mlx5_ib_dev *dev,
+ struct mlx5_flow_spec *spec,
+ u32 underlay_qpn)
+{
+ void *misc_params_c = MLX5_ADDR_OF(fte_match_param,
+ spec->match_criteria,
+ misc_parameters);
+ void *misc_params_v = MLX5_ADDR_OF(fte_match_param, spec->match_value,
+ misc_parameters);
+
+ if (underlay_qpn &&
+ MLX5_CAP_FLOWTABLE_NIC_RX(dev->mdev,
+ ft_field_support.bth_dst_qp)) {
+ MLX5_SET(fte_match_set_misc,
+ misc_params_v, bth_dst_qp, underlay_qpn);
+ MLX5_SET(fte_match_set_misc,
+ misc_params_c, bth_dst_qp, 0xffffff);
+ }
+}
+
+static void mlx5_ib_set_rule_source_port(struct mlx5_ib_dev *dev,
+ struct mlx5_flow_spec *spec,
+ struct mlx5_eswitch_rep *rep)
+{
+ struct mlx5_eswitch *esw = dev->mdev->priv.eswitch;
+ void *misc;
+
+ if (mlx5_eswitch_vport_match_metadata_enabled(esw)) {
+ misc = MLX5_ADDR_OF(fte_match_param, spec->match_value,
+ misc_parameters_2);
+
+ MLX5_SET(fte_match_set_misc2, misc, metadata_reg_c_0,
+ mlx5_eswitch_get_vport_metadata_for_match(esw,
+ rep->vport));
+ misc = MLX5_ADDR_OF(fte_match_param, spec->match_criteria,
+ misc_parameters_2);
+
+ MLX5_SET(fte_match_set_misc2, misc, metadata_reg_c_0,
+ mlx5_eswitch_get_vport_metadata_mask());
+ } else {
+ misc = MLX5_ADDR_OF(fte_match_param, spec->match_value,
+ misc_parameters);
+
+ MLX5_SET(fte_match_set_misc, misc, source_port, rep->vport);
+
+ misc = MLX5_ADDR_OF(fte_match_param, spec->match_criteria,
+ misc_parameters);
+
+ MLX5_SET_TO_ONES(fte_match_set_misc, misc, source_port);
+ }
+}
+
+static struct mlx5_ib_flow_handler *_create_flow_rule(struct mlx5_ib_dev *dev,
+ struct mlx5_ib_flow_prio *ft_prio,
+ const struct ib_flow_attr *flow_attr,
+ struct mlx5_flow_destination *dst,
+ u32 underlay_qpn,
+ struct mlx5_ib_create_flow *ucmd)
+{
+ struct mlx5_flow_table *ft = ft_prio->flow_table;
+ struct mlx5_ib_flow_handler *handler;
+ struct mlx5_flow_act flow_act = {};
+ struct mlx5_flow_spec *spec;
+ struct mlx5_flow_destination dest_arr[2] = {};
+ struct mlx5_flow_destination *rule_dst = dest_arr;
+ const void *ib_flow = (const void *)flow_attr + sizeof(*flow_attr);
+ unsigned int spec_index;
+ u32 prev_type = 0;
+ int err = 0;
+ int dest_num = 0;
+ bool is_egress = flow_attr->flags & IB_FLOW_ATTR_FLAGS_EGRESS;
+
+ if (!is_valid_attr(dev->mdev, flow_attr))
+ return ERR_PTR(-EINVAL);
+
+ if (dev->is_rep && is_egress)
+ return ERR_PTR(-EINVAL);
+
+ spec = kvzalloc(sizeof(*spec), GFP_KERNEL);
+ handler = kzalloc(sizeof(*handler), GFP_KERNEL);
+ if (!handler || !spec) {
+ err = -ENOMEM;
+ goto free;
+ }
+
+ INIT_LIST_HEAD(&handler->list);
+
+ for (spec_index = 0; spec_index < flow_attr->num_of_specs; spec_index++) {
+ err = parse_flow_attr(dev->mdev, spec,
+ ib_flow, flow_attr, &flow_act,
+ prev_type);
+ if (err < 0)
+ goto free;
+
+ prev_type = ((union ib_flow_spec *)ib_flow)->type;
+ ib_flow += ((union ib_flow_spec *)ib_flow)->size;
+ }
+
+ if (dst && !(flow_act.action & MLX5_FLOW_CONTEXT_ACTION_DROP)) {
+ memcpy(&dest_arr[0], dst, sizeof(*dst));
+ dest_num++;
+ }
+
+ if (!flow_is_multicast_only(flow_attr))
+ set_underlay_qp(dev, spec, underlay_qpn);
+
+ if (dev->is_rep) {
+ struct mlx5_eswitch_rep *rep;
+
+ rep = dev->port[flow_attr->port - 1].rep;
+ if (!rep) {
+ err = -EINVAL;
+ goto free;
+ }
+
+ mlx5_ib_set_rule_source_port(dev, spec, rep);
+ }
+
+ spec->match_criteria_enable = get_match_criteria_enable(spec->match_criteria);
+
+ if (is_egress &&
+ !is_valid_spec(dev->mdev, spec, &flow_act, is_egress)) {
+ err = -EINVAL;
+ goto free;
+ }
+
+ if (flow_act.action & MLX5_FLOW_CONTEXT_ACTION_COUNT) {
+ struct mlx5_ib_mcounters *mcounters;
+
+ err = mlx5_ib_flow_counters_set_data(flow_act.counters, ucmd);
+ if (err)
+ goto free;
+
+ mcounters = to_mcounters(flow_act.counters);
+ handler->ibcounters = flow_act.counters;
+ dest_arr[dest_num].type =
+ MLX5_FLOW_DESTINATION_TYPE_COUNTER;
+ dest_arr[dest_num].counter_id =
+ mlx5_fc_id(mcounters->hw_cntrs_hndl);
+ dest_num++;
+ }
+
+ if (flow_act.action & MLX5_FLOW_CONTEXT_ACTION_DROP) {
+ if (!dest_num)
+ rule_dst = NULL;
+ } else {
+ if (flow_attr->flags & IB_FLOW_ATTR_FLAGS_DONT_TRAP)
+ flow_act.action |=
+ MLX5_FLOW_CONTEXT_ACTION_FWD_NEXT_PRIO;
+ if (is_egress)
+ flow_act.action |= MLX5_FLOW_CONTEXT_ACTION_ALLOW;
+ else if (dest_num)
+ flow_act.action |= MLX5_FLOW_CONTEXT_ACTION_FWD_DEST;
+ }
+
+ if ((spec->flow_context.flags & FLOW_CONTEXT_HAS_TAG) &&
+ (flow_attr->type == IB_FLOW_ATTR_ALL_DEFAULT ||
+ flow_attr->type == IB_FLOW_ATTR_MC_DEFAULT)) {
+ mlx5_ib_warn(dev, "Flow tag %u and attribute type %x isn't allowed in leftovers\n",
+ spec->flow_context.flow_tag, flow_attr->type);
+ err = -EINVAL;
+ goto free;
+ }
+ handler->rule = mlx5_add_flow_rules(ft, spec,
+ &flow_act,
+ rule_dst, dest_num);
+
+ if (IS_ERR(handler->rule)) {
+ err = PTR_ERR(handler->rule);
+ goto free;
+ }
+
+ ft_prio->refcount++;
+ handler->prio = ft_prio;
+ handler->dev = dev;
+
+ ft_prio->flow_table = ft;
+free:
+ if (err && handler) {
+ mlx5_ib_counters_clear_description(handler->ibcounters);
+ kfree(handler);
+ }
+ kvfree(spec);
+ return err ? ERR_PTR(err) : handler;
+}
+
+static struct mlx5_ib_flow_handler *create_flow_rule(struct mlx5_ib_dev *dev,
+ struct mlx5_ib_flow_prio *ft_prio,
+ const struct ib_flow_attr *flow_attr,
+ struct mlx5_flow_destination *dst)
+{
+ return _create_flow_rule(dev, ft_prio, flow_attr, dst, 0, NULL);
+}
+
+enum {
+ LEFTOVERS_MC,
+ LEFTOVERS_UC,
+};
+
+static struct mlx5_ib_flow_handler *create_leftovers_rule(struct mlx5_ib_dev *dev,
+ struct mlx5_ib_flow_prio *ft_prio,
+ struct ib_flow_attr *flow_attr,
+ struct mlx5_flow_destination *dst)
+{
+ struct mlx5_ib_flow_handler *handler_ucast = NULL;
+ struct mlx5_ib_flow_handler *handler = NULL;
+
+ static struct {
+ struct ib_flow_attr flow_attr;
+ struct ib_flow_spec_eth eth_flow;
+ } leftovers_specs[] = {
+ [LEFTOVERS_MC] = {
+ .flow_attr = {
+ .num_of_specs = 1,
+ .size = sizeof(leftovers_specs[0])
+ },
+ .eth_flow = {
+ .type = IB_FLOW_SPEC_ETH,
+ .size = sizeof(struct ib_flow_spec_eth),
+ .mask = {.dst_mac = {0x1} },
+ .val = {.dst_mac = {0x1} }
+ }
+ },
+ [LEFTOVERS_UC] = {
+ .flow_attr = {
+ .num_of_specs = 1,
+ .size = sizeof(leftovers_specs[0])
+ },
+ .eth_flow = {
+ .type = IB_FLOW_SPEC_ETH,
+ .size = sizeof(struct ib_flow_spec_eth),
+ .mask = {.dst_mac = {0x1} },
+ .val = {.dst_mac = {} }
+ }
+ }
+ };
+
+ handler = create_flow_rule(dev, ft_prio,
+ &leftovers_specs[LEFTOVERS_MC].flow_attr,
+ dst);
+ if (!IS_ERR(handler) &&
+ flow_attr->type == IB_FLOW_ATTR_ALL_DEFAULT) {
+ handler_ucast = create_flow_rule(dev, ft_prio,
+ &leftovers_specs[LEFTOVERS_UC].flow_attr,
+ dst);
+ if (IS_ERR(handler_ucast)) {
+ mlx5_del_flow_rules(handler->rule);
+ ft_prio->refcount--;
+ kfree(handler);
+ handler = handler_ucast;
+ } else {
+ list_add(&handler_ucast->list, &handler->list);
+ }
+ }
+
+ return handler;
+}
+
+static struct mlx5_ib_flow_handler *create_sniffer_rule(struct mlx5_ib_dev *dev,
+ struct mlx5_ib_flow_prio *ft_rx,
+ struct mlx5_ib_flow_prio *ft_tx,
+ struct mlx5_flow_destination *dst)
+{
+ struct mlx5_ib_flow_handler *handler_rx;
+ struct mlx5_ib_flow_handler *handler_tx;
+ int err;
+ static const struct ib_flow_attr flow_attr = {
+ .num_of_specs = 0,
+ .size = sizeof(flow_attr)
+ };
+
+ handler_rx = create_flow_rule(dev, ft_rx, &flow_attr, dst);
+ if (IS_ERR(handler_rx)) {
+ err = PTR_ERR(handler_rx);
+ goto err;
+ }
+
+ handler_tx = create_flow_rule(dev, ft_tx, &flow_attr, dst);
+ if (IS_ERR(handler_tx)) {
+ err = PTR_ERR(handler_tx);
+ goto err_tx;
+ }
+
+ list_add(&handler_tx->list, &handler_rx->list);
+
+ return handler_rx;
+
+err_tx:
+ mlx5_del_flow_rules(handler_rx->rule);
+ ft_rx->refcount--;
+ kfree(handler_rx);
+err:
+ return ERR_PTR(err);
+}
+
+
+static struct ib_flow *mlx5_ib_create_flow(struct ib_qp *qp,
+ struct ib_flow_attr *flow_attr,
+ int domain,
+ struct ib_udata *udata)
+{
+ struct mlx5_ib_dev *dev = to_mdev(qp->device);
+ struct mlx5_ib_qp *mqp = to_mqp(qp);
+ struct mlx5_ib_flow_handler *handler = NULL;
+ struct mlx5_flow_destination *dst = NULL;
+ struct mlx5_ib_flow_prio *ft_prio_tx = NULL;
+ struct mlx5_ib_flow_prio *ft_prio;
+ bool is_egress = flow_attr->flags & IB_FLOW_ATTR_FLAGS_EGRESS;
+ struct mlx5_ib_create_flow *ucmd = NULL, ucmd_hdr;
+ size_t min_ucmd_sz, required_ucmd_sz;
+ int err;
+ int underlay_qpn;
+
+ if (udata && udata->inlen) {
+ min_ucmd_sz = offsetof(typeof(ucmd_hdr), reserved) +
+ sizeof(ucmd_hdr.reserved);
+ if (udata->inlen < min_ucmd_sz)
+ return ERR_PTR(-EOPNOTSUPP);
+
+ err = ib_copy_from_udata(&ucmd_hdr, udata, min_ucmd_sz);
+ if (err)
+ return ERR_PTR(err);
+
+ /* currently supports only one counters data */
+ if (ucmd_hdr.ncounters_data > 1)
+ return ERR_PTR(-EINVAL);
+
+ required_ucmd_sz = min_ucmd_sz +
+ sizeof(struct mlx5_ib_flow_counters_data) *
+ ucmd_hdr.ncounters_data;
+ if (udata->inlen > required_ucmd_sz &&
+ !ib_is_udata_cleared(udata, required_ucmd_sz,
+ udata->inlen - required_ucmd_sz))
+ return ERR_PTR(-EOPNOTSUPP);
+
+ ucmd = kzalloc(required_ucmd_sz, GFP_KERNEL);
+ if (!ucmd)
+ return ERR_PTR(-ENOMEM);
+
+ err = ib_copy_from_udata(ucmd, udata, required_ucmd_sz);
+ if (err)
+ goto free_ucmd;
+ }
+
+ if (flow_attr->priority > MLX5_IB_FLOW_LAST_PRIO) {
+ err = -ENOMEM;
+ goto free_ucmd;
+ }
+
+ if (domain != IB_FLOW_DOMAIN_USER ||
+ flow_attr->port > dev->num_ports ||
+ (flow_attr->flags & ~(IB_FLOW_ATTR_FLAGS_DONT_TRAP |
+ IB_FLOW_ATTR_FLAGS_EGRESS))) {
+ err = -EINVAL;
+ goto free_ucmd;
+ }
+
+ if (is_egress &&
+ (flow_attr->type == IB_FLOW_ATTR_ALL_DEFAULT ||
+ flow_attr->type == IB_FLOW_ATTR_MC_DEFAULT)) {
+ err = -EINVAL;
+ goto free_ucmd;
+ }
+
+ dst = kzalloc(sizeof(*dst), GFP_KERNEL);
+ if (!dst) {
+ err = -ENOMEM;
+ goto free_ucmd;
+ }
+
+ mutex_lock(&dev->flow_db->lock);
+
+ ft_prio = get_flow_table(dev, flow_attr,
+ is_egress ? MLX5_IB_FT_TX : MLX5_IB_FT_RX);
+ if (IS_ERR(ft_prio)) {
+ err = PTR_ERR(ft_prio);
+ goto unlock;
+ }
+ if (flow_attr->type == IB_FLOW_ATTR_SNIFFER) {
+ ft_prio_tx = get_flow_table(dev, flow_attr, MLX5_IB_FT_TX);
+ if (IS_ERR(ft_prio_tx)) {
+ err = PTR_ERR(ft_prio_tx);
+ ft_prio_tx = NULL;
+ goto destroy_ft;
+ }
+ }
+
+ if (is_egress) {
+ dst->type = MLX5_FLOW_DESTINATION_TYPE_PORT;
+ } else {
+ dst->type = MLX5_FLOW_DESTINATION_TYPE_TIR;
+ if (mqp->is_rss)
+ dst->tir_num = mqp->rss_qp.tirn;
+ else
+ dst->tir_num = mqp->raw_packet_qp.rq.tirn;
+ }
+
+ if (flow_attr->type == IB_FLOW_ATTR_NORMAL) {
+ underlay_qpn = (mqp->flags & IB_QP_CREATE_SOURCE_QPN) ?
+ mqp->underlay_qpn :
+ 0;
+ handler = _create_flow_rule(dev, ft_prio, flow_attr, dst,
+ underlay_qpn, ucmd);
+ } else if (flow_attr->type == IB_FLOW_ATTR_ALL_DEFAULT ||
+ flow_attr->type == IB_FLOW_ATTR_MC_DEFAULT) {
+ handler = create_leftovers_rule(dev, ft_prio, flow_attr,
+ dst);
+ } else if (flow_attr->type == IB_FLOW_ATTR_SNIFFER) {
+ handler = create_sniffer_rule(dev, ft_prio, ft_prio_tx, dst);
+ } else {
+ err = -EINVAL;
+ goto destroy_ft;
+ }
+
+ if (IS_ERR(handler)) {
+ err = PTR_ERR(handler);
+ handler = NULL;
+ goto destroy_ft;
+ }
+
+ mutex_unlock(&dev->flow_db->lock);
+ kfree(dst);
+ kfree(ucmd);
+
+ return &handler->ibflow;
+
+destroy_ft:
+ put_flow_table(dev, ft_prio, false);
+ if (ft_prio_tx)
+ put_flow_table(dev, ft_prio_tx, false);
+unlock:
+ mutex_unlock(&dev->flow_db->lock);
+ kfree(dst);
+free_ucmd:
+ kfree(ucmd);
+ return ERR_PTR(err);
+}
+
+static struct mlx5_ib_flow_prio *
+_get_flow_table(struct mlx5_ib_dev *dev,
+ struct mlx5_ib_flow_matcher *fs_matcher,
+ bool mcast)
+{
+ struct mlx5_flow_namespace *ns = NULL;
+ struct mlx5_ib_flow_prio *prio = NULL;
+ int max_table_size = 0;
+ bool esw_encap;
+ u32 flags = 0;
+ int priority;
+
+ if (mcast)
+ priority = MLX5_IB_FLOW_MCAST_PRIO;
+ else
+ priority = ib_prio_to_core_prio(fs_matcher->priority, false);
+
+ esw_encap = mlx5_eswitch_get_encap_mode(dev->mdev) !=
+ DEVLINK_ESWITCH_ENCAP_MODE_NONE;
+ if (fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_BYPASS) {
+ max_table_size = BIT(MLX5_CAP_FLOWTABLE_NIC_RX(dev->mdev,
+ log_max_ft_size));
+ if (MLX5_CAP_FLOWTABLE_NIC_RX(dev->mdev, decap) && !esw_encap)
+ flags |= MLX5_FLOW_TABLE_TUNNEL_EN_DECAP;
+ if (MLX5_CAP_FLOWTABLE_NIC_RX(dev->mdev,
+ reformat_l3_tunnel_to_l2) &&
+ !esw_encap)
+ flags |= MLX5_FLOW_TABLE_TUNNEL_EN_REFORMAT;
+ } else if (fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_EGRESS) {
+ max_table_size = BIT(
+ MLX5_CAP_FLOWTABLE_NIC_TX(dev->mdev, log_max_ft_size));
+ if (MLX5_CAP_FLOWTABLE_NIC_TX(dev->mdev, reformat) && !esw_encap)
+ flags |= MLX5_FLOW_TABLE_TUNNEL_EN_REFORMAT;
+ } else if (fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_FDB) {
+ max_table_size = BIT(
+ MLX5_CAP_ESW_FLOWTABLE_FDB(dev->mdev, log_max_ft_size));
+ if (MLX5_CAP_ESW_FLOWTABLE_FDB(dev->mdev, decap) && esw_encap)
+ flags |= MLX5_FLOW_TABLE_TUNNEL_EN_DECAP;
+ if (MLX5_CAP_ESW_FLOWTABLE_FDB(dev->mdev, reformat_l3_tunnel_to_l2) &&
+ esw_encap)
+ flags |= MLX5_FLOW_TABLE_TUNNEL_EN_REFORMAT;
+ priority = FDB_BYPASS_PATH;
+ } else if (fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_RDMA_RX) {
+ max_table_size =
+ BIT(MLX5_CAP_FLOWTABLE_RDMA_RX(dev->mdev,
+ log_max_ft_size));
+ priority = fs_matcher->priority;
+ } else if (fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_RDMA_TX) {
+ max_table_size =
+ BIT(MLX5_CAP_FLOWTABLE_RDMA_TX(dev->mdev,
+ log_max_ft_size));
+ priority = fs_matcher->priority;
+ }
+
+ max_table_size = min_t(int, max_table_size, MLX5_FS_MAX_ENTRIES);
+
+ ns = mlx5_get_flow_namespace(dev->mdev, fs_matcher->ns_type);
+ if (!ns)
+ return ERR_PTR(-EOPNOTSUPP);
+
+ if (fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_BYPASS)
+ prio = &dev->flow_db->prios[priority];
+ else if (fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_EGRESS)
+ prio = &dev->flow_db->egress_prios[priority];
+ else if (fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_FDB)
+ prio = &dev->flow_db->fdb;
+ else if (fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_RDMA_RX)
+ prio = &dev->flow_db->rdma_rx[priority];
+ else if (fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_RDMA_TX)
+ prio = &dev->flow_db->rdma_tx[priority];
+
+ if (!prio)
+ return ERR_PTR(-EINVAL);
+
+ if (prio->flow_table)
+ return prio;
+
+ return _get_prio(ns, prio, priority, max_table_size,
+ MLX5_FS_MAX_TYPES, flags);
+}
+
+static struct mlx5_ib_flow_handler *
+_create_raw_flow_rule(struct mlx5_ib_dev *dev,
+ struct mlx5_ib_flow_prio *ft_prio,
+ struct mlx5_flow_destination *dst,
+ struct mlx5_ib_flow_matcher *fs_matcher,
+ struct mlx5_flow_context *flow_context,
+ struct mlx5_flow_act *flow_act,
+ void *cmd_in, int inlen,
+ int dst_num)
+{
+ struct mlx5_ib_flow_handler *handler;
+ struct mlx5_flow_spec *spec;
+ struct mlx5_flow_table *ft = ft_prio->flow_table;
+ int err = 0;
+
+ spec = kvzalloc(sizeof(*spec), GFP_KERNEL);
+ handler = kzalloc(sizeof(*handler), GFP_KERNEL);
+ if (!handler || !spec) {
+ err = -ENOMEM;
+ goto free;
+ }
+
+ INIT_LIST_HEAD(&handler->list);
+
+ memcpy(spec->match_value, cmd_in, inlen);
+ memcpy(spec->match_criteria, fs_matcher->matcher_mask.match_params,
+ fs_matcher->mask_len);
+ spec->match_criteria_enable = fs_matcher->match_criteria_enable;
+ spec->flow_context = *flow_context;
+
+ handler->rule = mlx5_add_flow_rules(ft, spec,
+ flow_act, dst, dst_num);
+
+ if (IS_ERR(handler->rule)) {
+ err = PTR_ERR(handler->rule);
+ goto free;
+ }
+
+ ft_prio->refcount++;
+ handler->prio = ft_prio;
+ handler->dev = dev;
+ ft_prio->flow_table = ft;
+
+free:
+ if (err)
+ kfree(handler);
+ kvfree(spec);
+ return err ? ERR_PTR(err) : handler;
+}
+
+static bool raw_fs_is_multicast(struct mlx5_ib_flow_matcher *fs_matcher,
+ void *match_v)
+{
+ void *match_c;
+ void *match_v_set_lyr_2_4, *match_c_set_lyr_2_4;
+ void *dmac, *dmac_mask;
+ void *ipv4, *ipv4_mask;
+
+ if (!(fs_matcher->match_criteria_enable &
+ (1 << MATCH_CRITERIA_ENABLE_OUTER_BIT)))
+ return false;
+
+ match_c = fs_matcher->matcher_mask.match_params;
+ match_v_set_lyr_2_4 = MLX5_ADDR_OF(fte_match_param, match_v,
+ outer_headers);
+ match_c_set_lyr_2_4 = MLX5_ADDR_OF(fte_match_param, match_c,
+ outer_headers);
+
+ dmac = MLX5_ADDR_OF(fte_match_set_lyr_2_4, match_v_set_lyr_2_4,
+ dmac_47_16);
+ dmac_mask = MLX5_ADDR_OF(fte_match_set_lyr_2_4, match_c_set_lyr_2_4,
+ dmac_47_16);
+
+ if (is_multicast_ether_addr(dmac) &&
+ is_multicast_ether_addr(dmac_mask))
+ return true;
+
+ ipv4 = MLX5_ADDR_OF(fte_match_set_lyr_2_4, match_v_set_lyr_2_4,
+ dst_ipv4_dst_ipv6.ipv4_layout.ipv4);
+
+ ipv4_mask = MLX5_ADDR_OF(fte_match_set_lyr_2_4, match_c_set_lyr_2_4,
+ dst_ipv4_dst_ipv6.ipv4_layout.ipv4);
+
+ if (ipv4_is_multicast(*(__be32 *)(ipv4)) &&
+ ipv4_is_multicast(*(__be32 *)(ipv4_mask)))
+ return true;
+
+ return false;
+}
+
+static struct mlx5_ib_flow_handler *raw_fs_rule_add(
+ struct mlx5_ib_dev *dev, struct mlx5_ib_flow_matcher *fs_matcher,
+ struct mlx5_flow_context *flow_context, struct mlx5_flow_act *flow_act,
+ u32 counter_id, void *cmd_in, int inlen, int dest_id, int dest_type)
+{
+ struct mlx5_flow_destination *dst;
+ struct mlx5_ib_flow_prio *ft_prio;
+ struct mlx5_ib_flow_handler *handler;
+ int dst_num = 0;
+ bool mcast;
+ int err;
+
+ if (fs_matcher->flow_type != MLX5_IB_FLOW_TYPE_NORMAL)
+ return ERR_PTR(-EOPNOTSUPP);
+
+ if (fs_matcher->priority > MLX5_IB_FLOW_LAST_PRIO)
+ return ERR_PTR(-ENOMEM);
+
+ dst = kcalloc(2, sizeof(*dst), GFP_KERNEL);
+ if (!dst)
+ return ERR_PTR(-ENOMEM);
+
+ mcast = raw_fs_is_multicast(fs_matcher, cmd_in);
+ mutex_lock(&dev->flow_db->lock);
+
+ ft_prio = _get_flow_table(dev, fs_matcher, mcast);
+ if (IS_ERR(ft_prio)) {
+ err = PTR_ERR(ft_prio);
+ goto unlock;
+ }
+
+ if (dest_type == MLX5_FLOW_DESTINATION_TYPE_TIR) {
+ dst[dst_num].type = dest_type;
+ dst[dst_num++].tir_num = dest_id;
+ flow_act->action |= MLX5_FLOW_CONTEXT_ACTION_FWD_DEST;
+ } else if (dest_type == MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE) {
+ dst[dst_num].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE_NUM;
+ dst[dst_num++].ft_num = dest_id;
+ flow_act->action |= MLX5_FLOW_CONTEXT_ACTION_FWD_DEST;
+ } else if (dest_type == MLX5_FLOW_DESTINATION_TYPE_PORT) {
+ dst[dst_num++].type = MLX5_FLOW_DESTINATION_TYPE_PORT;
+ flow_act->action |= MLX5_FLOW_CONTEXT_ACTION_ALLOW;
+ }
+
+
+ if (flow_act->action & MLX5_FLOW_CONTEXT_ACTION_COUNT) {
+ dst[dst_num].type = MLX5_FLOW_DESTINATION_TYPE_COUNTER;
+ dst[dst_num].counter_id = counter_id;
+ dst_num++;
+ }
+
+ handler = _create_raw_flow_rule(dev, ft_prio, dst, fs_matcher,
+ flow_context, flow_act,
+ cmd_in, inlen, dst_num);
+
+ if (IS_ERR(handler)) {
+ err = PTR_ERR(handler);
+ goto destroy_ft;
+ }
+
+ mutex_unlock(&dev->flow_db->lock);
+ atomic_inc(&fs_matcher->usecnt);
+ handler->flow_matcher = fs_matcher;
+
+ kfree(dst);
+
+ return handler;
+
+destroy_ft:
+ put_flow_table(dev, ft_prio, false);
+unlock:
+ mutex_unlock(&dev->flow_db->lock);
+ kfree(dst);
+
+ return ERR_PTR(err);
+}
+
+static u32 mlx5_ib_flow_action_flags_to_accel_xfrm_flags(u32 mlx5_flags)
+{
+ u32 flags = 0;
+
+ if (mlx5_flags & MLX5_IB_UAPI_FLOW_ACTION_FLAGS_REQUIRE_METADATA)
+ flags |= MLX5_ACCEL_XFRM_FLAG_REQUIRE_METADATA;
+
+ return flags;
+}
+
+#define MLX5_FLOW_ACTION_ESP_CREATE_LAST_SUPPORTED \
+ MLX5_IB_UAPI_FLOW_ACTION_FLAGS_REQUIRE_METADATA
+static struct ib_flow_action *
+mlx5_ib_create_flow_action_esp(struct ib_device *device,
+ const struct ib_flow_action_attrs_esp *attr,
+ struct uverbs_attr_bundle *attrs)
+{
+ struct mlx5_ib_dev *mdev = to_mdev(device);
+ struct ib_uverbs_flow_action_esp_keymat_aes_gcm *aes_gcm;
+ struct mlx5_accel_esp_xfrm_attrs accel_attrs = {};
+ struct mlx5_ib_flow_action *action;
+ u64 action_flags;
+ u64 flags;
+ int err = 0;
+
+ err = uverbs_get_flags64(
+ &action_flags, attrs, MLX5_IB_ATTR_CREATE_FLOW_ACTION_FLAGS,
+ ((MLX5_FLOW_ACTION_ESP_CREATE_LAST_SUPPORTED << 1) - 1));
+ if (err)
+ return ERR_PTR(err);
+
+ flags = mlx5_ib_flow_action_flags_to_accel_xfrm_flags(action_flags);
+
+ /* We current only support a subset of the standard features. Only a
+ * keymat of type AES_GCM, with icv_len == 16, iv_algo == SEQ and esn
+ * (with overlap). Full offload mode isn't supported.
+ */
+ if (!attr->keymat || attr->replay || attr->encap ||
+ attr->spi || attr->seq || attr->tfc_pad ||
+ attr->hard_limit_pkts ||
+ (attr->flags & ~(IB_FLOW_ACTION_ESP_FLAGS_ESN_TRIGGERED |
+ IB_UVERBS_FLOW_ACTION_ESP_FLAGS_ENCRYPT)))
+ return ERR_PTR(-EOPNOTSUPP);
+
+ if (attr->keymat->protocol !=
+ IB_UVERBS_FLOW_ACTION_ESP_KEYMAT_AES_GCM)
+ return ERR_PTR(-EOPNOTSUPP);
+
+ aes_gcm = &attr->keymat->keymat.aes_gcm;
+
+ if (aes_gcm->icv_len != 16 ||
+ aes_gcm->iv_algo != IB_UVERBS_FLOW_ACTION_IV_ALGO_SEQ)
+ return ERR_PTR(-EOPNOTSUPP);
+
+ action = kmalloc(sizeof(*action), GFP_KERNEL);
+ if (!action)
+ return ERR_PTR(-ENOMEM);
+
+ action->esp_aes_gcm.ib_flags = attr->flags;
+ memcpy(&accel_attrs.keymat.aes_gcm.aes_key, &aes_gcm->aes_key,
+ sizeof(accel_attrs.keymat.aes_gcm.aes_key));
+ accel_attrs.keymat.aes_gcm.key_len = aes_gcm->key_len * 8;
+ memcpy(&accel_attrs.keymat.aes_gcm.salt, &aes_gcm->salt,
+ sizeof(accel_attrs.keymat.aes_gcm.salt));
+ memcpy(&accel_attrs.keymat.aes_gcm.seq_iv, &aes_gcm->iv,
+ sizeof(accel_attrs.keymat.aes_gcm.seq_iv));
+ accel_attrs.keymat.aes_gcm.icv_len = aes_gcm->icv_len * 8;
+ accel_attrs.keymat.aes_gcm.iv_algo = MLX5_ACCEL_ESP_AES_GCM_IV_ALGO_SEQ;
+ accel_attrs.keymat_type = MLX5_ACCEL_ESP_KEYMAT_AES_GCM;
+
+ accel_attrs.esn = attr->esn;
+ if (attr->flags & IB_FLOW_ACTION_ESP_FLAGS_ESN_TRIGGERED)
+ accel_attrs.flags |= MLX5_ACCEL_ESP_FLAGS_ESN_TRIGGERED;
+ if (attr->flags & IB_UVERBS_FLOW_ACTION_ESP_FLAGS_ESN_NEW_WINDOW)
+ accel_attrs.flags |= MLX5_ACCEL_ESP_FLAGS_ESN_STATE_OVERLAP;
+
+ if (attr->flags & IB_UVERBS_FLOW_ACTION_ESP_FLAGS_ENCRYPT)
+ accel_attrs.action |= MLX5_ACCEL_ESP_ACTION_ENCRYPT;
+
+ action->esp_aes_gcm.ctx =
+ mlx5_accel_esp_create_xfrm(mdev->mdev, &accel_attrs, flags);
+ if (IS_ERR(action->esp_aes_gcm.ctx)) {
+ err = PTR_ERR(action->esp_aes_gcm.ctx);
+ goto err_parse;
+ }
+
+ action->esp_aes_gcm.ib_flags = attr->flags;
+
+ return &action->ib_action;
+
+err_parse:
+ kfree(action);
+ return ERR_PTR(err);
+}
+
+static int
+mlx5_ib_modify_flow_action_esp(struct ib_flow_action *action,
+ const struct ib_flow_action_attrs_esp *attr,
+ struct uverbs_attr_bundle *attrs)
+{
+ struct mlx5_ib_flow_action *maction = to_mflow_act(action);
+ struct mlx5_accel_esp_xfrm_attrs accel_attrs;
+ int err = 0;
+
+ if (attr->keymat || attr->replay || attr->encap ||
+ attr->spi || attr->seq || attr->tfc_pad ||
+ attr->hard_limit_pkts ||
+ (attr->flags & ~(IB_FLOW_ACTION_ESP_FLAGS_ESN_TRIGGERED |
+ IB_FLOW_ACTION_ESP_FLAGS_MOD_ESP_ATTRS |
+ IB_UVERBS_FLOW_ACTION_ESP_FLAGS_ESN_NEW_WINDOW)))
+ return -EOPNOTSUPP;
+
+ /* Only the ESN value or the MLX5_ACCEL_ESP_FLAGS_ESN_STATE_OVERLAP can
+ * be modified.
+ */
+ if (!(maction->esp_aes_gcm.ib_flags &
+ IB_FLOW_ACTION_ESP_FLAGS_ESN_TRIGGERED) &&
+ attr->flags & (IB_FLOW_ACTION_ESP_FLAGS_ESN_TRIGGERED |
+ IB_UVERBS_FLOW_ACTION_ESP_FLAGS_ESN_NEW_WINDOW))
+ return -EINVAL;
+
+ memcpy(&accel_attrs, &maction->esp_aes_gcm.ctx->attrs,
+ sizeof(accel_attrs));
+
+ accel_attrs.esn = attr->esn;
+ if (attr->flags & IB_UVERBS_FLOW_ACTION_ESP_FLAGS_ESN_NEW_WINDOW)
+ accel_attrs.flags |= MLX5_ACCEL_ESP_FLAGS_ESN_STATE_OVERLAP;
+ else
+ accel_attrs.flags &= ~MLX5_ACCEL_ESP_FLAGS_ESN_STATE_OVERLAP;
+
+ err = mlx5_accel_esp_modify_xfrm(maction->esp_aes_gcm.ctx,
+ &accel_attrs);
+ if (err)
+ return err;
+
+ maction->esp_aes_gcm.ib_flags &=
+ ~IB_UVERBS_FLOW_ACTION_ESP_FLAGS_ESN_NEW_WINDOW;
+ maction->esp_aes_gcm.ib_flags |=
+ attr->flags & IB_UVERBS_FLOW_ACTION_ESP_FLAGS_ESN_NEW_WINDOW;
+
+ return 0;
+}
+
+static void destroy_flow_action_raw(struct mlx5_ib_flow_action *maction)
+{
+ switch (maction->flow_action_raw.sub_type) {
+ case MLX5_IB_FLOW_ACTION_MODIFY_HEADER:
+ mlx5_modify_header_dealloc(maction->flow_action_raw.dev->mdev,
+ maction->flow_action_raw.modify_hdr);
+ break;
+ case MLX5_IB_FLOW_ACTION_PACKET_REFORMAT:
+ mlx5_packet_reformat_dealloc(maction->flow_action_raw.dev->mdev,
+ maction->flow_action_raw.pkt_reformat);
+ break;
+ case MLX5_IB_FLOW_ACTION_DECAP:
+ break;
+ default:
+ break;
+ }
+}
+
+static int mlx5_ib_destroy_flow_action(struct ib_flow_action *action)
+{
+ struct mlx5_ib_flow_action *maction = to_mflow_act(action);
+
+ switch (action->type) {
+ case IB_FLOW_ACTION_ESP:
+ /*
+ * We only support aes_gcm by now, so we implicitly know this is
+ * the underline crypto.
+ */
+ mlx5_accel_esp_destroy_xfrm(maction->esp_aes_gcm.ctx);
+ break;
+ case IB_FLOW_ACTION_UNSPECIFIED:
+ destroy_flow_action_raw(maction);
+ break;
+ default:
+ WARN_ON(true);
+ break;
+ }
+
+ kfree(maction);
+ return 0;
+}
+
+static int
+mlx5_ib_ft_type_to_namespace(enum mlx5_ib_uapi_flow_table_type table_type,
+ enum mlx5_flow_namespace_type *namespace)
+{
+ switch (table_type) {
+ case MLX5_IB_UAPI_FLOW_TABLE_TYPE_NIC_RX:
+ *namespace = MLX5_FLOW_NAMESPACE_BYPASS;
+ break;
+ case MLX5_IB_UAPI_FLOW_TABLE_TYPE_NIC_TX:
+ *namespace = MLX5_FLOW_NAMESPACE_EGRESS;
+ break;
+ case MLX5_IB_UAPI_FLOW_TABLE_TYPE_FDB:
+ *namespace = MLX5_FLOW_NAMESPACE_FDB;
+ break;
+ case MLX5_IB_UAPI_FLOW_TABLE_TYPE_RDMA_RX:
+ *namespace = MLX5_FLOW_NAMESPACE_RDMA_RX;
+ break;
+ case MLX5_IB_UAPI_FLOW_TABLE_TYPE_RDMA_TX:
+ *namespace = MLX5_FLOW_NAMESPACE_RDMA_TX;
+ break;
+ default:
+ return -EINVAL;
+ }
+
+ return 0;
+}
+
+static const struct uverbs_attr_spec mlx5_ib_flow_type[] = {
+ [MLX5_IB_FLOW_TYPE_NORMAL] = {
+ .type = UVERBS_ATTR_TYPE_PTR_IN,
+ .u.ptr = {
+ .len = sizeof(u16), /* data is priority */
+ .min_len = sizeof(u16),
+ }
+ },
+ [MLX5_IB_FLOW_TYPE_SNIFFER] = {
+ .type = UVERBS_ATTR_TYPE_PTR_IN,
+ UVERBS_ATTR_NO_DATA(),
+ },
+ [MLX5_IB_FLOW_TYPE_ALL_DEFAULT] = {
+ .type = UVERBS_ATTR_TYPE_PTR_IN,
+ UVERBS_ATTR_NO_DATA(),
+ },
+ [MLX5_IB_FLOW_TYPE_MC_DEFAULT] = {
+ .type = UVERBS_ATTR_TYPE_PTR_IN,
+ UVERBS_ATTR_NO_DATA(),
+ },
+};
+
+static bool is_flow_dest(void *obj, int *dest_id, int *dest_type)
+{
+ struct devx_obj *devx_obj = obj;
+ u16 opcode = MLX5_GET(general_obj_in_cmd_hdr, devx_obj->dinbox, opcode);
+
+ switch (opcode) {
+ case MLX5_CMD_OP_DESTROY_TIR:
+ *dest_type = MLX5_FLOW_DESTINATION_TYPE_TIR;
+ *dest_id = MLX5_GET(general_obj_in_cmd_hdr, devx_obj->dinbox,
+ obj_id);
+ return true;
+
+ case MLX5_CMD_OP_DESTROY_FLOW_TABLE:
+ *dest_type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
+ *dest_id = MLX5_GET(destroy_flow_table_in, devx_obj->dinbox,
+ table_id);
+ return true;
+ default:
+ return false;
+ }
+}
+
+static int get_dests(struct uverbs_attr_bundle *attrs,
+ struct mlx5_ib_flow_matcher *fs_matcher, int *dest_id,
+ int *dest_type, struct ib_qp **qp, u32 *flags)
+{
+ bool dest_devx, dest_qp;
+ void *devx_obj;
+ int err;
+
+ dest_devx = uverbs_attr_is_valid(attrs,
+ MLX5_IB_ATTR_CREATE_FLOW_DEST_DEVX);
+ dest_qp = uverbs_attr_is_valid(attrs,
+ MLX5_IB_ATTR_CREATE_FLOW_DEST_QP);
+
+ *flags = 0;
+ err = uverbs_get_flags32(flags, attrs, MLX5_IB_ATTR_CREATE_FLOW_FLAGS,
+ MLX5_IB_ATTR_CREATE_FLOW_FLAGS_DEFAULT_MISS |
+ MLX5_IB_ATTR_CREATE_FLOW_FLAGS_DROP);
+ if (err)
+ return err;
+
+ /* Both flags are not allowed */
+ if (*flags & MLX5_IB_ATTR_CREATE_FLOW_FLAGS_DEFAULT_MISS &&
+ *flags & MLX5_IB_ATTR_CREATE_FLOW_FLAGS_DROP)
+ return -EINVAL;
+
+ if (fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_BYPASS) {
+ if (dest_devx && (dest_qp || *flags))
+ return -EINVAL;
+ else if (dest_qp && *flags)
+ return -EINVAL;
+ }
+
+ /* Allow only DEVX object, drop as dest for FDB */
+ if (fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_FDB && !(dest_devx ||
+ (*flags & MLX5_IB_ATTR_CREATE_FLOW_FLAGS_DROP)))
+ return -EINVAL;
+
+ /* Allow only DEVX object or QP as dest when inserting to RDMA_RX */
+ if ((fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_RDMA_RX) &&
+ ((!dest_devx && !dest_qp) || (dest_devx && dest_qp)))
+ return -EINVAL;
+
+ *qp = NULL;
+ if (dest_devx) {
+ devx_obj =
+ uverbs_attr_get_obj(attrs,
+ MLX5_IB_ATTR_CREATE_FLOW_DEST_DEVX);
+
+ /* Verify that the given DEVX object is a flow
+ * steering destination.
+ */
+ if (!is_flow_dest(devx_obj, dest_id, dest_type))
+ return -EINVAL;
+ /* Allow only flow table as dest when inserting to FDB or RDMA_RX */
+ if ((fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_FDB ||
+ fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_RDMA_RX) &&
+ *dest_type != MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE)
+ return -EINVAL;
+ } else if (dest_qp) {
+ struct mlx5_ib_qp *mqp;
+
+ *qp = uverbs_attr_get_obj(attrs,
+ MLX5_IB_ATTR_CREATE_FLOW_DEST_QP);
+ if (IS_ERR(*qp))
+ return PTR_ERR(*qp);
+
+ if ((*qp)->qp_type != IB_QPT_RAW_PACKET)
+ return -EINVAL;
+
+ mqp = to_mqp(*qp);
+ if (mqp->is_rss)
+ *dest_id = mqp->rss_qp.tirn;
+ else
+ *dest_id = mqp->raw_packet_qp.rq.tirn;
+ *dest_type = MLX5_FLOW_DESTINATION_TYPE_TIR;
+ } else if (fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_EGRESS ||
+ fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_RDMA_TX) {
+ *dest_type = MLX5_FLOW_DESTINATION_TYPE_PORT;
+ }
+
+ if (*dest_type == MLX5_FLOW_DESTINATION_TYPE_TIR &&
+ (fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_EGRESS ||
+ fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_RDMA_TX))
+ return -EINVAL;
+
+ return 0;
+}
+
+static bool is_flow_counter(void *obj, u32 offset, u32 *counter_id)
+{
+ struct devx_obj *devx_obj = obj;
+ u16 opcode = MLX5_GET(general_obj_in_cmd_hdr, devx_obj->dinbox, opcode);
+
+ if (opcode == MLX5_CMD_OP_DEALLOC_FLOW_COUNTER) {
+
+ if (offset && offset >= devx_obj->flow_counter_bulk_size)
+ return false;
+
+ *counter_id = MLX5_GET(dealloc_flow_counter_in,
+ devx_obj->dinbox,
+ flow_counter_id);
+ *counter_id += offset;
+ return true;
+ }
+
+ return false;
+}
+
+#define MLX5_IB_CREATE_FLOW_MAX_FLOW_ACTIONS 2
+static int UVERBS_HANDLER(MLX5_IB_METHOD_CREATE_FLOW)(
+ struct uverbs_attr_bundle *attrs)
+{
+ struct mlx5_flow_context flow_context = {.flow_tag =
+ MLX5_FS_DEFAULT_FLOW_TAG};
+ u32 *offset_attr, offset = 0, counter_id = 0;
+ int dest_id, dest_type = -1, inlen, len, ret, i;
+ struct mlx5_ib_flow_handler *flow_handler;
+ struct mlx5_ib_flow_matcher *fs_matcher;
+ struct ib_uobject **arr_flow_actions;
+ struct ib_uflow_resources *uflow_res;
+ struct mlx5_flow_act flow_act = {};
+ struct ib_qp *qp = NULL;
+ void *devx_obj, *cmd_in;
+ struct ib_uobject *uobj;
+ struct mlx5_ib_dev *dev;
+ u32 flags;
+
+ if (!capable(CAP_NET_RAW))
+ return -EPERM;
+
+ fs_matcher = uverbs_attr_get_obj(attrs,
+ MLX5_IB_ATTR_CREATE_FLOW_MATCHER);
+ uobj = uverbs_attr_get_uobject(attrs, MLX5_IB_ATTR_CREATE_FLOW_HANDLE);
+ dev = mlx5_udata_to_mdev(&attrs->driver_udata);
+
+ if (get_dests(attrs, fs_matcher, &dest_id, &dest_type, &qp, &flags))
+ return -EINVAL;
+
+ if (flags & MLX5_IB_ATTR_CREATE_FLOW_FLAGS_DEFAULT_MISS)
+ flow_act.action |= MLX5_FLOW_CONTEXT_ACTION_FWD_NEXT_NS;
+
+ if (flags & MLX5_IB_ATTR_CREATE_FLOW_FLAGS_DROP)
+ flow_act.action |= MLX5_FLOW_CONTEXT_ACTION_DROP;
+
+ len = uverbs_attr_get_uobjs_arr(attrs,
+ MLX5_IB_ATTR_CREATE_FLOW_ARR_COUNTERS_DEVX, &arr_flow_actions);
+ if (len) {
+ devx_obj = arr_flow_actions[0]->object;
+
+ if (uverbs_attr_is_valid(attrs,
+ MLX5_IB_ATTR_CREATE_FLOW_ARR_COUNTERS_DEVX_OFFSET)) {
+
+ int num_offsets = uverbs_attr_ptr_get_array_size(
+ attrs,
+ MLX5_IB_ATTR_CREATE_FLOW_ARR_COUNTERS_DEVX_OFFSET,
+ sizeof(u32));
+
+ if (num_offsets != 1)
+ return -EINVAL;
+
+ offset_attr = uverbs_attr_get_alloced_ptr(
+ attrs,
+ MLX5_IB_ATTR_CREATE_FLOW_ARR_COUNTERS_DEVX_OFFSET);
+ offset = *offset_attr;
+ }
+
+ if (!is_flow_counter(devx_obj, offset, &counter_id))
+ return -EINVAL;
+
+ flow_act.action |= MLX5_FLOW_CONTEXT_ACTION_COUNT;
+ }
+
+ cmd_in = uverbs_attr_get_alloced_ptr(
+ attrs, MLX5_IB_ATTR_CREATE_FLOW_MATCH_VALUE);
+ inlen = uverbs_attr_get_len(attrs,
+ MLX5_IB_ATTR_CREATE_FLOW_MATCH_VALUE);
+
+ uflow_res = flow_resources_alloc(MLX5_IB_CREATE_FLOW_MAX_FLOW_ACTIONS);
+ if (!uflow_res)
+ return -ENOMEM;
+
+ len = uverbs_attr_get_uobjs_arr(attrs,
+ MLX5_IB_ATTR_CREATE_FLOW_ARR_FLOW_ACTIONS, &arr_flow_actions);
+ for (i = 0; i < len; i++) {
+ struct mlx5_ib_flow_action *maction =
+ to_mflow_act(arr_flow_actions[i]->object);
+
+ ret = parse_flow_flow_action(maction, false, &flow_act);
+ if (ret)
+ goto err_out;
+ flow_resources_add(uflow_res, IB_FLOW_SPEC_ACTION_HANDLE,
+ arr_flow_actions[i]->object);
+ }
+
+ ret = uverbs_copy_from(&flow_context.flow_tag, attrs,
+ MLX5_IB_ATTR_CREATE_FLOW_TAG);
+ if (!ret) {
+ if (flow_context.flow_tag >= BIT(24)) {
+ ret = -EINVAL;
+ goto err_out;
+ }
+ flow_context.flags |= FLOW_CONTEXT_HAS_TAG;
+ }
+
+ flow_handler =
+ raw_fs_rule_add(dev, fs_matcher, &flow_context, &flow_act,
+ counter_id, cmd_in, inlen, dest_id, dest_type);
+ if (IS_ERR(flow_handler)) {
+ ret = PTR_ERR(flow_handler);
+ goto err_out;
+ }
+
+ ib_set_flow(uobj, &flow_handler->ibflow, qp, &dev->ib_dev, uflow_res);
+
+ return 0;
+err_out:
+ ib_uverbs_flow_resources_free(uflow_res);
+ return ret;
+}
+
+static int flow_matcher_cleanup(struct ib_uobject *uobject,
+ enum rdma_remove_reason why,
+ struct uverbs_attr_bundle *attrs)
+{
+ struct mlx5_ib_flow_matcher *obj = uobject->object;
+ int ret;
+
+ ret = ib_destroy_usecnt(&obj->usecnt, why, uobject);
+ if (ret)
+ return ret;
+
+ kfree(obj);
+ return 0;
+}
+
+static int mlx5_ib_matcher_ns(struct uverbs_attr_bundle *attrs,
+ struct mlx5_ib_flow_matcher *obj)
+{
+ enum mlx5_ib_uapi_flow_table_type ft_type =
+ MLX5_IB_UAPI_FLOW_TABLE_TYPE_NIC_RX;
+ u32 flags;
+ int err;
+
+ /* New users should use MLX5_IB_ATTR_FLOW_MATCHER_FT_TYPE and older
+ * users should switch to it. We leave this to not break userspace
+ */
+ if (uverbs_attr_is_valid(attrs, MLX5_IB_ATTR_FLOW_MATCHER_FT_TYPE) &&
+ uverbs_attr_is_valid(attrs, MLX5_IB_ATTR_FLOW_MATCHER_FLOW_FLAGS))
+ return -EINVAL;
+
+ if (uverbs_attr_is_valid(attrs, MLX5_IB_ATTR_FLOW_MATCHER_FT_TYPE)) {
+ err = uverbs_get_const(&ft_type, attrs,
+ MLX5_IB_ATTR_FLOW_MATCHER_FT_TYPE);
+ if (err)
+ return err;
+
+ err = mlx5_ib_ft_type_to_namespace(ft_type, &obj->ns_type);
+ if (err)
+ return err;
+
+ return 0;
+ }
+
+ if (uverbs_attr_is_valid(attrs, MLX5_IB_ATTR_FLOW_MATCHER_FLOW_FLAGS)) {
+ err = uverbs_get_flags32(&flags, attrs,
+ MLX5_IB_ATTR_FLOW_MATCHER_FLOW_FLAGS,
+ IB_FLOW_ATTR_FLAGS_EGRESS);
+ if (err)
+ return err;
+
+ if (flags) {
+ mlx5_ib_ft_type_to_namespace(
+ MLX5_IB_UAPI_FLOW_TABLE_TYPE_NIC_TX,
+ &obj->ns_type);
+ return 0;
+ }
+ }
+
+ obj->ns_type = MLX5_FLOW_NAMESPACE_BYPASS;
+
+ return 0;
+}
+
+static int UVERBS_HANDLER(MLX5_IB_METHOD_FLOW_MATCHER_CREATE)(
+ struct uverbs_attr_bundle *attrs)
+{
+ struct ib_uobject *uobj = uverbs_attr_get_uobject(
+ attrs, MLX5_IB_ATTR_FLOW_MATCHER_CREATE_HANDLE);
+ struct mlx5_ib_dev *dev = mlx5_udata_to_mdev(&attrs->driver_udata);
+ struct mlx5_ib_flow_matcher *obj;
+ int err;
+
+ obj = kzalloc(sizeof(struct mlx5_ib_flow_matcher), GFP_KERNEL);
+ if (!obj)
+ return -ENOMEM;
+
+ obj->mask_len = uverbs_attr_get_len(
+ attrs, MLX5_IB_ATTR_FLOW_MATCHER_MATCH_MASK);
+ err = uverbs_copy_from(&obj->matcher_mask,
+ attrs,
+ MLX5_IB_ATTR_FLOW_MATCHER_MATCH_MASK);
+ if (err)
+ goto end;
+
+ obj->flow_type = uverbs_attr_get_enum_id(
+ attrs, MLX5_IB_ATTR_FLOW_MATCHER_FLOW_TYPE);
+
+ if (obj->flow_type == MLX5_IB_FLOW_TYPE_NORMAL) {
+ err = uverbs_copy_from(&obj->priority,
+ attrs,
+ MLX5_IB_ATTR_FLOW_MATCHER_FLOW_TYPE);
+ if (err)
+ goto end;
+ }
+
+ err = uverbs_copy_from(&obj->match_criteria_enable,
+ attrs,
+ MLX5_IB_ATTR_FLOW_MATCHER_MATCH_CRITERIA);
+ if (err)
+ goto end;
+
+ err = mlx5_ib_matcher_ns(attrs, obj);
+ if (err)
+ goto end;
+
+ uobj->object = obj;
+ obj->mdev = dev->mdev;
+ atomic_set(&obj->usecnt, 0);
+ return 0;
+
+end:
+ kfree(obj);
+ return err;
+}
+
+static struct ib_flow_action *
+mlx5_ib_create_modify_header(struct mlx5_ib_dev *dev,
+ enum mlx5_ib_uapi_flow_table_type ft_type,
+ u8 num_actions, void *in)
+{
+ enum mlx5_flow_namespace_type namespace;
+ struct mlx5_ib_flow_action *maction;
+ int ret;
+
+ ret = mlx5_ib_ft_type_to_namespace(ft_type, &namespace);
+ if (ret)
+ return ERR_PTR(-EINVAL);
+
+ maction = kzalloc(sizeof(*maction), GFP_KERNEL);
+ if (!maction)
+ return ERR_PTR(-ENOMEM);
+
+ maction->flow_action_raw.modify_hdr =
+ mlx5_modify_header_alloc(dev->mdev, namespace, num_actions, in);
+
+ if (IS_ERR(maction->flow_action_raw.modify_hdr)) {
+ ret = PTR_ERR(maction->flow_action_raw.modify_hdr);
+ kfree(maction);
+ return ERR_PTR(ret);
+ }
+ maction->flow_action_raw.sub_type =
+ MLX5_IB_FLOW_ACTION_MODIFY_HEADER;
+ maction->flow_action_raw.dev = dev;
+
+ return &maction->ib_action;
+}
+
+static bool mlx5_ib_modify_header_supported(struct mlx5_ib_dev *dev)
+{
+ return MLX5_CAP_FLOWTABLE_NIC_RX(dev->mdev,
+ max_modify_header_actions) ||
+ MLX5_CAP_FLOWTABLE_NIC_TX(dev->mdev,
+ max_modify_header_actions) ||
+ MLX5_CAP_FLOWTABLE_RDMA_TX(dev->mdev,
+ max_modify_header_actions);
+}
+
+static int UVERBS_HANDLER(MLX5_IB_METHOD_FLOW_ACTION_CREATE_MODIFY_HEADER)(
+ struct uverbs_attr_bundle *attrs)
+{
+ struct ib_uobject *uobj = uverbs_attr_get_uobject(
+ attrs, MLX5_IB_ATTR_CREATE_MODIFY_HEADER_HANDLE);
+ struct mlx5_ib_dev *mdev = mlx5_udata_to_mdev(&attrs->driver_udata);
+ enum mlx5_ib_uapi_flow_table_type ft_type;
+ struct ib_flow_action *action;
+ int num_actions;
+ void *in;
+ int ret;
+
+ if (!mlx5_ib_modify_header_supported(mdev))
+ return -EOPNOTSUPP;
+
+ in = uverbs_attr_get_alloced_ptr(attrs,
+ MLX5_IB_ATTR_CREATE_MODIFY_HEADER_ACTIONS_PRM);
+
+ num_actions = uverbs_attr_ptr_get_array_size(
+ attrs, MLX5_IB_ATTR_CREATE_MODIFY_HEADER_ACTIONS_PRM,
+ MLX5_UN_SZ_BYTES(set_add_copy_action_in_auto));
+ if (num_actions < 0)
+ return num_actions;
+
+ ret = uverbs_get_const(&ft_type, attrs,
+ MLX5_IB_ATTR_CREATE_MODIFY_HEADER_FT_TYPE);
+ if (ret)
+ return ret;
+ action = mlx5_ib_create_modify_header(mdev, ft_type, num_actions, in);
+ if (IS_ERR(action))
+ return PTR_ERR(action);
+
+ uverbs_flow_action_fill_action(action, uobj, &mdev->ib_dev,
+ IB_FLOW_ACTION_UNSPECIFIED);
+
+ return 0;
+}
+
+static bool mlx5_ib_flow_action_packet_reformat_valid(struct mlx5_ib_dev *ibdev,
+ u8 packet_reformat_type,
+ u8 ft_type)
+{
+ switch (packet_reformat_type) {
+ case MLX5_IB_UAPI_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TO_L2_TUNNEL:
+ if (ft_type == MLX5_IB_UAPI_FLOW_TABLE_TYPE_NIC_TX)
+ return MLX5_CAP_FLOWTABLE(ibdev->mdev,
+ encap_general_header);
+ break;
+ case MLX5_IB_UAPI_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TO_L3_TUNNEL:
+ if (ft_type == MLX5_IB_UAPI_FLOW_TABLE_TYPE_NIC_TX)
+ return MLX5_CAP_FLOWTABLE_NIC_TX(ibdev->mdev,
+ reformat_l2_to_l3_tunnel);
+ break;
+ case MLX5_IB_UAPI_FLOW_ACTION_PACKET_REFORMAT_TYPE_L3_TUNNEL_TO_L2:
+ if (ft_type == MLX5_IB_UAPI_FLOW_TABLE_TYPE_NIC_RX)
+ return MLX5_CAP_FLOWTABLE_NIC_RX(ibdev->mdev,
+ reformat_l3_tunnel_to_l2);
+ break;
+ case MLX5_IB_UAPI_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TUNNEL_TO_L2:
+ if (ft_type == MLX5_IB_UAPI_FLOW_TABLE_TYPE_NIC_RX)
+ return MLX5_CAP_FLOWTABLE_NIC_RX(ibdev->mdev, decap);
+ break;
+ default:
+ break;
+ }
+
+ return false;
+}
+
+static int mlx5_ib_dv_to_prm_packet_reforamt_type(u8 dv_prt, u8 *prm_prt)
+{
+ switch (dv_prt) {
+ case MLX5_IB_UAPI_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TO_L2_TUNNEL:
+ *prm_prt = MLX5_REFORMAT_TYPE_L2_TO_L2_TUNNEL;
+ break;
+ case MLX5_IB_UAPI_FLOW_ACTION_PACKET_REFORMAT_TYPE_L3_TUNNEL_TO_L2:
+ *prm_prt = MLX5_REFORMAT_TYPE_L3_TUNNEL_TO_L2;
+ break;
+ case MLX5_IB_UAPI_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TO_L3_TUNNEL:
+ *prm_prt = MLX5_REFORMAT_TYPE_L2_TO_L3_TUNNEL;
+ break;
+ default:
+ return -EINVAL;
+ }
+
+ return 0;
+}
+
+static int mlx5_ib_flow_action_create_packet_reformat_ctx(
+ struct mlx5_ib_dev *dev,
+ struct mlx5_ib_flow_action *maction,
+ u8 ft_type, u8 dv_prt,
+ void *in, size_t len)
+{
+ enum mlx5_flow_namespace_type namespace;
+ u8 prm_prt;
+ int ret;
+
+ ret = mlx5_ib_ft_type_to_namespace(ft_type, &namespace);
+ if (ret)
+ return ret;
+
+ ret = mlx5_ib_dv_to_prm_packet_reforamt_type(dv_prt, &prm_prt);
+ if (ret)
+ return ret;
+
+ maction->flow_action_raw.pkt_reformat =
+ mlx5_packet_reformat_alloc(dev->mdev, prm_prt, len,
+ in, namespace);
+ if (IS_ERR(maction->flow_action_raw.pkt_reformat)) {
+ ret = PTR_ERR(maction->flow_action_raw.pkt_reformat);
+ return ret;
+ }
+
+ maction->flow_action_raw.sub_type =
+ MLX5_IB_FLOW_ACTION_PACKET_REFORMAT;
+ maction->flow_action_raw.dev = dev;
+
+ return 0;
+}
+
+static int UVERBS_HANDLER(MLX5_IB_METHOD_FLOW_ACTION_CREATE_PACKET_REFORMAT)(
+ struct uverbs_attr_bundle *attrs)
+{
+ struct ib_uobject *uobj = uverbs_attr_get_uobject(attrs,
+ MLX5_IB_ATTR_CREATE_PACKET_REFORMAT_HANDLE);
+ struct mlx5_ib_dev *mdev = mlx5_udata_to_mdev(&attrs->driver_udata);
+ enum mlx5_ib_uapi_flow_action_packet_reformat_type dv_prt;
+ enum mlx5_ib_uapi_flow_table_type ft_type;
+ struct mlx5_ib_flow_action *maction;
+ int ret;
+
+ ret = uverbs_get_const(&ft_type, attrs,
+ MLX5_IB_ATTR_CREATE_PACKET_REFORMAT_FT_TYPE);
+ if (ret)
+ return ret;
+
+ ret = uverbs_get_const(&dv_prt, attrs,
+ MLX5_IB_ATTR_CREATE_PACKET_REFORMAT_TYPE);
+ if (ret)
+ return ret;
+
+ if (!mlx5_ib_flow_action_packet_reformat_valid(mdev, dv_prt, ft_type))
+ return -EOPNOTSUPP;
+
+ maction = kzalloc(sizeof(*maction), GFP_KERNEL);
+ if (!maction)
+ return -ENOMEM;
+
+ if (dv_prt ==
+ MLX5_IB_UAPI_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TUNNEL_TO_L2) {
+ maction->flow_action_raw.sub_type =
+ MLX5_IB_FLOW_ACTION_DECAP;
+ maction->flow_action_raw.dev = mdev;
+ } else {
+ void *in;
+ int len;
+
+ in = uverbs_attr_get_alloced_ptr(attrs,
+ MLX5_IB_ATTR_CREATE_PACKET_REFORMAT_DATA_BUF);
+ if (IS_ERR(in)) {
+ ret = PTR_ERR(in);
+ goto free_maction;
+ }
+
+ len = uverbs_attr_get_len(attrs,
+ MLX5_IB_ATTR_CREATE_PACKET_REFORMAT_DATA_BUF);
+
+ ret = mlx5_ib_flow_action_create_packet_reformat_ctx(mdev,
+ maction, ft_type, dv_prt, in, len);
+ if (ret)
+ goto free_maction;
+ }
+
+ uverbs_flow_action_fill_action(&maction->ib_action, uobj, &mdev->ib_dev,
+ IB_FLOW_ACTION_UNSPECIFIED);
+ return 0;
+
+free_maction:
+ kfree(maction);
+ return ret;
+}
+
+DECLARE_UVERBS_NAMED_METHOD(
+ MLX5_IB_METHOD_CREATE_FLOW,
+ UVERBS_ATTR_IDR(MLX5_IB_ATTR_CREATE_FLOW_HANDLE,
+ UVERBS_OBJECT_FLOW,
+ UVERBS_ACCESS_NEW,
+ UA_MANDATORY),
+ UVERBS_ATTR_PTR_IN(
+ MLX5_IB_ATTR_CREATE_FLOW_MATCH_VALUE,
+ UVERBS_ATTR_SIZE(1, sizeof(struct mlx5_ib_match_params)),
+ UA_MANDATORY,
+ UA_ALLOC_AND_COPY),
+ UVERBS_ATTR_IDR(MLX5_IB_ATTR_CREATE_FLOW_MATCHER,
+ MLX5_IB_OBJECT_FLOW_MATCHER,
+ UVERBS_ACCESS_READ,
+ UA_MANDATORY),
+ UVERBS_ATTR_IDR(MLX5_IB_ATTR_CREATE_FLOW_DEST_QP,
+ UVERBS_OBJECT_QP,
+ UVERBS_ACCESS_READ),
+ UVERBS_ATTR_IDR(MLX5_IB_ATTR_CREATE_FLOW_DEST_DEVX,
+ MLX5_IB_OBJECT_DEVX_OBJ,
+ UVERBS_ACCESS_READ),
+ UVERBS_ATTR_IDRS_ARR(MLX5_IB_ATTR_CREATE_FLOW_ARR_FLOW_ACTIONS,
+ UVERBS_OBJECT_FLOW_ACTION,
+ UVERBS_ACCESS_READ, 1,
+ MLX5_IB_CREATE_FLOW_MAX_FLOW_ACTIONS,
+ UA_OPTIONAL),
+ UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_CREATE_FLOW_TAG,
+ UVERBS_ATTR_TYPE(u32),
+ UA_OPTIONAL),
+ UVERBS_ATTR_IDRS_ARR(MLX5_IB_ATTR_CREATE_FLOW_ARR_COUNTERS_DEVX,
+ MLX5_IB_OBJECT_DEVX_OBJ,
+ UVERBS_ACCESS_READ, 1, 1,
+ UA_OPTIONAL),
+ UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_CREATE_FLOW_ARR_COUNTERS_DEVX_OFFSET,
+ UVERBS_ATTR_MIN_SIZE(sizeof(u32)),
+ UA_OPTIONAL,
+ UA_ALLOC_AND_COPY),
+ UVERBS_ATTR_FLAGS_IN(MLX5_IB_ATTR_CREATE_FLOW_FLAGS,
+ enum mlx5_ib_create_flow_flags,
+ UA_OPTIONAL));
+
+DECLARE_UVERBS_NAMED_METHOD_DESTROY(
+ MLX5_IB_METHOD_DESTROY_FLOW,
+ UVERBS_ATTR_IDR(MLX5_IB_ATTR_CREATE_FLOW_HANDLE,
+ UVERBS_OBJECT_FLOW,
+ UVERBS_ACCESS_DESTROY,
+ UA_MANDATORY));
+
+ADD_UVERBS_METHODS(mlx5_ib_fs,
+ UVERBS_OBJECT_FLOW,
+ &UVERBS_METHOD(MLX5_IB_METHOD_CREATE_FLOW),
+ &UVERBS_METHOD(MLX5_IB_METHOD_DESTROY_FLOW));
+
+DECLARE_UVERBS_NAMED_METHOD(
+ MLX5_IB_METHOD_FLOW_ACTION_CREATE_MODIFY_HEADER,
+ UVERBS_ATTR_IDR(MLX5_IB_ATTR_CREATE_MODIFY_HEADER_HANDLE,
+ UVERBS_OBJECT_FLOW_ACTION,
+ UVERBS_ACCESS_NEW,
+ UA_MANDATORY),
+ UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_CREATE_MODIFY_HEADER_ACTIONS_PRM,
+ UVERBS_ATTR_MIN_SIZE(MLX5_UN_SZ_BYTES(
+ set_add_copy_action_in_auto)),
+ UA_MANDATORY,
+ UA_ALLOC_AND_COPY),
+ UVERBS_ATTR_CONST_IN(MLX5_IB_ATTR_CREATE_MODIFY_HEADER_FT_TYPE,
+ enum mlx5_ib_uapi_flow_table_type,
+ UA_MANDATORY));
+
+DECLARE_UVERBS_NAMED_METHOD(
+ MLX5_IB_METHOD_FLOW_ACTION_CREATE_PACKET_REFORMAT,
+ UVERBS_ATTR_IDR(MLX5_IB_ATTR_CREATE_PACKET_REFORMAT_HANDLE,
+ UVERBS_OBJECT_FLOW_ACTION,
+ UVERBS_ACCESS_NEW,
+ UA_MANDATORY),
+ UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_CREATE_PACKET_REFORMAT_DATA_BUF,
+ UVERBS_ATTR_MIN_SIZE(1),
+ UA_ALLOC_AND_COPY,
+ UA_OPTIONAL),
+ UVERBS_ATTR_CONST_IN(MLX5_IB_ATTR_CREATE_PACKET_REFORMAT_TYPE,
+ enum mlx5_ib_uapi_flow_action_packet_reformat_type,
+ UA_MANDATORY),
+ UVERBS_ATTR_CONST_IN(MLX5_IB_ATTR_CREATE_PACKET_REFORMAT_FT_TYPE,
+ enum mlx5_ib_uapi_flow_table_type,
+ UA_MANDATORY));
+
+ADD_UVERBS_METHODS(
+ mlx5_ib_flow_actions,
+ UVERBS_OBJECT_FLOW_ACTION,
+ &UVERBS_METHOD(MLX5_IB_METHOD_FLOW_ACTION_CREATE_MODIFY_HEADER),
+ &UVERBS_METHOD(MLX5_IB_METHOD_FLOW_ACTION_CREATE_PACKET_REFORMAT));
+
+DECLARE_UVERBS_NAMED_METHOD(
+ MLX5_IB_METHOD_FLOW_MATCHER_CREATE,
+ UVERBS_ATTR_IDR(MLX5_IB_ATTR_FLOW_MATCHER_CREATE_HANDLE,
+ MLX5_IB_OBJECT_FLOW_MATCHER,
+ UVERBS_ACCESS_NEW,
+ UA_MANDATORY),
+ UVERBS_ATTR_PTR_IN(
+ MLX5_IB_ATTR_FLOW_MATCHER_MATCH_MASK,
+ UVERBS_ATTR_SIZE(1, sizeof(struct mlx5_ib_match_params)),
+ UA_MANDATORY),
+ UVERBS_ATTR_ENUM_IN(MLX5_IB_ATTR_FLOW_MATCHER_FLOW_TYPE,
+ mlx5_ib_flow_type,
+ UA_MANDATORY),
+ UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_FLOW_MATCHER_MATCH_CRITERIA,
+ UVERBS_ATTR_TYPE(u8),
+ UA_MANDATORY),
+ UVERBS_ATTR_FLAGS_IN(MLX5_IB_ATTR_FLOW_MATCHER_FLOW_FLAGS,
+ enum ib_flow_flags,
+ UA_OPTIONAL),
+ UVERBS_ATTR_CONST_IN(MLX5_IB_ATTR_FLOW_MATCHER_FT_TYPE,
+ enum mlx5_ib_uapi_flow_table_type,
+ UA_OPTIONAL));
+
+DECLARE_UVERBS_NAMED_METHOD_DESTROY(
+ MLX5_IB_METHOD_FLOW_MATCHER_DESTROY,
+ UVERBS_ATTR_IDR(MLX5_IB_ATTR_FLOW_MATCHER_DESTROY_HANDLE,
+ MLX5_IB_OBJECT_FLOW_MATCHER,
+ UVERBS_ACCESS_DESTROY,
+ UA_MANDATORY));
+
+DECLARE_UVERBS_NAMED_OBJECT(MLX5_IB_OBJECT_FLOW_MATCHER,
+ UVERBS_TYPE_ALLOC_IDR(flow_matcher_cleanup),
+ &UVERBS_METHOD(MLX5_IB_METHOD_FLOW_MATCHER_CREATE),
+ &UVERBS_METHOD(MLX5_IB_METHOD_FLOW_MATCHER_DESTROY));
+
+const struct uapi_definition mlx5_ib_flow_defs[] = {
+ UAPI_DEF_CHAIN_OBJ_TREE_NAMED(
+ MLX5_IB_OBJECT_FLOW_MATCHER),
+ UAPI_DEF_CHAIN_OBJ_TREE(
+ UVERBS_OBJECT_FLOW,
+ &mlx5_ib_fs),
+ UAPI_DEF_CHAIN_OBJ_TREE(UVERBS_OBJECT_FLOW_ACTION,
+ &mlx5_ib_flow_actions),
+ {},
+};
+
+static const struct ib_device_ops flow_ops = {
+ .create_flow = mlx5_ib_create_flow,
+ .destroy_flow = mlx5_ib_destroy_flow,
+ .destroy_flow_action = mlx5_ib_destroy_flow_action,
+};
+
+static const struct ib_device_ops flow_ipsec_ops = {
+ .create_flow_action_esp = mlx5_ib_create_flow_action_esp,
+ .modify_flow_action_esp = mlx5_ib_modify_flow_action_esp,
+};
+
+int mlx5_ib_fs_init(struct mlx5_ib_dev *dev)
+{
+ dev->flow_db = kzalloc(sizeof(*dev->flow_db), GFP_KERNEL);
+
+ if (!dev->flow_db)
+ return -ENOMEM;
+
+ mutex_init(&dev->flow_db->lock);
+
+ ib_set_device_ops(&dev->ib_dev, &flow_ops);
+ if (mlx5_accel_ipsec_device_caps(dev->mdev) &
+ MLX5_ACCEL_IPSEC_CAP_DEVICE)
+ ib_set_device_ops(&dev->ib_dev, &flow_ipsec_ops);
+
+ return 0;
+}
diff --git a/drivers/infiniband/hw/mlx5/fs.h b/drivers/infiniband/hw/mlx5/fs.h
new file mode 100644
index 0000000..ad320ad
--- /dev/null
+++ b/drivers/infiniband/hw/mlx5/fs.h
@@ -0,0 +1,29 @@
+/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */
+/*
+ * Copyright (c) 2013-2020, Mellanox Technologies inc. All rights reserved.
+ */
+
+#ifndef _MLX5_IB_FS_H
+#define _MLX5_IB_FS_H
+
+#include "mlx5_ib.h"
+
+#if IS_ENABLED(CONFIG_INFINIBAND_USER_ACCESS)
+int mlx5_ib_fs_init(struct mlx5_ib_dev *dev);
+#else
+static inline int mlx5_ib_fs_init(struct mlx5_ib_dev *dev)
+{
+ dev->flow_db = kzalloc(sizeof(*dev->flow_db), GFP_KERNEL);
+
+ if (!dev->flow_db)
+ return -ENOMEM;
+
+ mutex_init(&dev->flow_db->lock);
+ return 0;
+}
+#endif
+static inline void mlx5_ib_fs_cleanup(struct mlx5_ib_dev *dev)
+{
+ kfree(dev->flow_db);
+}
+#endif /* _MLX5_IB_FS_H */
diff --git a/drivers/infiniband/hw/mlx5/main.c b/drivers/infiniband/hw/mlx5/main.c
index 6f99ed0..fbc45a5 100644
--- a/drivers/infiniband/hw/mlx5/main.c
+++ b/drivers/infiniband/hw/mlx5/main.c
@@ -1,33 +1,6 @@
+// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
/*
- * Copyright (c) 2013-2015, Mellanox Technologies. All rights reserved.
- *
- * This software is available to you under a choice of one of two
- * licenses. You may choose to be licensed under the terms of the GNU
- * General Public License (GPL) Version 2, available from the file
- * COPYING in the main directory of this source tree, or the
- * OpenIB.org BSD license below:
- *
- * Redistribution and use in source and binary forms, with or
- * without modification, are permitted provided that the following
- * conditions are met:
- *
- * - Redistributions of source code must retain the above
- * copyright notice, this list of conditions and the following
- * disclaimer.
- *
- * - Redistributions in binary form must reproduce the above
- * copyright notice, this list of conditions and the following
- * disclaimer in the documentation and/or other materials
- * provided with the distribution.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
- * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
- * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
- * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
- * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
- * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
- * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
+ * Copyright (c) 2013-2020, Mellanox Technologies inc. All rights reserved.
*/
#include <linux/debugfs.h>
@@ -59,10 +32,13 @@
#include "mlx5_ib.h"
#include "ib_rep.h"
#include "cmd.h"
+#include "devx.h"
+#include "fs.h"
#include "srq.h"
#include "qp.h"
#include "wr.h"
-#include <linux/mlx5/fs_helpers.h>
+#include "restrack.h"
+#include "counters.h"
#include <linux/mlx5/accel.h>
#include <rdma/uverbs_std_types.h>
#include <rdma/mlx5_user_ioctl_verbs.h>
@@ -311,9 +287,6 @@
*native_port_num = 1;
port = &ibdev->port[ib_port_num - 1];
- if (!port)
- return NULL;
-
spin_lock(&port->mp.mpi_lock);
mpi = ibdev->port[ib_port_num - 1].mp.mpi;
if (mpi && !mpi->unaffiliate) {
@@ -1765,6 +1738,92 @@
mlx5_ib_disable_lb(dev, true, false);
}
+static int set_ucontext_resp(struct ib_ucontext *uctx,
+ struct mlx5_ib_alloc_ucontext_resp *resp)
+{
+ struct ib_device *ibdev = uctx->device;
+ struct mlx5_ib_dev *dev = to_mdev(ibdev);
+ struct mlx5_ib_ucontext *context = to_mucontext(uctx);
+ struct mlx5_bfreg_info *bfregi = &context->bfregi;
+ int err;
+
+ if (MLX5_CAP_GEN(dev->mdev, dump_fill_mkey)) {
+ err = mlx5_cmd_dump_fill_mkey(dev->mdev,
+ &resp->dump_fill_mkey);
+ if (err)
+ return err;
+ resp->comp_mask |=
+ MLX5_IB_ALLOC_UCONTEXT_RESP_MASK_DUMP_FILL_MKEY;
+ }
+
+ resp->qp_tab_size = 1 << MLX5_CAP_GEN(dev->mdev, log_max_qp);
+ if (dev->wc_support)
+ resp->bf_reg_size = 1 << MLX5_CAP_GEN(dev->mdev,
+ log_bf_reg_size);
+ resp->cache_line_size = cache_line_size();
+ resp->max_sq_desc_sz = MLX5_CAP_GEN(dev->mdev, max_wqe_sz_sq);
+ resp->max_rq_desc_sz = MLX5_CAP_GEN(dev->mdev, max_wqe_sz_rq);
+ resp->max_send_wqebb = 1 << MLX5_CAP_GEN(dev->mdev, log_max_qp_sz);
+ resp->max_recv_wr = 1 << MLX5_CAP_GEN(dev->mdev, log_max_qp_sz);
+ resp->max_srq_recv_wr = 1 << MLX5_CAP_GEN(dev->mdev, log_max_srq_sz);
+ resp->cqe_version = context->cqe_version;
+ resp->log_uar_size = MLX5_CAP_GEN(dev->mdev, uar_4k) ?
+ MLX5_ADAPTER_PAGE_SHIFT : PAGE_SHIFT;
+ resp->num_uars_per_page = MLX5_CAP_GEN(dev->mdev, uar_4k) ?
+ MLX5_CAP_GEN(dev->mdev,
+ num_of_uars_per_page) : 1;
+
+ if (mlx5_accel_ipsec_device_caps(dev->mdev) &
+ MLX5_ACCEL_IPSEC_CAP_DEVICE) {
+ if (mlx5_get_flow_namespace(dev->mdev,
+ MLX5_FLOW_NAMESPACE_EGRESS))
+ resp->flow_action_flags |= MLX5_USER_ALLOC_UCONTEXT_FLOW_ACTION_FLAGS_ESP_AES_GCM;
+ if (mlx5_accel_ipsec_device_caps(dev->mdev) &
+ MLX5_ACCEL_IPSEC_CAP_REQUIRED_METADATA)
+ resp->flow_action_flags |= MLX5_USER_ALLOC_UCONTEXT_FLOW_ACTION_FLAGS_ESP_AES_GCM_REQ_METADATA;
+ if (MLX5_CAP_FLOWTABLE(dev->mdev, flow_table_properties_nic_receive.ft_field_support.outer_esp_spi))
+ resp->flow_action_flags |= MLX5_USER_ALLOC_UCONTEXT_FLOW_ACTION_FLAGS_ESP_AES_GCM_SPI_STEERING;
+ if (mlx5_accel_ipsec_device_caps(dev->mdev) &
+ MLX5_ACCEL_IPSEC_CAP_TX_IV_IS_ESN)
+ resp->flow_action_flags |= MLX5_USER_ALLOC_UCONTEXT_FLOW_ACTION_FLAGS_ESP_AES_GCM_TX_IV_IS_ESN;
+ /* MLX5_USER_ALLOC_UCONTEXT_FLOW_ACTION_FLAGS_ESP_AES_GCM_FULL_OFFLOAD is currently always 0 */
+ }
+
+ resp->tot_bfregs = bfregi->lib_uar_dyn ? 0 :
+ bfregi->total_num_bfregs - bfregi->num_dyn_bfregs;
+ resp->num_ports = dev->num_ports;
+ resp->cmds_supp_uhw |= MLX5_USER_CMDS_SUPP_UHW_QUERY_DEVICE |
+ MLX5_USER_CMDS_SUPP_UHW_CREATE_AH;
+
+ if (mlx5_ib_port_link_layer(ibdev, 1) == IB_LINK_LAYER_ETHERNET) {
+ mlx5_query_min_inline(dev->mdev, &resp->eth_min_inline);
+ resp->eth_min_inline++;
+ }
+
+ if (dev->mdev->clock_info)
+ resp->clock_info_versions = BIT(MLX5_IB_CLOCK_INFO_V1);
+
+ /*
+ * We don't want to expose information from the PCI bar that is located
+ * after 4096 bytes, so if the arch only supports larger pages, let's
+ * pretend we don't support reading the HCA's core clock. This is also
+ * forced by mmap function.
+ */
+ if (PAGE_SIZE <= 4096) {
+ resp->comp_mask |=
+ MLX5_IB_ALLOC_UCONTEXT_RESP_MASK_CORE_CLOCK_OFFSET;
+ resp->hca_core_clock_offset =
+ offsetof(struct mlx5_init_seg,
+ internal_timer_h) % PAGE_SIZE;
+ }
+
+ if (MLX5_CAP_GEN(dev->mdev, ece_support))
+ resp->comp_mask |= MLX5_IB_ALLOC_UCONTEXT_RESP_MASK_ECE;
+
+ resp->num_dyn_bfregs = bfregi->num_dyn_bfregs;
+ return 0;
+}
+
static int mlx5_ib_alloc_ucontext(struct ib_ucontext *uctx,
struct ib_udata *udata)
{
@@ -1772,14 +1831,12 @@
struct mlx5_ib_dev *dev = to_mdev(ibdev);
struct mlx5_ib_alloc_ucontext_req_v2 req = {};
struct mlx5_ib_alloc_ucontext_resp resp = {};
- struct mlx5_core_dev *mdev = dev->mdev;
struct mlx5_ib_ucontext *context = to_mucontext(uctx);
struct mlx5_bfreg_info *bfregi;
int ver;
int err;
size_t min_req_v2 = offsetof(struct mlx5_ib_alloc_ucontext_req_v2,
max_cqe_version);
- u32 dump_fill_mkey;
bool lib_uar_4k;
bool lib_uar_dyn;
@@ -1808,37 +1865,6 @@
if (req.num_low_latency_bfregs > req.total_num_bfregs - 1)
return -EINVAL;
- resp.qp_tab_size = 1 << MLX5_CAP_GEN(dev->mdev, log_max_qp);
- if (dev->wc_support)
- resp.bf_reg_size = 1 << MLX5_CAP_GEN(dev->mdev, log_bf_reg_size);
- resp.cache_line_size = cache_line_size();
- resp.max_sq_desc_sz = MLX5_CAP_GEN(dev->mdev, max_wqe_sz_sq);
- resp.max_rq_desc_sz = MLX5_CAP_GEN(dev->mdev, max_wqe_sz_rq);
- resp.max_send_wqebb = 1 << MLX5_CAP_GEN(dev->mdev, log_max_qp_sz);
- resp.max_recv_wr = 1 << MLX5_CAP_GEN(dev->mdev, log_max_qp_sz);
- resp.max_srq_recv_wr = 1 << MLX5_CAP_GEN(dev->mdev, log_max_srq_sz);
- resp.cqe_version = min_t(__u8,
- (__u8)MLX5_CAP_GEN(dev->mdev, cqe_version),
- req.max_cqe_version);
- resp.log_uar_size = MLX5_CAP_GEN(dev->mdev, uar_4k) ?
- MLX5_ADAPTER_PAGE_SHIFT : PAGE_SHIFT;
- resp.num_uars_per_page = MLX5_CAP_GEN(dev->mdev, uar_4k) ?
- MLX5_CAP_GEN(dev->mdev, num_of_uars_per_page) : 1;
- resp.response_length = min(offsetof(typeof(resp), response_length) +
- sizeof(resp.response_length), udata->outlen);
-
- if (mlx5_accel_ipsec_device_caps(dev->mdev) & MLX5_ACCEL_IPSEC_CAP_DEVICE) {
- if (mlx5_get_flow_namespace(dev->mdev, MLX5_FLOW_NAMESPACE_EGRESS))
- resp.flow_action_flags |= MLX5_USER_ALLOC_UCONTEXT_FLOW_ACTION_FLAGS_ESP_AES_GCM;
- if (mlx5_accel_ipsec_device_caps(dev->mdev) & MLX5_ACCEL_IPSEC_CAP_REQUIRED_METADATA)
- resp.flow_action_flags |= MLX5_USER_ALLOC_UCONTEXT_FLOW_ACTION_FLAGS_ESP_AES_GCM_REQ_METADATA;
- if (MLX5_CAP_FLOWTABLE(dev->mdev, flow_table_properties_nic_receive.ft_field_support.outer_esp_spi))
- resp.flow_action_flags |= MLX5_USER_ALLOC_UCONTEXT_FLOW_ACTION_FLAGS_ESP_AES_GCM_SPI_STEERING;
- if (mlx5_accel_ipsec_device_caps(dev->mdev) & MLX5_ACCEL_IPSEC_CAP_TX_IV_IS_ESN)
- resp.flow_action_flags |= MLX5_USER_ALLOC_UCONTEXT_FLOW_ACTION_FLAGS_ESP_AES_GCM_TX_IV_IS_ESN;
- /* MLX5_USER_ALLOC_UCONTEXT_FLOW_ACTION_FLAGS_ESP_AES_GCM_FULL_OFFLOAD is currently always 0 */
- }
-
lib_uar_4k = req.lib_caps & MLX5_LIB_CAP_4K_UAR;
lib_uar_dyn = req.lib_caps & MLX5_LIB_CAP_DYN_UAR;
bfregi = &context->bfregi;
@@ -1887,87 +1913,24 @@
if (err)
goto out_devx;
- if (MLX5_CAP_GEN(dev->mdev, dump_fill_mkey)) {
- err = mlx5_cmd_dump_fill_mkey(dev->mdev, &dump_fill_mkey);
- if (err)
- goto out_mdev;
- }
-
INIT_LIST_HEAD(&context->db_page_list);
mutex_init(&context->db_page_mutex);
- resp.tot_bfregs = lib_uar_dyn ? 0 : req.total_num_bfregs;
- resp.num_ports = dev->num_ports;
+ context->cqe_version = min_t(__u8,
+ (__u8)MLX5_CAP_GEN(dev->mdev, cqe_version),
+ req.max_cqe_version);
- if (offsetofend(typeof(resp), cqe_version) <= udata->outlen)
- resp.response_length += sizeof(resp.cqe_version);
+ err = set_ucontext_resp(uctx, &resp);
+ if (err)
+ goto out_mdev;
- if (offsetofend(typeof(resp), cmds_supp_uhw) <= udata->outlen) {
- resp.cmds_supp_uhw |= MLX5_USER_CMDS_SUPP_UHW_QUERY_DEVICE |
- MLX5_USER_CMDS_SUPP_UHW_CREATE_AH;
- resp.response_length += sizeof(resp.cmds_supp_uhw);
- }
-
- if (offsetofend(typeof(resp), eth_min_inline) <= udata->outlen) {
- if (mlx5_ib_port_link_layer(ibdev, 1) == IB_LINK_LAYER_ETHERNET) {
- mlx5_query_min_inline(dev->mdev, &resp.eth_min_inline);
- resp.eth_min_inline++;
- }
- resp.response_length += sizeof(resp.eth_min_inline);
- }
-
- if (offsetofend(typeof(resp), clock_info_versions) <= udata->outlen) {
- if (mdev->clock_info)
- resp.clock_info_versions = BIT(MLX5_IB_CLOCK_INFO_V1);
- resp.response_length += sizeof(resp.clock_info_versions);
- }
-
- /*
- * We don't want to expose information from the PCI bar that is located
- * after 4096 bytes, so if the arch only supports larger pages, let's
- * pretend we don't support reading the HCA's core clock. This is also
- * forced by mmap function.
- */
- if (offsetofend(typeof(resp), hca_core_clock_offset) <= udata->outlen) {
- if (PAGE_SIZE <= 4096) {
- resp.comp_mask |=
- MLX5_IB_ALLOC_UCONTEXT_RESP_MASK_CORE_CLOCK_OFFSET;
- resp.hca_core_clock_offset =
- offsetof(struct mlx5_init_seg, internal_timer_h) % PAGE_SIZE;
- }
- resp.response_length += sizeof(resp.hca_core_clock_offset);
- }
-
- if (offsetofend(typeof(resp), log_uar_size) <= udata->outlen)
- resp.response_length += sizeof(resp.log_uar_size);
-
- if (offsetofend(typeof(resp), num_uars_per_page) <= udata->outlen)
- resp.response_length += sizeof(resp.num_uars_per_page);
-
- if (offsetofend(typeof(resp), num_dyn_bfregs) <= udata->outlen) {
- resp.num_dyn_bfregs = bfregi->num_dyn_bfregs;
- resp.response_length += sizeof(resp.num_dyn_bfregs);
- }
-
- if (offsetofend(typeof(resp), dump_fill_mkey) <= udata->outlen) {
- if (MLX5_CAP_GEN(dev->mdev, dump_fill_mkey)) {
- resp.dump_fill_mkey = dump_fill_mkey;
- resp.comp_mask |=
- MLX5_IB_ALLOC_UCONTEXT_RESP_MASK_DUMP_FILL_MKEY;
- }
- resp.response_length += sizeof(resp.dump_fill_mkey);
- }
-
- if (MLX5_CAP_GEN(dev->mdev, ece_support))
- resp.comp_mask |= MLX5_IB_ALLOC_UCONTEXT_RESP_MASK_ECE;
-
+ resp.response_length = min(udata->outlen, sizeof(resp));
err = ib_copy_to_udata(udata, &resp, resp.response_length);
if (err)
goto out_mdev;
bfregi->ver = ver;
bfregi->num_low_latency_bfregs = req.num_low_latency_bfregs;
- context->cqe_version = resp.cqe_version;
context->lib_caps = req.lib_caps;
print_lib_caps(dev, context->lib_caps);
@@ -2000,6 +1963,29 @@
return err;
}
+static int mlx5_ib_query_ucontext(struct ib_ucontext *ibcontext,
+ struct uverbs_attr_bundle *attrs)
+{
+ struct mlx5_ib_alloc_ucontext_resp uctx_resp = {};
+ int ret;
+
+ ret = set_ucontext_resp(ibcontext, &uctx_resp);
+ if (ret)
+ return ret;
+
+ uctx_resp.response_length =
+ min_t(size_t,
+ uverbs_attr_get_len(attrs,
+ MLX5_IB_ATTR_QUERY_CONTEXT_RESP_UCTX),
+ sizeof(uctx_resp));
+
+ ret = uverbs_copy_to_struct_or_zero(attrs,
+ MLX5_IB_ATTR_QUERY_CONTEXT_RESP_UCTX,
+ &uctx_resp,
+ sizeof(uctx_resp));
+ return ret;
+}
+
static void mlx5_ib_dealloc_ucontext(struct ib_ucontext *ibcontext)
{
struct mlx5_ib_ucontext *context = to_mucontext(ibcontext);
@@ -2591,1820 +2577,6 @@
mlx5_cmd_dealloc_pd(mdev->mdev, mpd->pdn, mpd->uid);
}
-enum {
- MATCH_CRITERIA_ENABLE_OUTER_BIT,
- MATCH_CRITERIA_ENABLE_MISC_BIT,
- MATCH_CRITERIA_ENABLE_INNER_BIT,
- MATCH_CRITERIA_ENABLE_MISC2_BIT
-};
-
-#define HEADER_IS_ZERO(match_criteria, headers) \
- !(memchr_inv(MLX5_ADDR_OF(fte_match_param, match_criteria, headers), \
- 0, MLX5_FLD_SZ_BYTES(fte_match_param, headers))) \
-
-static u8 get_match_criteria_enable(u32 *match_criteria)
-{
- u8 match_criteria_enable;
-
- match_criteria_enable =
- (!HEADER_IS_ZERO(match_criteria, outer_headers)) <<
- MATCH_CRITERIA_ENABLE_OUTER_BIT;
- match_criteria_enable |=
- (!HEADER_IS_ZERO(match_criteria, misc_parameters)) <<
- MATCH_CRITERIA_ENABLE_MISC_BIT;
- match_criteria_enable |=
- (!HEADER_IS_ZERO(match_criteria, inner_headers)) <<
- MATCH_CRITERIA_ENABLE_INNER_BIT;
- match_criteria_enable |=
- (!HEADER_IS_ZERO(match_criteria, misc_parameters_2)) <<
- MATCH_CRITERIA_ENABLE_MISC2_BIT;
-
- return match_criteria_enable;
-}
-
-static int set_proto(void *outer_c, void *outer_v, u8 mask, u8 val)
-{
- u8 entry_mask;
- u8 entry_val;
- int err = 0;
-
- if (!mask)
- goto out;
-
- entry_mask = MLX5_GET(fte_match_set_lyr_2_4, outer_c,
- ip_protocol);
- entry_val = MLX5_GET(fte_match_set_lyr_2_4, outer_v,
- ip_protocol);
- if (!entry_mask) {
- MLX5_SET(fte_match_set_lyr_2_4, outer_c, ip_protocol, mask);
- MLX5_SET(fte_match_set_lyr_2_4, outer_v, ip_protocol, val);
- goto out;
- }
- /* Don't override existing ip protocol */
- if (mask != entry_mask || val != entry_val)
- err = -EINVAL;
-out:
- return err;
-}
-
-static void set_flow_label(void *misc_c, void *misc_v, u32 mask, u32 val,
- bool inner)
-{
- if (inner) {
- MLX5_SET(fte_match_set_misc,
- misc_c, inner_ipv6_flow_label, mask);
- MLX5_SET(fte_match_set_misc,
- misc_v, inner_ipv6_flow_label, val);
- } else {
- MLX5_SET(fte_match_set_misc,
- misc_c, outer_ipv6_flow_label, mask);
- MLX5_SET(fte_match_set_misc,
- misc_v, outer_ipv6_flow_label, val);
- }
-}
-
-static void set_tos(void *outer_c, void *outer_v, u8 mask, u8 val)
-{
- MLX5_SET(fte_match_set_lyr_2_4, outer_c, ip_ecn, mask);
- MLX5_SET(fte_match_set_lyr_2_4, outer_v, ip_ecn, val);
- MLX5_SET(fte_match_set_lyr_2_4, outer_c, ip_dscp, mask >> 2);
- MLX5_SET(fte_match_set_lyr_2_4, outer_v, ip_dscp, val >> 2);
-}
-
-static int check_mpls_supp_fields(u32 field_support, const __be32 *set_mask)
-{
- if (MLX5_GET(fte_match_mpls, set_mask, mpls_label) &&
- !(field_support & MLX5_FIELD_SUPPORT_MPLS_LABEL))
- return -EOPNOTSUPP;
-
- if (MLX5_GET(fte_match_mpls, set_mask, mpls_exp) &&
- !(field_support & MLX5_FIELD_SUPPORT_MPLS_EXP))
- return -EOPNOTSUPP;
-
- if (MLX5_GET(fte_match_mpls, set_mask, mpls_s_bos) &&
- !(field_support & MLX5_FIELD_SUPPORT_MPLS_S_BOS))
- return -EOPNOTSUPP;
-
- if (MLX5_GET(fte_match_mpls, set_mask, mpls_ttl) &&
- !(field_support & MLX5_FIELD_SUPPORT_MPLS_TTL))
- return -EOPNOTSUPP;
-
- return 0;
-}
-
-#define LAST_ETH_FIELD vlan_tag
-#define LAST_IB_FIELD sl
-#define LAST_IPV4_FIELD tos
-#define LAST_IPV6_FIELD traffic_class
-#define LAST_TCP_UDP_FIELD src_port
-#define LAST_TUNNEL_FIELD tunnel_id
-#define LAST_FLOW_TAG_FIELD tag_id
-#define LAST_DROP_FIELD size
-#define LAST_COUNTERS_FIELD counters
-
-/* Field is the last supported field */
-#define FIELDS_NOT_SUPPORTED(filter, field)\
- memchr_inv((void *)&filter.field +\
- sizeof(filter.field), 0,\
- sizeof(filter) -\
- offsetof(typeof(filter), field) -\
- sizeof(filter.field))
-
-int parse_flow_flow_action(struct mlx5_ib_flow_action *maction,
- bool is_egress,
- struct mlx5_flow_act *action)
-{
-
- switch (maction->ib_action.type) {
- case IB_FLOW_ACTION_ESP:
- if (action->action & (MLX5_FLOW_CONTEXT_ACTION_ENCRYPT |
- MLX5_FLOW_CONTEXT_ACTION_DECRYPT))
- return -EINVAL;
- /* Currently only AES_GCM keymat is supported by the driver */
- action->esp_id = (uintptr_t)maction->esp_aes_gcm.ctx;
- action->action |= is_egress ?
- MLX5_FLOW_CONTEXT_ACTION_ENCRYPT :
- MLX5_FLOW_CONTEXT_ACTION_DECRYPT;
- return 0;
- case IB_FLOW_ACTION_UNSPECIFIED:
- if (maction->flow_action_raw.sub_type ==
- MLX5_IB_FLOW_ACTION_MODIFY_HEADER) {
- if (action->action & MLX5_FLOW_CONTEXT_ACTION_MOD_HDR)
- return -EINVAL;
- action->action |= MLX5_FLOW_CONTEXT_ACTION_MOD_HDR;
- action->modify_hdr =
- maction->flow_action_raw.modify_hdr;
- return 0;
- }
- if (maction->flow_action_raw.sub_type ==
- MLX5_IB_FLOW_ACTION_DECAP) {
- if (action->action & MLX5_FLOW_CONTEXT_ACTION_DECAP)
- return -EINVAL;
- action->action |= MLX5_FLOW_CONTEXT_ACTION_DECAP;
- return 0;
- }
- if (maction->flow_action_raw.sub_type ==
- MLX5_IB_FLOW_ACTION_PACKET_REFORMAT) {
- if (action->action &
- MLX5_FLOW_CONTEXT_ACTION_PACKET_REFORMAT)
- return -EINVAL;
- action->action |=
- MLX5_FLOW_CONTEXT_ACTION_PACKET_REFORMAT;
- action->pkt_reformat =
- maction->flow_action_raw.pkt_reformat;
- return 0;
- }
- /* fall through */
- default:
- return -EOPNOTSUPP;
- }
-}
-
-static int parse_flow_attr(struct mlx5_core_dev *mdev,
- struct mlx5_flow_spec *spec,
- const union ib_flow_spec *ib_spec,
- const struct ib_flow_attr *flow_attr,
- struct mlx5_flow_act *action, u32 prev_type)
-{
- struct mlx5_flow_context *flow_context = &spec->flow_context;
- u32 *match_c = spec->match_criteria;
- u32 *match_v = spec->match_value;
- void *misc_params_c = MLX5_ADDR_OF(fte_match_param, match_c,
- misc_parameters);
- void *misc_params_v = MLX5_ADDR_OF(fte_match_param, match_v,
- misc_parameters);
- void *misc_params2_c = MLX5_ADDR_OF(fte_match_param, match_c,
- misc_parameters_2);
- void *misc_params2_v = MLX5_ADDR_OF(fte_match_param, match_v,
- misc_parameters_2);
- void *headers_c;
- void *headers_v;
- int match_ipv;
- int ret;
-
- if (ib_spec->type & IB_FLOW_SPEC_INNER) {
- headers_c = MLX5_ADDR_OF(fte_match_param, match_c,
- inner_headers);
- headers_v = MLX5_ADDR_OF(fte_match_param, match_v,
- inner_headers);
- match_ipv = MLX5_CAP_FLOWTABLE_NIC_RX(mdev,
- ft_field_support.inner_ip_version);
- } else {
- headers_c = MLX5_ADDR_OF(fte_match_param, match_c,
- outer_headers);
- headers_v = MLX5_ADDR_OF(fte_match_param, match_v,
- outer_headers);
- match_ipv = MLX5_CAP_FLOWTABLE_NIC_RX(mdev,
- ft_field_support.outer_ip_version);
- }
-
- switch (ib_spec->type & ~IB_FLOW_SPEC_INNER) {
- case IB_FLOW_SPEC_ETH:
- if (FIELDS_NOT_SUPPORTED(ib_spec->eth.mask, LAST_ETH_FIELD))
- return -EOPNOTSUPP;
-
- ether_addr_copy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_c,
- dmac_47_16),
- ib_spec->eth.mask.dst_mac);
- ether_addr_copy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
- dmac_47_16),
- ib_spec->eth.val.dst_mac);
-
- ether_addr_copy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_c,
- smac_47_16),
- ib_spec->eth.mask.src_mac);
- ether_addr_copy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
- smac_47_16),
- ib_spec->eth.val.src_mac);
-
- if (ib_spec->eth.mask.vlan_tag) {
- MLX5_SET(fte_match_set_lyr_2_4, headers_c,
- cvlan_tag, 1);
- MLX5_SET(fte_match_set_lyr_2_4, headers_v,
- cvlan_tag, 1);
-
- MLX5_SET(fte_match_set_lyr_2_4, headers_c,
- first_vid, ntohs(ib_spec->eth.mask.vlan_tag));
- MLX5_SET(fte_match_set_lyr_2_4, headers_v,
- first_vid, ntohs(ib_spec->eth.val.vlan_tag));
-
- MLX5_SET(fte_match_set_lyr_2_4, headers_c,
- first_cfi,
- ntohs(ib_spec->eth.mask.vlan_tag) >> 12);
- MLX5_SET(fte_match_set_lyr_2_4, headers_v,
- first_cfi,
- ntohs(ib_spec->eth.val.vlan_tag) >> 12);
-
- MLX5_SET(fte_match_set_lyr_2_4, headers_c,
- first_prio,
- ntohs(ib_spec->eth.mask.vlan_tag) >> 13);
- MLX5_SET(fte_match_set_lyr_2_4, headers_v,
- first_prio,
- ntohs(ib_spec->eth.val.vlan_tag) >> 13);
- }
- MLX5_SET(fte_match_set_lyr_2_4, headers_c,
- ethertype, ntohs(ib_spec->eth.mask.ether_type));
- MLX5_SET(fte_match_set_lyr_2_4, headers_v,
- ethertype, ntohs(ib_spec->eth.val.ether_type));
- break;
- case IB_FLOW_SPEC_IPV4:
- if (FIELDS_NOT_SUPPORTED(ib_spec->ipv4.mask, LAST_IPV4_FIELD))
- return -EOPNOTSUPP;
-
- if (match_ipv) {
- MLX5_SET(fte_match_set_lyr_2_4, headers_c,
- ip_version, 0xf);
- MLX5_SET(fte_match_set_lyr_2_4, headers_v,
- ip_version, MLX5_FS_IPV4_VERSION);
- } else {
- MLX5_SET(fte_match_set_lyr_2_4, headers_c,
- ethertype, 0xffff);
- MLX5_SET(fte_match_set_lyr_2_4, headers_v,
- ethertype, ETH_P_IP);
- }
-
- memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_c,
- src_ipv4_src_ipv6.ipv4_layout.ipv4),
- &ib_spec->ipv4.mask.src_ip,
- sizeof(ib_spec->ipv4.mask.src_ip));
- memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
- src_ipv4_src_ipv6.ipv4_layout.ipv4),
- &ib_spec->ipv4.val.src_ip,
- sizeof(ib_spec->ipv4.val.src_ip));
- memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_c,
- dst_ipv4_dst_ipv6.ipv4_layout.ipv4),
- &ib_spec->ipv4.mask.dst_ip,
- sizeof(ib_spec->ipv4.mask.dst_ip));
- memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
- dst_ipv4_dst_ipv6.ipv4_layout.ipv4),
- &ib_spec->ipv4.val.dst_ip,
- sizeof(ib_spec->ipv4.val.dst_ip));
-
- set_tos(headers_c, headers_v,
- ib_spec->ipv4.mask.tos, ib_spec->ipv4.val.tos);
-
- if (set_proto(headers_c, headers_v,
- ib_spec->ipv4.mask.proto,
- ib_spec->ipv4.val.proto))
- return -EINVAL;
- break;
- case IB_FLOW_SPEC_IPV6:
- if (FIELDS_NOT_SUPPORTED(ib_spec->ipv6.mask, LAST_IPV6_FIELD))
- return -EOPNOTSUPP;
-
- if (match_ipv) {
- MLX5_SET(fte_match_set_lyr_2_4, headers_c,
- ip_version, 0xf);
- MLX5_SET(fte_match_set_lyr_2_4, headers_v,
- ip_version, MLX5_FS_IPV6_VERSION);
- } else {
- MLX5_SET(fte_match_set_lyr_2_4, headers_c,
- ethertype, 0xffff);
- MLX5_SET(fte_match_set_lyr_2_4, headers_v,
- ethertype, ETH_P_IPV6);
- }
-
- memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_c,
- src_ipv4_src_ipv6.ipv6_layout.ipv6),
- &ib_spec->ipv6.mask.src_ip,
- sizeof(ib_spec->ipv6.mask.src_ip));
- memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
- src_ipv4_src_ipv6.ipv6_layout.ipv6),
- &ib_spec->ipv6.val.src_ip,
- sizeof(ib_spec->ipv6.val.src_ip));
- memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_c,
- dst_ipv4_dst_ipv6.ipv6_layout.ipv6),
- &ib_spec->ipv6.mask.dst_ip,
- sizeof(ib_spec->ipv6.mask.dst_ip));
- memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
- dst_ipv4_dst_ipv6.ipv6_layout.ipv6),
- &ib_spec->ipv6.val.dst_ip,
- sizeof(ib_spec->ipv6.val.dst_ip));
-
- set_tos(headers_c, headers_v,
- ib_spec->ipv6.mask.traffic_class,
- ib_spec->ipv6.val.traffic_class);
-
- if (set_proto(headers_c, headers_v,
- ib_spec->ipv6.mask.next_hdr,
- ib_spec->ipv6.val.next_hdr))
- return -EINVAL;
-
- set_flow_label(misc_params_c, misc_params_v,
- ntohl(ib_spec->ipv6.mask.flow_label),
- ntohl(ib_spec->ipv6.val.flow_label),
- ib_spec->type & IB_FLOW_SPEC_INNER);
- break;
- case IB_FLOW_SPEC_ESP:
- if (ib_spec->esp.mask.seq)
- return -EOPNOTSUPP;
-
- MLX5_SET(fte_match_set_misc, misc_params_c, outer_esp_spi,
- ntohl(ib_spec->esp.mask.spi));
- MLX5_SET(fte_match_set_misc, misc_params_v, outer_esp_spi,
- ntohl(ib_spec->esp.val.spi));
- break;
- case IB_FLOW_SPEC_TCP:
- if (FIELDS_NOT_SUPPORTED(ib_spec->tcp_udp.mask,
- LAST_TCP_UDP_FIELD))
- return -EOPNOTSUPP;
-
- if (set_proto(headers_c, headers_v, 0xff, IPPROTO_TCP))
- return -EINVAL;
-
- MLX5_SET(fte_match_set_lyr_2_4, headers_c, tcp_sport,
- ntohs(ib_spec->tcp_udp.mask.src_port));
- MLX5_SET(fte_match_set_lyr_2_4, headers_v, tcp_sport,
- ntohs(ib_spec->tcp_udp.val.src_port));
-
- MLX5_SET(fte_match_set_lyr_2_4, headers_c, tcp_dport,
- ntohs(ib_spec->tcp_udp.mask.dst_port));
- MLX5_SET(fte_match_set_lyr_2_4, headers_v, tcp_dport,
- ntohs(ib_spec->tcp_udp.val.dst_port));
- break;
- case IB_FLOW_SPEC_UDP:
- if (FIELDS_NOT_SUPPORTED(ib_spec->tcp_udp.mask,
- LAST_TCP_UDP_FIELD))
- return -EOPNOTSUPP;
-
- if (set_proto(headers_c, headers_v, 0xff, IPPROTO_UDP))
- return -EINVAL;
-
- MLX5_SET(fte_match_set_lyr_2_4, headers_c, udp_sport,
- ntohs(ib_spec->tcp_udp.mask.src_port));
- MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_sport,
- ntohs(ib_spec->tcp_udp.val.src_port));
-
- MLX5_SET(fte_match_set_lyr_2_4, headers_c, udp_dport,
- ntohs(ib_spec->tcp_udp.mask.dst_port));
- MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport,
- ntohs(ib_spec->tcp_udp.val.dst_port));
- break;
- case IB_FLOW_SPEC_GRE:
- if (ib_spec->gre.mask.c_ks_res0_ver)
- return -EOPNOTSUPP;
-
- if (set_proto(headers_c, headers_v, 0xff, IPPROTO_GRE))
- return -EINVAL;
-
- MLX5_SET(fte_match_set_lyr_2_4, headers_c, ip_protocol,
- 0xff);
- MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol,
- IPPROTO_GRE);
-
- MLX5_SET(fte_match_set_misc, misc_params_c, gre_protocol,
- ntohs(ib_spec->gre.mask.protocol));
- MLX5_SET(fte_match_set_misc, misc_params_v, gre_protocol,
- ntohs(ib_spec->gre.val.protocol));
-
- memcpy(MLX5_ADDR_OF(fte_match_set_misc, misc_params_c,
- gre_key.nvgre.hi),
- &ib_spec->gre.mask.key,
- sizeof(ib_spec->gre.mask.key));
- memcpy(MLX5_ADDR_OF(fte_match_set_misc, misc_params_v,
- gre_key.nvgre.hi),
- &ib_spec->gre.val.key,
- sizeof(ib_spec->gre.val.key));
- break;
- case IB_FLOW_SPEC_MPLS:
- switch (prev_type) {
- case IB_FLOW_SPEC_UDP:
- if (check_mpls_supp_fields(MLX5_CAP_FLOWTABLE_NIC_RX(mdev,
- ft_field_support.outer_first_mpls_over_udp),
- &ib_spec->mpls.mask.tag))
- return -EOPNOTSUPP;
-
- memcpy(MLX5_ADDR_OF(fte_match_set_misc2, misc_params2_v,
- outer_first_mpls_over_udp),
- &ib_spec->mpls.val.tag,
- sizeof(ib_spec->mpls.val.tag));
- memcpy(MLX5_ADDR_OF(fte_match_set_misc2, misc_params2_c,
- outer_first_mpls_over_udp),
- &ib_spec->mpls.mask.tag,
- sizeof(ib_spec->mpls.mask.tag));
- break;
- case IB_FLOW_SPEC_GRE:
- if (check_mpls_supp_fields(MLX5_CAP_FLOWTABLE_NIC_RX(mdev,
- ft_field_support.outer_first_mpls_over_gre),
- &ib_spec->mpls.mask.tag))
- return -EOPNOTSUPP;
-
- memcpy(MLX5_ADDR_OF(fte_match_set_misc2, misc_params2_v,
- outer_first_mpls_over_gre),
- &ib_spec->mpls.val.tag,
- sizeof(ib_spec->mpls.val.tag));
- memcpy(MLX5_ADDR_OF(fte_match_set_misc2, misc_params2_c,
- outer_first_mpls_over_gre),
- &ib_spec->mpls.mask.tag,
- sizeof(ib_spec->mpls.mask.tag));
- break;
- default:
- if (ib_spec->type & IB_FLOW_SPEC_INNER) {
- if (check_mpls_supp_fields(MLX5_CAP_FLOWTABLE_NIC_RX(mdev,
- ft_field_support.inner_first_mpls),
- &ib_spec->mpls.mask.tag))
- return -EOPNOTSUPP;
-
- memcpy(MLX5_ADDR_OF(fte_match_set_misc2, misc_params2_v,
- inner_first_mpls),
- &ib_spec->mpls.val.tag,
- sizeof(ib_spec->mpls.val.tag));
- memcpy(MLX5_ADDR_OF(fte_match_set_misc2, misc_params2_c,
- inner_first_mpls),
- &ib_spec->mpls.mask.tag,
- sizeof(ib_spec->mpls.mask.tag));
- } else {
- if (check_mpls_supp_fields(MLX5_CAP_FLOWTABLE_NIC_RX(mdev,
- ft_field_support.outer_first_mpls),
- &ib_spec->mpls.mask.tag))
- return -EOPNOTSUPP;
-
- memcpy(MLX5_ADDR_OF(fte_match_set_misc2, misc_params2_v,
- outer_first_mpls),
- &ib_spec->mpls.val.tag,
- sizeof(ib_spec->mpls.val.tag));
- memcpy(MLX5_ADDR_OF(fte_match_set_misc2, misc_params2_c,
- outer_first_mpls),
- &ib_spec->mpls.mask.tag,
- sizeof(ib_spec->mpls.mask.tag));
- }
- }
- break;
- case IB_FLOW_SPEC_VXLAN_TUNNEL:
- if (FIELDS_NOT_SUPPORTED(ib_spec->tunnel.mask,
- LAST_TUNNEL_FIELD))
- return -EOPNOTSUPP;
-
- MLX5_SET(fte_match_set_misc, misc_params_c, vxlan_vni,
- ntohl(ib_spec->tunnel.mask.tunnel_id));
- MLX5_SET(fte_match_set_misc, misc_params_v, vxlan_vni,
- ntohl(ib_spec->tunnel.val.tunnel_id));
- break;
- case IB_FLOW_SPEC_ACTION_TAG:
- if (FIELDS_NOT_SUPPORTED(ib_spec->flow_tag,
- LAST_FLOW_TAG_FIELD))
- return -EOPNOTSUPP;
- if (ib_spec->flow_tag.tag_id >= BIT(24))
- return -EINVAL;
-
- flow_context->flow_tag = ib_spec->flow_tag.tag_id;
- flow_context->flags |= FLOW_CONTEXT_HAS_TAG;
- break;
- case IB_FLOW_SPEC_ACTION_DROP:
- if (FIELDS_NOT_SUPPORTED(ib_spec->drop,
- LAST_DROP_FIELD))
- return -EOPNOTSUPP;
- action->action |= MLX5_FLOW_CONTEXT_ACTION_DROP;
- break;
- case IB_FLOW_SPEC_ACTION_HANDLE:
- ret = parse_flow_flow_action(to_mflow_act(ib_spec->action.act),
- flow_attr->flags & IB_FLOW_ATTR_FLAGS_EGRESS, action);
- if (ret)
- return ret;
- break;
- case IB_FLOW_SPEC_ACTION_COUNT:
- if (FIELDS_NOT_SUPPORTED(ib_spec->flow_count,
- LAST_COUNTERS_FIELD))
- return -EOPNOTSUPP;
-
- /* for now support only one counters spec per flow */
- if (action->action & MLX5_FLOW_CONTEXT_ACTION_COUNT)
- return -EINVAL;
-
- action->counters = ib_spec->flow_count.counters;
- action->action |= MLX5_FLOW_CONTEXT_ACTION_COUNT;
- break;
- default:
- return -EINVAL;
- }
-
- return 0;
-}
-
-/* If a flow could catch both multicast and unicast packets,
- * it won't fall into the multicast flow steering table and this rule
- * could steal other multicast packets.
- */
-static bool flow_is_multicast_only(const struct ib_flow_attr *ib_attr)
-{
- union ib_flow_spec *flow_spec;
-
- if (ib_attr->type != IB_FLOW_ATTR_NORMAL ||
- ib_attr->num_of_specs < 1)
- return false;
-
- flow_spec = (union ib_flow_spec *)(ib_attr + 1);
- if (flow_spec->type == IB_FLOW_SPEC_IPV4) {
- struct ib_flow_spec_ipv4 *ipv4_spec;
-
- ipv4_spec = (struct ib_flow_spec_ipv4 *)flow_spec;
- if (ipv4_is_multicast(ipv4_spec->val.dst_ip))
- return true;
-
- return false;
- }
-
- if (flow_spec->type == IB_FLOW_SPEC_ETH) {
- struct ib_flow_spec_eth *eth_spec;
-
- eth_spec = (struct ib_flow_spec_eth *)flow_spec;
- return is_multicast_ether_addr(eth_spec->mask.dst_mac) &&
- is_multicast_ether_addr(eth_spec->val.dst_mac);
- }
-
- return false;
-}
-
-enum valid_spec {
- VALID_SPEC_INVALID,
- VALID_SPEC_VALID,
- VALID_SPEC_NA,
-};
-
-static enum valid_spec
-is_valid_esp_aes_gcm(struct mlx5_core_dev *mdev,
- const struct mlx5_flow_spec *spec,
- const struct mlx5_flow_act *flow_act,
- bool egress)
-{
- const u32 *match_c = spec->match_criteria;
- bool is_crypto =
- (flow_act->action & (MLX5_FLOW_CONTEXT_ACTION_ENCRYPT |
- MLX5_FLOW_CONTEXT_ACTION_DECRYPT));
- bool is_ipsec = mlx5_fs_is_ipsec_flow(match_c);
- bool is_drop = flow_act->action & MLX5_FLOW_CONTEXT_ACTION_DROP;
-
- /*
- * Currently only crypto is supported in egress, when regular egress
- * rules would be supported, always return VALID_SPEC_NA.
- */
- if (!is_crypto)
- return VALID_SPEC_NA;
-
- return is_crypto && is_ipsec &&
- (!egress || (!is_drop &&
- !(spec->flow_context.flags & FLOW_CONTEXT_HAS_TAG))) ?
- VALID_SPEC_VALID : VALID_SPEC_INVALID;
-}
-
-static bool is_valid_spec(struct mlx5_core_dev *mdev,
- const struct mlx5_flow_spec *spec,
- const struct mlx5_flow_act *flow_act,
- bool egress)
-{
- /* We curretly only support ipsec egress flow */
- return is_valid_esp_aes_gcm(mdev, spec, flow_act, egress) != VALID_SPEC_INVALID;
-}
-
-static bool is_valid_ethertype(struct mlx5_core_dev *mdev,
- const struct ib_flow_attr *flow_attr,
- bool check_inner)
-{
- union ib_flow_spec *ib_spec = (union ib_flow_spec *)(flow_attr + 1);
- int match_ipv = check_inner ?
- MLX5_CAP_FLOWTABLE_NIC_RX(mdev,
- ft_field_support.inner_ip_version) :
- MLX5_CAP_FLOWTABLE_NIC_RX(mdev,
- ft_field_support.outer_ip_version);
- int inner_bit = check_inner ? IB_FLOW_SPEC_INNER : 0;
- bool ipv4_spec_valid, ipv6_spec_valid;
- unsigned int ip_spec_type = 0;
- bool has_ethertype = false;
- unsigned int spec_index;
- bool mask_valid = true;
- u16 eth_type = 0;
- bool type_valid;
-
- /* Validate that ethertype is correct */
- for (spec_index = 0; spec_index < flow_attr->num_of_specs; spec_index++) {
- if ((ib_spec->type == (IB_FLOW_SPEC_ETH | inner_bit)) &&
- ib_spec->eth.mask.ether_type) {
- mask_valid = (ib_spec->eth.mask.ether_type ==
- htons(0xffff));
- has_ethertype = true;
- eth_type = ntohs(ib_spec->eth.val.ether_type);
- } else if ((ib_spec->type == (IB_FLOW_SPEC_IPV4 | inner_bit)) ||
- (ib_spec->type == (IB_FLOW_SPEC_IPV6 | inner_bit))) {
- ip_spec_type = ib_spec->type;
- }
- ib_spec = (void *)ib_spec + ib_spec->size;
- }
-
- type_valid = (!has_ethertype) || (!ip_spec_type);
- if (!type_valid && mask_valid) {
- ipv4_spec_valid = (eth_type == ETH_P_IP) &&
- (ip_spec_type == (IB_FLOW_SPEC_IPV4 | inner_bit));
- ipv6_spec_valid = (eth_type == ETH_P_IPV6) &&
- (ip_spec_type == (IB_FLOW_SPEC_IPV6 | inner_bit));
-
- type_valid = (ipv4_spec_valid) || (ipv6_spec_valid) ||
- (((eth_type == ETH_P_MPLS_UC) ||
- (eth_type == ETH_P_MPLS_MC)) && match_ipv);
- }
-
- return type_valid;
-}
-
-static bool is_valid_attr(struct mlx5_core_dev *mdev,
- const struct ib_flow_attr *flow_attr)
-{
- return is_valid_ethertype(mdev, flow_attr, false) &&
- is_valid_ethertype(mdev, flow_attr, true);
-}
-
-static void put_flow_table(struct mlx5_ib_dev *dev,
- struct mlx5_ib_flow_prio *prio, bool ft_added)
-{
- prio->refcount -= !!ft_added;
- if (!prio->refcount) {
- mlx5_destroy_flow_table(prio->flow_table);
- prio->flow_table = NULL;
- }
-}
-
-static void counters_clear_description(struct ib_counters *counters)
-{
- struct mlx5_ib_mcounters *mcounters = to_mcounters(counters);
-
- mutex_lock(&mcounters->mcntrs_mutex);
- kfree(mcounters->counters_data);
- mcounters->counters_data = NULL;
- mcounters->cntrs_max_index = 0;
- mutex_unlock(&mcounters->mcntrs_mutex);
-}
-
-static int mlx5_ib_destroy_flow(struct ib_flow *flow_id)
-{
- struct mlx5_ib_flow_handler *handler = container_of(flow_id,
- struct mlx5_ib_flow_handler,
- ibflow);
- struct mlx5_ib_flow_handler *iter, *tmp;
- struct mlx5_ib_dev *dev = handler->dev;
-
- mutex_lock(&dev->flow_db->lock);
-
- list_for_each_entry_safe(iter, tmp, &handler->list, list) {
- mlx5_del_flow_rules(iter->rule);
- put_flow_table(dev, iter->prio, true);
- list_del(&iter->list);
- kfree(iter);
- }
-
- mlx5_del_flow_rules(handler->rule);
- put_flow_table(dev, handler->prio, true);
- if (handler->ibcounters &&
- atomic_read(&handler->ibcounters->usecnt) == 1)
- counters_clear_description(handler->ibcounters);
-
- mutex_unlock(&dev->flow_db->lock);
- if (handler->flow_matcher)
- atomic_dec(&handler->flow_matcher->usecnt);
- kfree(handler);
-
- return 0;
-}
-
-static int ib_prio_to_core_prio(unsigned int priority, bool dont_trap)
-{
- priority *= 2;
- if (!dont_trap)
- priority++;
- return priority;
-}
-
-enum flow_table_type {
- MLX5_IB_FT_RX,
- MLX5_IB_FT_TX
-};
-
-#define MLX5_FS_MAX_TYPES 6
-#define MLX5_FS_MAX_ENTRIES BIT(16)
-
-static struct mlx5_ib_flow_prio *_get_prio(struct mlx5_flow_namespace *ns,
- struct mlx5_ib_flow_prio *prio,
- int priority,
- int num_entries, int num_groups,
- u32 flags)
-{
- struct mlx5_flow_table_attr ft_attr = {};
- struct mlx5_flow_table *ft;
-
- ft_attr.prio = priority;
- ft_attr.max_fte = num_entries;
- ft_attr.flags = flags;
- ft_attr.autogroup.max_num_groups = num_groups;
- ft = mlx5_create_auto_grouped_flow_table(ns, &ft_attr);
- if (IS_ERR(ft))
- return ERR_CAST(ft);
-
- prio->flow_table = ft;
- prio->refcount = 0;
- return prio;
-}
-
-static struct mlx5_ib_flow_prio *get_flow_table(struct mlx5_ib_dev *dev,
- struct ib_flow_attr *flow_attr,
- enum flow_table_type ft_type)
-{
- bool dont_trap = flow_attr->flags & IB_FLOW_ATTR_FLAGS_DONT_TRAP;
- struct mlx5_flow_namespace *ns = NULL;
- struct mlx5_ib_flow_prio *prio;
- struct mlx5_flow_table *ft;
- int max_table_size;
- int num_entries;
- int num_groups;
- bool esw_encap;
- u32 flags = 0;
- int priority;
-
- max_table_size = BIT(MLX5_CAP_FLOWTABLE_NIC_RX(dev->mdev,
- log_max_ft_size));
- esw_encap = mlx5_eswitch_get_encap_mode(dev->mdev) !=
- DEVLINK_ESWITCH_ENCAP_MODE_NONE;
- if (flow_attr->type == IB_FLOW_ATTR_NORMAL) {
- enum mlx5_flow_namespace_type fn_type;
-
- if (flow_is_multicast_only(flow_attr) &&
- !dont_trap)
- priority = MLX5_IB_FLOW_MCAST_PRIO;
- else
- priority = ib_prio_to_core_prio(flow_attr->priority,
- dont_trap);
- if (ft_type == MLX5_IB_FT_RX) {
- fn_type = MLX5_FLOW_NAMESPACE_BYPASS;
- prio = &dev->flow_db->prios[priority];
- if (!dev->is_rep && !esw_encap &&
- MLX5_CAP_FLOWTABLE_NIC_RX(dev->mdev, decap))
- flags |= MLX5_FLOW_TABLE_TUNNEL_EN_DECAP;
- if (!dev->is_rep && !esw_encap &&
- MLX5_CAP_FLOWTABLE_NIC_RX(dev->mdev,
- reformat_l3_tunnel_to_l2))
- flags |= MLX5_FLOW_TABLE_TUNNEL_EN_REFORMAT;
- } else {
- max_table_size =
- BIT(MLX5_CAP_FLOWTABLE_NIC_TX(dev->mdev,
- log_max_ft_size));
- fn_type = MLX5_FLOW_NAMESPACE_EGRESS;
- prio = &dev->flow_db->egress_prios[priority];
- if (!dev->is_rep && !esw_encap &&
- MLX5_CAP_FLOWTABLE_NIC_TX(dev->mdev, reformat))
- flags |= MLX5_FLOW_TABLE_TUNNEL_EN_REFORMAT;
- }
- ns = mlx5_get_flow_namespace(dev->mdev, fn_type);
- num_entries = MLX5_FS_MAX_ENTRIES;
- num_groups = MLX5_FS_MAX_TYPES;
- } else if (flow_attr->type == IB_FLOW_ATTR_ALL_DEFAULT ||
- flow_attr->type == IB_FLOW_ATTR_MC_DEFAULT) {
- ns = mlx5_get_flow_namespace(dev->mdev,
- MLX5_FLOW_NAMESPACE_LEFTOVERS);
- build_leftovers_ft_param(&priority,
- &num_entries,
- &num_groups);
- prio = &dev->flow_db->prios[MLX5_IB_FLOW_LEFTOVERS_PRIO];
- } else if (flow_attr->type == IB_FLOW_ATTR_SNIFFER) {
- if (!MLX5_CAP_FLOWTABLE(dev->mdev,
- allow_sniffer_and_nic_rx_shared_tir))
- return ERR_PTR(-ENOTSUPP);
-
- ns = mlx5_get_flow_namespace(dev->mdev, ft_type == MLX5_IB_FT_RX ?
- MLX5_FLOW_NAMESPACE_SNIFFER_RX :
- MLX5_FLOW_NAMESPACE_SNIFFER_TX);
-
- prio = &dev->flow_db->sniffer[ft_type];
- priority = 0;
- num_entries = 1;
- num_groups = 1;
- }
-
- if (!ns)
- return ERR_PTR(-ENOTSUPP);
-
- max_table_size = min_t(int, num_entries, max_table_size);
-
- ft = prio->flow_table;
- if (!ft)
- return _get_prio(ns, prio, priority, max_table_size, num_groups,
- flags);
-
- return prio;
-}
-
-static void set_underlay_qp(struct mlx5_ib_dev *dev,
- struct mlx5_flow_spec *spec,
- u32 underlay_qpn)
-{
- void *misc_params_c = MLX5_ADDR_OF(fte_match_param,
- spec->match_criteria,
- misc_parameters);
- void *misc_params_v = MLX5_ADDR_OF(fte_match_param, spec->match_value,
- misc_parameters);
-
- if (underlay_qpn &&
- MLX5_CAP_FLOWTABLE_NIC_RX(dev->mdev,
- ft_field_support.bth_dst_qp)) {
- MLX5_SET(fte_match_set_misc,
- misc_params_v, bth_dst_qp, underlay_qpn);
- MLX5_SET(fte_match_set_misc,
- misc_params_c, bth_dst_qp, 0xffffff);
- }
-}
-
-static int read_flow_counters(struct ib_device *ibdev,
- struct mlx5_read_counters_attr *read_attr)
-{
- struct mlx5_fc *fc = read_attr->hw_cntrs_hndl;
- struct mlx5_ib_dev *dev = to_mdev(ibdev);
-
- return mlx5_fc_query(dev->mdev, fc,
- &read_attr->out[IB_COUNTER_PACKETS],
- &read_attr->out[IB_COUNTER_BYTES]);
-}
-
-/* flow counters currently expose two counters packets and bytes */
-#define FLOW_COUNTERS_NUM 2
-static int counters_set_description(struct ib_counters *counters,
- enum mlx5_ib_counters_type counters_type,
- struct mlx5_ib_flow_counters_desc *desc_data,
- u32 ncounters)
-{
- struct mlx5_ib_mcounters *mcounters = to_mcounters(counters);
- u32 cntrs_max_index = 0;
- int i;
-
- if (counters_type != MLX5_IB_COUNTERS_FLOW)
- return -EINVAL;
-
- /* init the fields for the object */
- mcounters->type = counters_type;
- mcounters->read_counters = read_flow_counters;
- mcounters->counters_num = FLOW_COUNTERS_NUM;
- mcounters->ncounters = ncounters;
- /* each counter entry have both description and index pair */
- for (i = 0; i < ncounters; i++) {
- if (desc_data[i].description > IB_COUNTER_BYTES)
- return -EINVAL;
-
- if (cntrs_max_index <= desc_data[i].index)
- cntrs_max_index = desc_data[i].index + 1;
- }
-
- mutex_lock(&mcounters->mcntrs_mutex);
- mcounters->counters_data = desc_data;
- mcounters->cntrs_max_index = cntrs_max_index;
- mutex_unlock(&mcounters->mcntrs_mutex);
-
- return 0;
-}
-
-#define MAX_COUNTERS_NUM (USHRT_MAX / (sizeof(u32) * 2))
-static int flow_counters_set_data(struct ib_counters *ibcounters,
- struct mlx5_ib_create_flow *ucmd)
-{
- struct mlx5_ib_mcounters *mcounters = to_mcounters(ibcounters);
- struct mlx5_ib_flow_counters_data *cntrs_data = NULL;
- struct mlx5_ib_flow_counters_desc *desc_data = NULL;
- bool hw_hndl = false;
- int ret = 0;
-
- if (ucmd && ucmd->ncounters_data != 0) {
- cntrs_data = ucmd->data;
- if (cntrs_data->ncounters > MAX_COUNTERS_NUM)
- return -EINVAL;
-
- desc_data = kcalloc(cntrs_data->ncounters,
- sizeof(*desc_data),
- GFP_KERNEL);
- if (!desc_data)
- return -ENOMEM;
-
- if (copy_from_user(desc_data,
- u64_to_user_ptr(cntrs_data->counters_data),
- sizeof(*desc_data) * cntrs_data->ncounters)) {
- ret = -EFAULT;
- goto free;
- }
- }
-
- if (!mcounters->hw_cntrs_hndl) {
- mcounters->hw_cntrs_hndl = mlx5_fc_create(
- to_mdev(ibcounters->device)->mdev, false);
- if (IS_ERR(mcounters->hw_cntrs_hndl)) {
- ret = PTR_ERR(mcounters->hw_cntrs_hndl);
- goto free;
- }
- hw_hndl = true;
- }
-
- if (desc_data) {
- /* counters already bound to at least one flow */
- if (mcounters->cntrs_max_index) {
- ret = -EINVAL;
- goto free_hndl;
- }
-
- ret = counters_set_description(ibcounters,
- MLX5_IB_COUNTERS_FLOW,
- desc_data,
- cntrs_data->ncounters);
- if (ret)
- goto free_hndl;
-
- } else if (!mcounters->cntrs_max_index) {
- /* counters not bound yet, must have udata passed */
- ret = -EINVAL;
- goto free_hndl;
- }
-
- return 0;
-
-free_hndl:
- if (hw_hndl) {
- mlx5_fc_destroy(to_mdev(ibcounters->device)->mdev,
- mcounters->hw_cntrs_hndl);
- mcounters->hw_cntrs_hndl = NULL;
- }
-free:
- kfree(desc_data);
- return ret;
-}
-
-static void mlx5_ib_set_rule_source_port(struct mlx5_ib_dev *dev,
- struct mlx5_flow_spec *spec,
- struct mlx5_eswitch_rep *rep)
-{
- struct mlx5_eswitch *esw = dev->mdev->priv.eswitch;
- void *misc;
-
- if (mlx5_eswitch_vport_match_metadata_enabled(esw)) {
- misc = MLX5_ADDR_OF(fte_match_param, spec->match_value,
- misc_parameters_2);
-
- MLX5_SET(fte_match_set_misc2, misc, metadata_reg_c_0,
- mlx5_eswitch_get_vport_metadata_for_match(esw,
- rep->vport));
- misc = MLX5_ADDR_OF(fte_match_param, spec->match_criteria,
- misc_parameters_2);
-
- MLX5_SET(fte_match_set_misc2, misc, metadata_reg_c_0,
- mlx5_eswitch_get_vport_metadata_mask());
- } else {
- misc = MLX5_ADDR_OF(fte_match_param, spec->match_value,
- misc_parameters);
-
- MLX5_SET(fte_match_set_misc, misc, source_port, rep->vport);
-
- misc = MLX5_ADDR_OF(fte_match_param, spec->match_criteria,
- misc_parameters);
-
- MLX5_SET_TO_ONES(fte_match_set_misc, misc, source_port);
- }
-}
-
-static struct mlx5_ib_flow_handler *_create_flow_rule(struct mlx5_ib_dev *dev,
- struct mlx5_ib_flow_prio *ft_prio,
- const struct ib_flow_attr *flow_attr,
- struct mlx5_flow_destination *dst,
- u32 underlay_qpn,
- struct mlx5_ib_create_flow *ucmd)
-{
- struct mlx5_flow_table *ft = ft_prio->flow_table;
- struct mlx5_ib_flow_handler *handler;
- struct mlx5_flow_act flow_act = {};
- struct mlx5_flow_spec *spec;
- struct mlx5_flow_destination dest_arr[2] = {};
- struct mlx5_flow_destination *rule_dst = dest_arr;
- const void *ib_flow = (const void *)flow_attr + sizeof(*flow_attr);
- unsigned int spec_index;
- u32 prev_type = 0;
- int err = 0;
- int dest_num = 0;
- bool is_egress = flow_attr->flags & IB_FLOW_ATTR_FLAGS_EGRESS;
-
- if (!is_valid_attr(dev->mdev, flow_attr))
- return ERR_PTR(-EINVAL);
-
- if (dev->is_rep && is_egress)
- return ERR_PTR(-EINVAL);
-
- spec = kvzalloc(sizeof(*spec), GFP_KERNEL);
- handler = kzalloc(sizeof(*handler), GFP_KERNEL);
- if (!handler || !spec) {
- err = -ENOMEM;
- goto free;
- }
-
- INIT_LIST_HEAD(&handler->list);
-
- for (spec_index = 0; spec_index < flow_attr->num_of_specs; spec_index++) {
- err = parse_flow_attr(dev->mdev, spec,
- ib_flow, flow_attr, &flow_act,
- prev_type);
- if (err < 0)
- goto free;
-
- prev_type = ((union ib_flow_spec *)ib_flow)->type;
- ib_flow += ((union ib_flow_spec *)ib_flow)->size;
- }
-
- if (dst && !(flow_act.action & MLX5_FLOW_CONTEXT_ACTION_DROP)) {
- memcpy(&dest_arr[0], dst, sizeof(*dst));
- dest_num++;
- }
-
- if (!flow_is_multicast_only(flow_attr))
- set_underlay_qp(dev, spec, underlay_qpn);
-
- if (dev->is_rep) {
- struct mlx5_eswitch_rep *rep;
-
- rep = dev->port[flow_attr->port - 1].rep;
- if (!rep) {
- err = -EINVAL;
- goto free;
- }
-
- mlx5_ib_set_rule_source_port(dev, spec, rep);
- }
-
- spec->match_criteria_enable = get_match_criteria_enable(spec->match_criteria);
-
- if (is_egress &&
- !is_valid_spec(dev->mdev, spec, &flow_act, is_egress)) {
- err = -EINVAL;
- goto free;
- }
-
- if (flow_act.action & MLX5_FLOW_CONTEXT_ACTION_COUNT) {
- struct mlx5_ib_mcounters *mcounters;
-
- err = flow_counters_set_data(flow_act.counters, ucmd);
- if (err)
- goto free;
-
- mcounters = to_mcounters(flow_act.counters);
- handler->ibcounters = flow_act.counters;
- dest_arr[dest_num].type =
- MLX5_FLOW_DESTINATION_TYPE_COUNTER;
- dest_arr[dest_num].counter_id =
- mlx5_fc_id(mcounters->hw_cntrs_hndl);
- dest_num++;
- }
-
- if (flow_act.action & MLX5_FLOW_CONTEXT_ACTION_DROP) {
- if (!dest_num)
- rule_dst = NULL;
- } else {
- if (flow_attr->flags & IB_FLOW_ATTR_FLAGS_DONT_TRAP)
- flow_act.action |=
- MLX5_FLOW_CONTEXT_ACTION_FWD_NEXT_PRIO;
- if (is_egress)
- flow_act.action |= MLX5_FLOW_CONTEXT_ACTION_ALLOW;
- else if (dest_num)
- flow_act.action |= MLX5_FLOW_CONTEXT_ACTION_FWD_DEST;
- }
-
- if ((spec->flow_context.flags & FLOW_CONTEXT_HAS_TAG) &&
- (flow_attr->type == IB_FLOW_ATTR_ALL_DEFAULT ||
- flow_attr->type == IB_FLOW_ATTR_MC_DEFAULT)) {
- mlx5_ib_warn(dev, "Flow tag %u and attribute type %x isn't allowed in leftovers\n",
- spec->flow_context.flow_tag, flow_attr->type);
- err = -EINVAL;
- goto free;
- }
- handler->rule = mlx5_add_flow_rules(ft, spec,
- &flow_act,
- rule_dst, dest_num);
-
- if (IS_ERR(handler->rule)) {
- err = PTR_ERR(handler->rule);
- goto free;
- }
-
- ft_prio->refcount++;
- handler->prio = ft_prio;
- handler->dev = dev;
-
- ft_prio->flow_table = ft;
-free:
- if (err && handler) {
- if (handler->ibcounters &&
- atomic_read(&handler->ibcounters->usecnt) == 1)
- counters_clear_description(handler->ibcounters);
- kfree(handler);
- }
- kvfree(spec);
- return err ? ERR_PTR(err) : handler;
-}
-
-static struct mlx5_ib_flow_handler *create_flow_rule(struct mlx5_ib_dev *dev,
- struct mlx5_ib_flow_prio *ft_prio,
- const struct ib_flow_attr *flow_attr,
- struct mlx5_flow_destination *dst)
-{
- return _create_flow_rule(dev, ft_prio, flow_attr, dst, 0, NULL);
-}
-
-enum {
- LEFTOVERS_MC,
- LEFTOVERS_UC,
-};
-
-static struct mlx5_ib_flow_handler *create_leftovers_rule(struct mlx5_ib_dev *dev,
- struct mlx5_ib_flow_prio *ft_prio,
- struct ib_flow_attr *flow_attr,
- struct mlx5_flow_destination *dst)
-{
- struct mlx5_ib_flow_handler *handler_ucast = NULL;
- struct mlx5_ib_flow_handler *handler = NULL;
-
- static struct {
- struct ib_flow_attr flow_attr;
- struct ib_flow_spec_eth eth_flow;
- } leftovers_specs[] = {
- [LEFTOVERS_MC] = {
- .flow_attr = {
- .num_of_specs = 1,
- .size = sizeof(leftovers_specs[0])
- },
- .eth_flow = {
- .type = IB_FLOW_SPEC_ETH,
- .size = sizeof(struct ib_flow_spec_eth),
- .mask = {.dst_mac = {0x1} },
- .val = {.dst_mac = {0x1} }
- }
- },
- [LEFTOVERS_UC] = {
- .flow_attr = {
- .num_of_specs = 1,
- .size = sizeof(leftovers_specs[0])
- },
- .eth_flow = {
- .type = IB_FLOW_SPEC_ETH,
- .size = sizeof(struct ib_flow_spec_eth),
- .mask = {.dst_mac = {0x1} },
- .val = {.dst_mac = {} }
- }
- }
- };
-
- handler = create_flow_rule(dev, ft_prio,
- &leftovers_specs[LEFTOVERS_MC].flow_attr,
- dst);
- if (!IS_ERR(handler) &&
- flow_attr->type == IB_FLOW_ATTR_ALL_DEFAULT) {
- handler_ucast = create_flow_rule(dev, ft_prio,
- &leftovers_specs[LEFTOVERS_UC].flow_attr,
- dst);
- if (IS_ERR(handler_ucast)) {
- mlx5_del_flow_rules(handler->rule);
- ft_prio->refcount--;
- kfree(handler);
- handler = handler_ucast;
- } else {
- list_add(&handler_ucast->list, &handler->list);
- }
- }
-
- return handler;
-}
-
-static struct mlx5_ib_flow_handler *create_sniffer_rule(struct mlx5_ib_dev *dev,
- struct mlx5_ib_flow_prio *ft_rx,
- struct mlx5_ib_flow_prio *ft_tx,
- struct mlx5_flow_destination *dst)
-{
- struct mlx5_ib_flow_handler *handler_rx;
- struct mlx5_ib_flow_handler *handler_tx;
- int err;
- static const struct ib_flow_attr flow_attr = {
- .num_of_specs = 0,
- .size = sizeof(flow_attr)
- };
-
- handler_rx = create_flow_rule(dev, ft_rx, &flow_attr, dst);
- if (IS_ERR(handler_rx)) {
- err = PTR_ERR(handler_rx);
- goto err;
- }
-
- handler_tx = create_flow_rule(dev, ft_tx, &flow_attr, dst);
- if (IS_ERR(handler_tx)) {
- err = PTR_ERR(handler_tx);
- goto err_tx;
- }
-
- list_add(&handler_tx->list, &handler_rx->list);
-
- return handler_rx;
-
-err_tx:
- mlx5_del_flow_rules(handler_rx->rule);
- ft_rx->refcount--;
- kfree(handler_rx);
-err:
- return ERR_PTR(err);
-}
-
-static struct ib_flow *mlx5_ib_create_flow(struct ib_qp *qp,
- struct ib_flow_attr *flow_attr,
- int domain,
- struct ib_udata *udata)
-{
- struct mlx5_ib_dev *dev = to_mdev(qp->device);
- struct mlx5_ib_qp *mqp = to_mqp(qp);
- struct mlx5_ib_flow_handler *handler = NULL;
- struct mlx5_flow_destination *dst = NULL;
- struct mlx5_ib_flow_prio *ft_prio_tx = NULL;
- struct mlx5_ib_flow_prio *ft_prio;
- bool is_egress = flow_attr->flags & IB_FLOW_ATTR_FLAGS_EGRESS;
- struct mlx5_ib_create_flow *ucmd = NULL, ucmd_hdr;
- size_t min_ucmd_sz, required_ucmd_sz;
- int err;
- int underlay_qpn;
-
- if (udata && udata->inlen) {
- min_ucmd_sz = offsetof(typeof(ucmd_hdr), reserved) +
- sizeof(ucmd_hdr.reserved);
- if (udata->inlen < min_ucmd_sz)
- return ERR_PTR(-EOPNOTSUPP);
-
- err = ib_copy_from_udata(&ucmd_hdr, udata, min_ucmd_sz);
- if (err)
- return ERR_PTR(err);
-
- /* currently supports only one counters data */
- if (ucmd_hdr.ncounters_data > 1)
- return ERR_PTR(-EINVAL);
-
- required_ucmd_sz = min_ucmd_sz +
- sizeof(struct mlx5_ib_flow_counters_data) *
- ucmd_hdr.ncounters_data;
- if (udata->inlen > required_ucmd_sz &&
- !ib_is_udata_cleared(udata, required_ucmd_sz,
- udata->inlen - required_ucmd_sz))
- return ERR_PTR(-EOPNOTSUPP);
-
- ucmd = kzalloc(required_ucmd_sz, GFP_KERNEL);
- if (!ucmd)
- return ERR_PTR(-ENOMEM);
-
- err = ib_copy_from_udata(ucmd, udata, required_ucmd_sz);
- if (err)
- goto free_ucmd;
- }
-
- if (flow_attr->priority > MLX5_IB_FLOW_LAST_PRIO) {
- err = -ENOMEM;
- goto free_ucmd;
- }
-
- if (domain != IB_FLOW_DOMAIN_USER ||
- flow_attr->port > dev->num_ports ||
- (flow_attr->flags & ~(IB_FLOW_ATTR_FLAGS_DONT_TRAP |
- IB_FLOW_ATTR_FLAGS_EGRESS))) {
- err = -EINVAL;
- goto free_ucmd;
- }
-
- if (is_egress &&
- (flow_attr->type == IB_FLOW_ATTR_ALL_DEFAULT ||
- flow_attr->type == IB_FLOW_ATTR_MC_DEFAULT)) {
- err = -EINVAL;
- goto free_ucmd;
- }
-
- dst = kzalloc(sizeof(*dst), GFP_KERNEL);
- if (!dst) {
- err = -ENOMEM;
- goto free_ucmd;
- }
-
- mutex_lock(&dev->flow_db->lock);
-
- ft_prio = get_flow_table(dev, flow_attr,
- is_egress ? MLX5_IB_FT_TX : MLX5_IB_FT_RX);
- if (IS_ERR(ft_prio)) {
- err = PTR_ERR(ft_prio);
- goto unlock;
- }
- if (flow_attr->type == IB_FLOW_ATTR_SNIFFER) {
- ft_prio_tx = get_flow_table(dev, flow_attr, MLX5_IB_FT_TX);
- if (IS_ERR(ft_prio_tx)) {
- err = PTR_ERR(ft_prio_tx);
- ft_prio_tx = NULL;
- goto destroy_ft;
- }
- }
-
- if (is_egress) {
- dst->type = MLX5_FLOW_DESTINATION_TYPE_PORT;
- } else {
- dst->type = MLX5_FLOW_DESTINATION_TYPE_TIR;
- if (mqp->is_rss)
- dst->tir_num = mqp->rss_qp.tirn;
- else
- dst->tir_num = mqp->raw_packet_qp.rq.tirn;
- }
-
- if (flow_attr->type == IB_FLOW_ATTR_NORMAL) {
- underlay_qpn = (mqp->flags & IB_QP_CREATE_SOURCE_QPN) ?
- mqp->underlay_qpn :
- 0;
- handler = _create_flow_rule(dev, ft_prio, flow_attr, dst,
- underlay_qpn, ucmd);
- } else if (flow_attr->type == IB_FLOW_ATTR_ALL_DEFAULT ||
- flow_attr->type == IB_FLOW_ATTR_MC_DEFAULT) {
- handler = create_leftovers_rule(dev, ft_prio, flow_attr,
- dst);
- } else if (flow_attr->type == IB_FLOW_ATTR_SNIFFER) {
- handler = create_sniffer_rule(dev, ft_prio, ft_prio_tx, dst);
- } else {
- err = -EINVAL;
- goto destroy_ft;
- }
-
- if (IS_ERR(handler)) {
- err = PTR_ERR(handler);
- handler = NULL;
- goto destroy_ft;
- }
-
- mutex_unlock(&dev->flow_db->lock);
- kfree(dst);
- kfree(ucmd);
-
- return &handler->ibflow;
-
-destroy_ft:
- put_flow_table(dev, ft_prio, false);
- if (ft_prio_tx)
- put_flow_table(dev, ft_prio_tx, false);
-unlock:
- mutex_unlock(&dev->flow_db->lock);
- kfree(dst);
-free_ucmd:
- kfree(ucmd);
- return ERR_PTR(err);
-}
-
-static struct mlx5_ib_flow_prio *
-_get_flow_table(struct mlx5_ib_dev *dev,
- struct mlx5_ib_flow_matcher *fs_matcher,
- bool mcast)
-{
- struct mlx5_flow_namespace *ns = NULL;
- struct mlx5_ib_flow_prio *prio = NULL;
- int max_table_size = 0;
- bool esw_encap;
- u32 flags = 0;
- int priority;
-
- if (mcast)
- priority = MLX5_IB_FLOW_MCAST_PRIO;
- else
- priority = ib_prio_to_core_prio(fs_matcher->priority, false);
-
- esw_encap = mlx5_eswitch_get_encap_mode(dev->mdev) !=
- DEVLINK_ESWITCH_ENCAP_MODE_NONE;
- if (fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_BYPASS) {
- max_table_size = BIT(MLX5_CAP_FLOWTABLE_NIC_RX(dev->mdev,
- log_max_ft_size));
- if (MLX5_CAP_FLOWTABLE_NIC_RX(dev->mdev, decap) && !esw_encap)
- flags |= MLX5_FLOW_TABLE_TUNNEL_EN_DECAP;
- if (MLX5_CAP_FLOWTABLE_NIC_RX(dev->mdev,
- reformat_l3_tunnel_to_l2) &&
- !esw_encap)
- flags |= MLX5_FLOW_TABLE_TUNNEL_EN_REFORMAT;
- } else if (fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_EGRESS) {
- max_table_size = BIT(
- MLX5_CAP_FLOWTABLE_NIC_TX(dev->mdev, log_max_ft_size));
- if (MLX5_CAP_FLOWTABLE_NIC_TX(dev->mdev, reformat) && !esw_encap)
- flags |= MLX5_FLOW_TABLE_TUNNEL_EN_REFORMAT;
- } else if (fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_FDB) {
- max_table_size = BIT(
- MLX5_CAP_ESW_FLOWTABLE_FDB(dev->mdev, log_max_ft_size));
- if (MLX5_CAP_ESW_FLOWTABLE_FDB(dev->mdev, decap) && esw_encap)
- flags |= MLX5_FLOW_TABLE_TUNNEL_EN_DECAP;
- if (MLX5_CAP_ESW_FLOWTABLE_FDB(dev->mdev, reformat_l3_tunnel_to_l2) &&
- esw_encap)
- flags |= MLX5_FLOW_TABLE_TUNNEL_EN_REFORMAT;
- priority = FDB_BYPASS_PATH;
- } else if (fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_RDMA_RX) {
- max_table_size =
- BIT(MLX5_CAP_FLOWTABLE_RDMA_RX(dev->mdev,
- log_max_ft_size));
- priority = fs_matcher->priority;
- } else if (fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_RDMA_TX) {
- max_table_size =
- BIT(MLX5_CAP_FLOWTABLE_RDMA_TX(dev->mdev,
- log_max_ft_size));
- priority = fs_matcher->priority;
- }
-
- max_table_size = min_t(int, max_table_size, MLX5_FS_MAX_ENTRIES);
-
- ns = mlx5_get_flow_namespace(dev->mdev, fs_matcher->ns_type);
- if (!ns)
- return ERR_PTR(-ENOTSUPP);
-
- if (fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_BYPASS)
- prio = &dev->flow_db->prios[priority];
- else if (fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_EGRESS)
- prio = &dev->flow_db->egress_prios[priority];
- else if (fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_FDB)
- prio = &dev->flow_db->fdb;
- else if (fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_RDMA_RX)
- prio = &dev->flow_db->rdma_rx[priority];
- else if (fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_RDMA_TX)
- prio = &dev->flow_db->rdma_tx[priority];
-
- if (!prio)
- return ERR_PTR(-EINVAL);
-
- if (prio->flow_table)
- return prio;
-
- return _get_prio(ns, prio, priority, max_table_size,
- MLX5_FS_MAX_TYPES, flags);
-}
-
-static struct mlx5_ib_flow_handler *
-_create_raw_flow_rule(struct mlx5_ib_dev *dev,
- struct mlx5_ib_flow_prio *ft_prio,
- struct mlx5_flow_destination *dst,
- struct mlx5_ib_flow_matcher *fs_matcher,
- struct mlx5_flow_context *flow_context,
- struct mlx5_flow_act *flow_act,
- void *cmd_in, int inlen,
- int dst_num)
-{
- struct mlx5_ib_flow_handler *handler;
- struct mlx5_flow_spec *spec;
- struct mlx5_flow_table *ft = ft_prio->flow_table;
- int err = 0;
-
- spec = kvzalloc(sizeof(*spec), GFP_KERNEL);
- handler = kzalloc(sizeof(*handler), GFP_KERNEL);
- if (!handler || !spec) {
- err = -ENOMEM;
- goto free;
- }
-
- INIT_LIST_HEAD(&handler->list);
-
- memcpy(spec->match_value, cmd_in, inlen);
- memcpy(spec->match_criteria, fs_matcher->matcher_mask.match_params,
- fs_matcher->mask_len);
- spec->match_criteria_enable = fs_matcher->match_criteria_enable;
- spec->flow_context = *flow_context;
-
- handler->rule = mlx5_add_flow_rules(ft, spec,
- flow_act, dst, dst_num);
-
- if (IS_ERR(handler->rule)) {
- err = PTR_ERR(handler->rule);
- goto free;
- }
-
- ft_prio->refcount++;
- handler->prio = ft_prio;
- handler->dev = dev;
- ft_prio->flow_table = ft;
-
-free:
- if (err)
- kfree(handler);
- kvfree(spec);
- return err ? ERR_PTR(err) : handler;
-}
-
-static bool raw_fs_is_multicast(struct mlx5_ib_flow_matcher *fs_matcher,
- void *match_v)
-{
- void *match_c;
- void *match_v_set_lyr_2_4, *match_c_set_lyr_2_4;
- void *dmac, *dmac_mask;
- void *ipv4, *ipv4_mask;
-
- if (!(fs_matcher->match_criteria_enable &
- (1 << MATCH_CRITERIA_ENABLE_OUTER_BIT)))
- return false;
-
- match_c = fs_matcher->matcher_mask.match_params;
- match_v_set_lyr_2_4 = MLX5_ADDR_OF(fte_match_param, match_v,
- outer_headers);
- match_c_set_lyr_2_4 = MLX5_ADDR_OF(fte_match_param, match_c,
- outer_headers);
-
- dmac = MLX5_ADDR_OF(fte_match_set_lyr_2_4, match_v_set_lyr_2_4,
- dmac_47_16);
- dmac_mask = MLX5_ADDR_OF(fte_match_set_lyr_2_4, match_c_set_lyr_2_4,
- dmac_47_16);
-
- if (is_multicast_ether_addr(dmac) &&
- is_multicast_ether_addr(dmac_mask))
- return true;
-
- ipv4 = MLX5_ADDR_OF(fte_match_set_lyr_2_4, match_v_set_lyr_2_4,
- dst_ipv4_dst_ipv6.ipv4_layout.ipv4);
-
- ipv4_mask = MLX5_ADDR_OF(fte_match_set_lyr_2_4, match_c_set_lyr_2_4,
- dst_ipv4_dst_ipv6.ipv4_layout.ipv4);
-
- if (ipv4_is_multicast(*(__be32 *)(ipv4)) &&
- ipv4_is_multicast(*(__be32 *)(ipv4_mask)))
- return true;
-
- return false;
-}
-
-struct mlx5_ib_flow_handler *
-mlx5_ib_raw_fs_rule_add(struct mlx5_ib_dev *dev,
- struct mlx5_ib_flow_matcher *fs_matcher,
- struct mlx5_flow_context *flow_context,
- struct mlx5_flow_act *flow_act,
- u32 counter_id,
- void *cmd_in, int inlen, int dest_id,
- int dest_type)
-{
- struct mlx5_flow_destination *dst;
- struct mlx5_ib_flow_prio *ft_prio;
- struct mlx5_ib_flow_handler *handler;
- int dst_num = 0;
- bool mcast;
- int err;
-
- if (fs_matcher->flow_type != MLX5_IB_FLOW_TYPE_NORMAL)
- return ERR_PTR(-EOPNOTSUPP);
-
- if (fs_matcher->priority > MLX5_IB_FLOW_LAST_PRIO)
- return ERR_PTR(-ENOMEM);
-
- dst = kcalloc(2, sizeof(*dst), GFP_KERNEL);
- if (!dst)
- return ERR_PTR(-ENOMEM);
-
- mcast = raw_fs_is_multicast(fs_matcher, cmd_in);
- mutex_lock(&dev->flow_db->lock);
-
- ft_prio = _get_flow_table(dev, fs_matcher, mcast);
- if (IS_ERR(ft_prio)) {
- err = PTR_ERR(ft_prio);
- goto unlock;
- }
-
- if (dest_type == MLX5_FLOW_DESTINATION_TYPE_TIR) {
- dst[dst_num].type = dest_type;
- dst[dst_num++].tir_num = dest_id;
- flow_act->action |= MLX5_FLOW_CONTEXT_ACTION_FWD_DEST;
- } else if (dest_type == MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE) {
- dst[dst_num].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE_NUM;
- dst[dst_num++].ft_num = dest_id;
- flow_act->action |= MLX5_FLOW_CONTEXT_ACTION_FWD_DEST;
- } else if (dest_type == MLX5_FLOW_DESTINATION_TYPE_PORT) {
- dst[dst_num++].type = MLX5_FLOW_DESTINATION_TYPE_PORT;
- flow_act->action |= MLX5_FLOW_CONTEXT_ACTION_ALLOW;
- }
-
-
- if (flow_act->action & MLX5_FLOW_CONTEXT_ACTION_COUNT) {
- dst[dst_num].type = MLX5_FLOW_DESTINATION_TYPE_COUNTER;
- dst[dst_num].counter_id = counter_id;
- dst_num++;
- }
-
- handler = _create_raw_flow_rule(dev, ft_prio, dst, fs_matcher,
- flow_context, flow_act,
- cmd_in, inlen, dst_num);
-
- if (IS_ERR(handler)) {
- err = PTR_ERR(handler);
- goto destroy_ft;
- }
-
- mutex_unlock(&dev->flow_db->lock);
- atomic_inc(&fs_matcher->usecnt);
- handler->flow_matcher = fs_matcher;
-
- kfree(dst);
-
- return handler;
-
-destroy_ft:
- put_flow_table(dev, ft_prio, false);
-unlock:
- mutex_unlock(&dev->flow_db->lock);
- kfree(dst);
-
- return ERR_PTR(err);
-}
-
-static u32 mlx5_ib_flow_action_flags_to_accel_xfrm_flags(u32 mlx5_flags)
-{
- u32 flags = 0;
-
- if (mlx5_flags & MLX5_IB_UAPI_FLOW_ACTION_FLAGS_REQUIRE_METADATA)
- flags |= MLX5_ACCEL_XFRM_FLAG_REQUIRE_METADATA;
-
- return flags;
-}
-
-#define MLX5_FLOW_ACTION_ESP_CREATE_LAST_SUPPORTED MLX5_IB_UAPI_FLOW_ACTION_FLAGS_REQUIRE_METADATA
-static struct ib_flow_action *
-mlx5_ib_create_flow_action_esp(struct ib_device *device,
- const struct ib_flow_action_attrs_esp *attr,
- struct uverbs_attr_bundle *attrs)
-{
- struct mlx5_ib_dev *mdev = to_mdev(device);
- struct ib_uverbs_flow_action_esp_keymat_aes_gcm *aes_gcm;
- struct mlx5_accel_esp_xfrm_attrs accel_attrs = {};
- struct mlx5_ib_flow_action *action;
- u64 action_flags;
- u64 flags;
- int err = 0;
-
- err = uverbs_get_flags64(
- &action_flags, attrs, MLX5_IB_ATTR_CREATE_FLOW_ACTION_FLAGS,
- ((MLX5_FLOW_ACTION_ESP_CREATE_LAST_SUPPORTED << 1) - 1));
- if (err)
- return ERR_PTR(err);
-
- flags = mlx5_ib_flow_action_flags_to_accel_xfrm_flags(action_flags);
-
- /* We current only support a subset of the standard features. Only a
- * keymat of type AES_GCM, with icv_len == 16, iv_algo == SEQ and esn
- * (with overlap). Full offload mode isn't supported.
- */
- if (!attr->keymat || attr->replay || attr->encap ||
- attr->spi || attr->seq || attr->tfc_pad ||
- attr->hard_limit_pkts ||
- (attr->flags & ~(IB_FLOW_ACTION_ESP_FLAGS_ESN_TRIGGERED |
- IB_UVERBS_FLOW_ACTION_ESP_FLAGS_ENCRYPT)))
- return ERR_PTR(-EOPNOTSUPP);
-
- if (attr->keymat->protocol !=
- IB_UVERBS_FLOW_ACTION_ESP_KEYMAT_AES_GCM)
- return ERR_PTR(-EOPNOTSUPP);
-
- aes_gcm = &attr->keymat->keymat.aes_gcm;
-
- if (aes_gcm->icv_len != 16 ||
- aes_gcm->iv_algo != IB_UVERBS_FLOW_ACTION_IV_ALGO_SEQ)
- return ERR_PTR(-EOPNOTSUPP);
-
- action = kmalloc(sizeof(*action), GFP_KERNEL);
- if (!action)
- return ERR_PTR(-ENOMEM);
-
- action->esp_aes_gcm.ib_flags = attr->flags;
- memcpy(&accel_attrs.keymat.aes_gcm.aes_key, &aes_gcm->aes_key,
- sizeof(accel_attrs.keymat.aes_gcm.aes_key));
- accel_attrs.keymat.aes_gcm.key_len = aes_gcm->key_len * 8;
- memcpy(&accel_attrs.keymat.aes_gcm.salt, &aes_gcm->salt,
- sizeof(accel_attrs.keymat.aes_gcm.salt));
- memcpy(&accel_attrs.keymat.aes_gcm.seq_iv, &aes_gcm->iv,
- sizeof(accel_attrs.keymat.aes_gcm.seq_iv));
- accel_attrs.keymat.aes_gcm.icv_len = aes_gcm->icv_len * 8;
- accel_attrs.keymat.aes_gcm.iv_algo = MLX5_ACCEL_ESP_AES_GCM_IV_ALGO_SEQ;
- accel_attrs.keymat_type = MLX5_ACCEL_ESP_KEYMAT_AES_GCM;
-
- accel_attrs.esn = attr->esn;
- if (attr->flags & IB_FLOW_ACTION_ESP_FLAGS_ESN_TRIGGERED)
- accel_attrs.flags |= MLX5_ACCEL_ESP_FLAGS_ESN_TRIGGERED;
- if (attr->flags & IB_UVERBS_FLOW_ACTION_ESP_FLAGS_ESN_NEW_WINDOW)
- accel_attrs.flags |= MLX5_ACCEL_ESP_FLAGS_ESN_STATE_OVERLAP;
-
- if (attr->flags & IB_UVERBS_FLOW_ACTION_ESP_FLAGS_ENCRYPT)
- accel_attrs.action |= MLX5_ACCEL_ESP_ACTION_ENCRYPT;
-
- action->esp_aes_gcm.ctx =
- mlx5_accel_esp_create_xfrm(mdev->mdev, &accel_attrs, flags);
- if (IS_ERR(action->esp_aes_gcm.ctx)) {
- err = PTR_ERR(action->esp_aes_gcm.ctx);
- goto err_parse;
- }
-
- action->esp_aes_gcm.ib_flags = attr->flags;
-
- return &action->ib_action;
-
-err_parse:
- kfree(action);
- return ERR_PTR(err);
-}
-
-static int
-mlx5_ib_modify_flow_action_esp(struct ib_flow_action *action,
- const struct ib_flow_action_attrs_esp *attr,
- struct uverbs_attr_bundle *attrs)
-{
- struct mlx5_ib_flow_action *maction = to_mflow_act(action);
- struct mlx5_accel_esp_xfrm_attrs accel_attrs;
- int err = 0;
-
- if (attr->keymat || attr->replay || attr->encap ||
- attr->spi || attr->seq || attr->tfc_pad ||
- attr->hard_limit_pkts ||
- (attr->flags & ~(IB_FLOW_ACTION_ESP_FLAGS_ESN_TRIGGERED |
- IB_FLOW_ACTION_ESP_FLAGS_MOD_ESP_ATTRS |
- IB_UVERBS_FLOW_ACTION_ESP_FLAGS_ESN_NEW_WINDOW)))
- return -EOPNOTSUPP;
-
- /* Only the ESN value or the MLX5_ACCEL_ESP_FLAGS_ESN_STATE_OVERLAP can
- * be modified.
- */
- if (!(maction->esp_aes_gcm.ib_flags &
- IB_FLOW_ACTION_ESP_FLAGS_ESN_TRIGGERED) &&
- attr->flags & (IB_FLOW_ACTION_ESP_FLAGS_ESN_TRIGGERED |
- IB_UVERBS_FLOW_ACTION_ESP_FLAGS_ESN_NEW_WINDOW))
- return -EINVAL;
-
- memcpy(&accel_attrs, &maction->esp_aes_gcm.ctx->attrs,
- sizeof(accel_attrs));
-
- accel_attrs.esn = attr->esn;
- if (attr->flags & IB_UVERBS_FLOW_ACTION_ESP_FLAGS_ESN_NEW_WINDOW)
- accel_attrs.flags |= MLX5_ACCEL_ESP_FLAGS_ESN_STATE_OVERLAP;
- else
- accel_attrs.flags &= ~MLX5_ACCEL_ESP_FLAGS_ESN_STATE_OVERLAP;
-
- err = mlx5_accel_esp_modify_xfrm(maction->esp_aes_gcm.ctx,
- &accel_attrs);
- if (err)
- return err;
-
- maction->esp_aes_gcm.ib_flags &=
- ~IB_UVERBS_FLOW_ACTION_ESP_FLAGS_ESN_NEW_WINDOW;
- maction->esp_aes_gcm.ib_flags |=
- attr->flags & IB_UVERBS_FLOW_ACTION_ESP_FLAGS_ESN_NEW_WINDOW;
-
- return 0;
-}
-
-static int mlx5_ib_destroy_flow_action(struct ib_flow_action *action)
-{
- struct mlx5_ib_flow_action *maction = to_mflow_act(action);
-
- switch (action->type) {
- case IB_FLOW_ACTION_ESP:
- /*
- * We only support aes_gcm by now, so we implicitly know this is
- * the underline crypto.
- */
- mlx5_accel_esp_destroy_xfrm(maction->esp_aes_gcm.ctx);
- break;
- case IB_FLOW_ACTION_UNSPECIFIED:
- mlx5_ib_destroy_flow_action_raw(maction);
- break;
- default:
- WARN_ON(true);
- break;
- }
-
- kfree(maction);
- return 0;
-}
-
static int mlx5_ib_mcg_attach(struct ib_qp *ibqp, union ib_gid *gid, u16 lid)
{
struct mlx5_ib_dev *dev = to_mdev(ibqp->device);
@@ -4848,137 +3020,6 @@
return __get_port_caps(dev, port);
}
-static void destroy_umrc_res(struct mlx5_ib_dev *dev)
-{
- int err;
-
- err = mlx5_mr_cache_cleanup(dev);
- if (err)
- mlx5_ib_warn(dev, "mr cache cleanup failed\n");
-
- if (dev->umrc.qp)
- mlx5_ib_destroy_qp(dev->umrc.qp, NULL);
- if (dev->umrc.cq)
- ib_free_cq(dev->umrc.cq);
- if (dev->umrc.pd)
- ib_dealloc_pd(dev->umrc.pd);
-}
-
-enum {
- MAX_UMR_WR = 128,
-};
-
-static int create_umr_res(struct mlx5_ib_dev *dev)
-{
- struct ib_qp_init_attr *init_attr = NULL;
- struct ib_qp_attr *attr = NULL;
- struct ib_pd *pd;
- struct ib_cq *cq;
- struct ib_qp *qp;
- int ret;
-
- attr = kzalloc(sizeof(*attr), GFP_KERNEL);
- init_attr = kzalloc(sizeof(*init_attr), GFP_KERNEL);
- if (!attr || !init_attr) {
- ret = -ENOMEM;
- goto error_0;
- }
-
- pd = ib_alloc_pd(&dev->ib_dev, 0);
- if (IS_ERR(pd)) {
- mlx5_ib_dbg(dev, "Couldn't create PD for sync UMR QP\n");
- ret = PTR_ERR(pd);
- goto error_0;
- }
-
- cq = ib_alloc_cq(&dev->ib_dev, NULL, 128, 0, IB_POLL_SOFTIRQ);
- if (IS_ERR(cq)) {
- mlx5_ib_dbg(dev, "Couldn't create CQ for sync UMR QP\n");
- ret = PTR_ERR(cq);
- goto error_2;
- }
-
- init_attr->send_cq = cq;
- init_attr->recv_cq = cq;
- init_attr->sq_sig_type = IB_SIGNAL_ALL_WR;
- init_attr->cap.max_send_wr = MAX_UMR_WR;
- init_attr->cap.max_send_sge = 1;
- init_attr->qp_type = MLX5_IB_QPT_REG_UMR;
- init_attr->port_num = 1;
- qp = mlx5_ib_create_qp(pd, init_attr, NULL);
- if (IS_ERR(qp)) {
- mlx5_ib_dbg(dev, "Couldn't create sync UMR QP\n");
- ret = PTR_ERR(qp);
- goto error_3;
- }
- qp->device = &dev->ib_dev;
- qp->real_qp = qp;
- qp->uobject = NULL;
- qp->qp_type = MLX5_IB_QPT_REG_UMR;
- qp->send_cq = init_attr->send_cq;
- qp->recv_cq = init_attr->recv_cq;
-
- attr->qp_state = IB_QPS_INIT;
- attr->port_num = 1;
- ret = mlx5_ib_modify_qp(qp, attr, IB_QP_STATE | IB_QP_PKEY_INDEX |
- IB_QP_PORT, NULL);
- if (ret) {
- mlx5_ib_dbg(dev, "Couldn't modify UMR QP\n");
- goto error_4;
- }
-
- memset(attr, 0, sizeof(*attr));
- attr->qp_state = IB_QPS_RTR;
- attr->path_mtu = IB_MTU_256;
-
- ret = mlx5_ib_modify_qp(qp, attr, IB_QP_STATE, NULL);
- if (ret) {
- mlx5_ib_dbg(dev, "Couldn't modify umr QP to rtr\n");
- goto error_4;
- }
-
- memset(attr, 0, sizeof(*attr));
- attr->qp_state = IB_QPS_RTS;
- ret = mlx5_ib_modify_qp(qp, attr, IB_QP_STATE, NULL);
- if (ret) {
- mlx5_ib_dbg(dev, "Couldn't modify umr QP to rts\n");
- goto error_4;
- }
-
- dev->umrc.qp = qp;
- dev->umrc.cq = cq;
- dev->umrc.pd = pd;
-
- sema_init(&dev->umrc.sem, MAX_UMR_WR);
- ret = mlx5_mr_cache_init(dev);
- if (ret) {
- mlx5_ib_warn(dev, "mr cache init failed %d\n", ret);
- goto error_4;
- }
-
- kfree(attr);
- kfree(init_attr);
-
- return 0;
-
-error_4:
- mlx5_ib_destroy_qp(qp, NULL);
- dev->umrc.qp = NULL;
-
-error_3:
- ib_free_cq(cq);
- dev->umrc.cq = NULL;
-
-error_2:
- ib_dealloc_pd(pd);
- dev->umrc.pd = NULL;
-
-error_0:
- kfree(attr);
- kfree(init_attr);
- return ret;
-}
-
static u8 mlx5_get_umr_fence(u8 umr_fence_cap)
{
switch (umr_fence_cap) {
@@ -4991,18 +3032,20 @@
}
}
-static int create_dev_resources(struct mlx5_ib_resources *devr)
+static int mlx5_ib_dev_res_init(struct mlx5_ib_dev *dev)
{
+ struct mlx5_ib_resources *devr = &dev->devr;
struct ib_srq_init_attr attr;
- struct mlx5_ib_dev *dev;
struct ib_device *ibdev;
struct ib_cq_init_attr cq_attr = {.cqe = 1};
int port;
int ret = 0;
- dev = container_of(devr, struct mlx5_ib_dev, devr);
ibdev = &dev->ib_dev;
+ if (!MLX5_CAP_GEN(dev->mdev, xrc))
+ return -EOPNOTSUPP;
+
mutex_init(&devr->mutex);
devr->p0 = rdma_zalloc_drv_obj(ibdev, ib_pd);
@@ -5030,34 +3073,19 @@
if (ret)
goto err_create_cq;
- devr->x0 = mlx5_ib_alloc_xrcd(&dev->ib_dev, NULL);
- if (IS_ERR(devr->x0)) {
- ret = PTR_ERR(devr->x0);
+ ret = mlx5_cmd_xrcd_alloc(dev->mdev, &devr->xrcdn0, 0);
+ if (ret)
goto error2;
- }
- devr->x0->device = &dev->ib_dev;
- devr->x0->inode = NULL;
- atomic_set(&devr->x0->usecnt, 0);
- mutex_init(&devr->x0->tgt_qp_mutex);
- INIT_LIST_HEAD(&devr->x0->tgt_qp_list);
- devr->x1 = mlx5_ib_alloc_xrcd(&dev->ib_dev, NULL);
- if (IS_ERR(devr->x1)) {
- ret = PTR_ERR(devr->x1);
+ ret = mlx5_cmd_xrcd_alloc(dev->mdev, &devr->xrcdn1, 0);
+ if (ret)
goto error3;
- }
- devr->x1->device = &dev->ib_dev;
- devr->x1->inode = NULL;
- atomic_set(&devr->x1->usecnt, 0);
- mutex_init(&devr->x1->tgt_qp_mutex);
- INIT_LIST_HEAD(&devr->x1->tgt_qp_list);
memset(&attr, 0, sizeof(attr));
attr.attr.max_sge = 1;
attr.attr.max_wr = 1;
attr.srq_type = IB_SRQT_XRC;
attr.ext.cq = devr->c0;
- attr.ext.xrc.xrcd = devr->x0;
devr->s0 = rdma_zalloc_drv_obj(ibdev, ib_srq);
if (!devr->s0) {
@@ -5068,13 +3096,11 @@
devr->s0->device = &dev->ib_dev;
devr->s0->pd = devr->p0;
devr->s0->srq_type = IB_SRQT_XRC;
- devr->s0->ext.xrc.xrcd = devr->x0;
devr->s0->ext.cq = devr->c0;
ret = mlx5_ib_create_srq(devr->s0, &attr, NULL);
if (ret)
goto err_create;
- atomic_inc(&devr->s0->ext.xrc.xrcd->usecnt);
atomic_inc(&devr->s0->ext.cq->usecnt);
atomic_inc(&devr->p0->usecnt);
atomic_set(&devr->s0->usecnt, 0);
@@ -5116,9 +3142,9 @@
err_create:
kfree(devr->s0);
error4:
- mlx5_ib_dealloc_xrcd(devr->x1, NULL);
+ mlx5_cmd_xrcd_dealloc(dev->mdev, devr->xrcdn1, 0);
error3:
- mlx5_ib_dealloc_xrcd(devr->x0, NULL);
+ mlx5_cmd_xrcd_dealloc(dev->mdev, devr->xrcdn0, 0);
error2:
mlx5_ib_destroy_cq(devr->c0, NULL);
err_create_cq:
@@ -5130,16 +3156,17 @@
return ret;
}
-static void destroy_dev_resources(struct mlx5_ib_resources *devr)
+static void mlx5_ib_dev_res_cleanup(struct mlx5_ib_dev *dev)
{
+ struct mlx5_ib_resources *devr = &dev->devr;
int port;
mlx5_ib_destroy_srq(devr->s1, NULL);
kfree(devr->s1);
mlx5_ib_destroy_srq(devr->s0, NULL);
kfree(devr->s0);
- mlx5_ib_dealloc_xrcd(devr->x0, NULL);
- mlx5_ib_dealloc_xrcd(devr->x1, NULL);
+ mlx5_cmd_xrcd_dealloc(dev->mdev, devr->xrcdn1, 0);
+ mlx5_cmd_xrcd_dealloc(dev->mdev, devr->xrcdn0, 0);
mlx5_ib_destroy_cq(devr->c0, NULL);
kfree(devr->c0);
mlx5_ib_dealloc_pd(devr->p0, NULL);
@@ -5332,466 +3359,6 @@
mlx5_nic_vport_disable_roce(dev->mdev);
}
-struct mlx5_ib_counter {
- const char *name;
- size_t offset;
-};
-
-#define INIT_Q_COUNTER(_name) \
- { .name = #_name, .offset = MLX5_BYTE_OFF(query_q_counter_out, _name)}
-
-static const struct mlx5_ib_counter basic_q_cnts[] = {
- INIT_Q_COUNTER(rx_write_requests),
- INIT_Q_COUNTER(rx_read_requests),
- INIT_Q_COUNTER(rx_atomic_requests),
- INIT_Q_COUNTER(out_of_buffer),
-};
-
-static const struct mlx5_ib_counter out_of_seq_q_cnts[] = {
- INIT_Q_COUNTER(out_of_sequence),
-};
-
-static const struct mlx5_ib_counter retrans_q_cnts[] = {
- INIT_Q_COUNTER(duplicate_request),
- INIT_Q_COUNTER(rnr_nak_retry_err),
- INIT_Q_COUNTER(packet_seq_err),
- INIT_Q_COUNTER(implied_nak_seq_err),
- INIT_Q_COUNTER(local_ack_timeout_err),
-};
-
-#define INIT_CONG_COUNTER(_name) \
- { .name = #_name, .offset = \
- MLX5_BYTE_OFF(query_cong_statistics_out, _name ## _high)}
-
-static const struct mlx5_ib_counter cong_cnts[] = {
- INIT_CONG_COUNTER(rp_cnp_ignored),
- INIT_CONG_COUNTER(rp_cnp_handled),
- INIT_CONG_COUNTER(np_ecn_marked_roce_packets),
- INIT_CONG_COUNTER(np_cnp_sent),
-};
-
-static const struct mlx5_ib_counter extended_err_cnts[] = {
- INIT_Q_COUNTER(resp_local_length_error),
- INIT_Q_COUNTER(resp_cqe_error),
- INIT_Q_COUNTER(req_cqe_error),
- INIT_Q_COUNTER(req_remote_invalid_request),
- INIT_Q_COUNTER(req_remote_access_errors),
- INIT_Q_COUNTER(resp_remote_access_errors),
- INIT_Q_COUNTER(resp_cqe_flush_error),
- INIT_Q_COUNTER(req_cqe_flush_error),
-};
-
-static const struct mlx5_ib_counter roce_accl_cnts[] = {
- INIT_Q_COUNTER(roce_adp_retrans),
- INIT_Q_COUNTER(roce_adp_retrans_to),
- INIT_Q_COUNTER(roce_slow_restart),
- INIT_Q_COUNTER(roce_slow_restart_cnps),
- INIT_Q_COUNTER(roce_slow_restart_trans),
-};
-
-#define INIT_EXT_PPCNT_COUNTER(_name) \
- { .name = #_name, .offset = \
- MLX5_BYTE_OFF(ppcnt_reg, \
- counter_set.eth_extended_cntrs_grp_data_layout._name##_high)}
-
-static const struct mlx5_ib_counter ext_ppcnt_cnts[] = {
- INIT_EXT_PPCNT_COUNTER(rx_icrc_encapsulated),
-};
-
-static bool is_mdev_switchdev_mode(const struct mlx5_core_dev *mdev)
-{
- return MLX5_ESWITCH_MANAGER(mdev) &&
- mlx5_ib_eswitch_mode(mdev->priv.eswitch) ==
- MLX5_ESWITCH_OFFLOADS;
-}
-
-static void mlx5_ib_dealloc_counters(struct mlx5_ib_dev *dev)
-{
- u32 in[MLX5_ST_SZ_DW(dealloc_q_counter_in)] = {};
- int num_cnt_ports;
- int i;
-
- num_cnt_ports = is_mdev_switchdev_mode(dev->mdev) ? 1 : dev->num_ports;
-
- MLX5_SET(dealloc_q_counter_in, in, opcode,
- MLX5_CMD_OP_DEALLOC_Q_COUNTER);
-
- for (i = 0; i < num_cnt_ports; i++) {
- if (dev->port[i].cnts.set_id) {
- MLX5_SET(dealloc_q_counter_in, in, counter_set_id,
- dev->port[i].cnts.set_id);
- mlx5_cmd_exec_in(dev->mdev, dealloc_q_counter, in);
- }
- kfree(dev->port[i].cnts.names);
- kfree(dev->port[i].cnts.offsets);
- }
-}
-
-static int __mlx5_ib_alloc_counters(struct mlx5_ib_dev *dev,
- struct mlx5_ib_counters *cnts)
-{
- u32 num_counters;
-
- num_counters = ARRAY_SIZE(basic_q_cnts);
-
- if (MLX5_CAP_GEN(dev->mdev, out_of_seq_cnt))
- num_counters += ARRAY_SIZE(out_of_seq_q_cnts);
-
- if (MLX5_CAP_GEN(dev->mdev, retransmission_q_counters))
- num_counters += ARRAY_SIZE(retrans_q_cnts);
-
- if (MLX5_CAP_GEN(dev->mdev, enhanced_error_q_counters))
- num_counters += ARRAY_SIZE(extended_err_cnts);
-
- if (MLX5_CAP_GEN(dev->mdev, roce_accl))
- num_counters += ARRAY_SIZE(roce_accl_cnts);
-
- cnts->num_q_counters = num_counters;
-
- if (MLX5_CAP_GEN(dev->mdev, cc_query_allowed)) {
- cnts->num_cong_counters = ARRAY_SIZE(cong_cnts);
- num_counters += ARRAY_SIZE(cong_cnts);
- }
- if (MLX5_CAP_PCAM_FEATURE(dev->mdev, rx_icrc_encapsulated_counter)) {
- cnts->num_ext_ppcnt_counters = ARRAY_SIZE(ext_ppcnt_cnts);
- num_counters += ARRAY_SIZE(ext_ppcnt_cnts);
- }
- cnts->names = kcalloc(num_counters, sizeof(cnts->names), GFP_KERNEL);
- if (!cnts->names)
- return -ENOMEM;
-
- cnts->offsets = kcalloc(num_counters,
- sizeof(cnts->offsets), GFP_KERNEL);
- if (!cnts->offsets)
- goto err_names;
-
- return 0;
-
-err_names:
- kfree(cnts->names);
- cnts->names = NULL;
- return -ENOMEM;
-}
-
-static void mlx5_ib_fill_counters(struct mlx5_ib_dev *dev,
- const char **names,
- size_t *offsets)
-{
- int i;
- int j = 0;
-
- for (i = 0; i < ARRAY_SIZE(basic_q_cnts); i++, j++) {
- names[j] = basic_q_cnts[i].name;
- offsets[j] = basic_q_cnts[i].offset;
- }
-
- if (MLX5_CAP_GEN(dev->mdev, out_of_seq_cnt)) {
- for (i = 0; i < ARRAY_SIZE(out_of_seq_q_cnts); i++, j++) {
- names[j] = out_of_seq_q_cnts[i].name;
- offsets[j] = out_of_seq_q_cnts[i].offset;
- }
- }
-
- if (MLX5_CAP_GEN(dev->mdev, retransmission_q_counters)) {
- for (i = 0; i < ARRAY_SIZE(retrans_q_cnts); i++, j++) {
- names[j] = retrans_q_cnts[i].name;
- offsets[j] = retrans_q_cnts[i].offset;
- }
- }
-
- if (MLX5_CAP_GEN(dev->mdev, enhanced_error_q_counters)) {
- for (i = 0; i < ARRAY_SIZE(extended_err_cnts); i++, j++) {
- names[j] = extended_err_cnts[i].name;
- offsets[j] = extended_err_cnts[i].offset;
- }
- }
-
- if (MLX5_CAP_GEN(dev->mdev, roce_accl)) {
- for (i = 0; i < ARRAY_SIZE(roce_accl_cnts); i++, j++) {
- names[j] = roce_accl_cnts[i].name;
- offsets[j] = roce_accl_cnts[i].offset;
- }
- }
-
- if (MLX5_CAP_GEN(dev->mdev, cc_query_allowed)) {
- for (i = 0; i < ARRAY_SIZE(cong_cnts); i++, j++) {
- names[j] = cong_cnts[i].name;
- offsets[j] = cong_cnts[i].offset;
- }
- }
-
- if (MLX5_CAP_PCAM_FEATURE(dev->mdev, rx_icrc_encapsulated_counter)) {
- for (i = 0; i < ARRAY_SIZE(ext_ppcnt_cnts); i++, j++) {
- names[j] = ext_ppcnt_cnts[i].name;
- offsets[j] = ext_ppcnt_cnts[i].offset;
- }
- }
-}
-
-static int mlx5_ib_alloc_counters(struct mlx5_ib_dev *dev)
-{
- u32 out[MLX5_ST_SZ_DW(alloc_q_counter_out)] = {};
- u32 in[MLX5_ST_SZ_DW(alloc_q_counter_in)] = {};
- int num_cnt_ports;
- int err = 0;
- int i;
- bool is_shared;
-
- MLX5_SET(alloc_q_counter_in, in, opcode, MLX5_CMD_OP_ALLOC_Q_COUNTER);
- is_shared = MLX5_CAP_GEN(dev->mdev, log_max_uctx) != 0;
- num_cnt_ports = is_mdev_switchdev_mode(dev->mdev) ? 1 : dev->num_ports;
-
- for (i = 0; i < num_cnt_ports; i++) {
- err = __mlx5_ib_alloc_counters(dev, &dev->port[i].cnts);
- if (err)
- goto err_alloc;
-
- mlx5_ib_fill_counters(dev, dev->port[i].cnts.names,
- dev->port[i].cnts.offsets);
-
- MLX5_SET(alloc_q_counter_in, in, uid,
- is_shared ? MLX5_SHARED_RESOURCE_UID : 0);
-
- err = mlx5_cmd_exec_inout(dev->mdev, alloc_q_counter, in, out);
- if (err) {
- mlx5_ib_warn(dev,
- "couldn't allocate queue counter for port %d, err %d\n",
- i + 1, err);
- goto err_alloc;
- }
-
- dev->port[i].cnts.set_id =
- MLX5_GET(alloc_q_counter_out, out, counter_set_id);
- }
- return 0;
-
-err_alloc:
- mlx5_ib_dealloc_counters(dev);
- return err;
-}
-
-static const struct mlx5_ib_counters *get_counters(struct mlx5_ib_dev *dev,
- u8 port_num)
-{
- return is_mdev_switchdev_mode(dev->mdev) ? &dev->port[0].cnts :
- &dev->port[port_num].cnts;
-}
-
-/**
- * mlx5_ib_get_counters_id - Returns counters id to use for device+port
- * @dev: Pointer to mlx5 IB device
- * @port_num: Zero based port number
- *
- * mlx5_ib_get_counters_id() Returns counters set id to use for given
- * device port combination in switchdev and non switchdev mode of the
- * parent device.
- */
-u16 mlx5_ib_get_counters_id(struct mlx5_ib_dev *dev, u8 port_num)
-{
- const struct mlx5_ib_counters *cnts = get_counters(dev, port_num);
-
- return cnts->set_id;
-}
-
-static struct rdma_hw_stats *mlx5_ib_alloc_hw_stats(struct ib_device *ibdev,
- u8 port_num)
-{
- struct mlx5_ib_dev *dev = to_mdev(ibdev);
- const struct mlx5_ib_counters *cnts;
- bool is_switchdev = is_mdev_switchdev_mode(dev->mdev);
-
- if ((is_switchdev && port_num) || (!is_switchdev && !port_num))
- return NULL;
-
- cnts = get_counters(dev, port_num - 1);
-
- return rdma_alloc_hw_stats_struct(cnts->names,
- cnts->num_q_counters +
- cnts->num_cong_counters +
- cnts->num_ext_ppcnt_counters,
- RDMA_HW_STATS_DEFAULT_LIFESPAN);
-}
-
-static int mlx5_ib_query_q_counters(struct mlx5_core_dev *mdev,
- const struct mlx5_ib_counters *cnts,
- struct rdma_hw_stats *stats,
- u16 set_id)
-{
- u32 out[MLX5_ST_SZ_DW(query_q_counter_out)] = {};
- u32 in[MLX5_ST_SZ_DW(query_q_counter_in)] = {};
- __be32 val;
- int ret, i;
-
- MLX5_SET(query_q_counter_in, in, opcode, MLX5_CMD_OP_QUERY_Q_COUNTER);
- MLX5_SET(query_q_counter_in, in, counter_set_id, set_id);
- ret = mlx5_cmd_exec_inout(mdev, query_q_counter, in, out);
- if (ret)
- return ret;
-
- for (i = 0; i < cnts->num_q_counters; i++) {
- val = *(__be32 *)((void *)out + cnts->offsets[i]);
- stats->value[i] = (u64)be32_to_cpu(val);
- }
-
- return 0;
-}
-
-static int mlx5_ib_query_ext_ppcnt_counters(struct mlx5_ib_dev *dev,
- const struct mlx5_ib_counters *cnts,
- struct rdma_hw_stats *stats)
-{
- int offset = cnts->num_q_counters + cnts->num_cong_counters;
- int sz = MLX5_ST_SZ_BYTES(ppcnt_reg);
- int ret, i;
- void *out;
-
- out = kvzalloc(sz, GFP_KERNEL);
- if (!out)
- return -ENOMEM;
-
- ret = mlx5_cmd_query_ext_ppcnt_counters(dev->mdev, out);
- if (ret)
- goto free;
-
- for (i = 0; i < cnts->num_ext_ppcnt_counters; i++)
- stats->value[i + offset] =
- be64_to_cpup((__be64 *)(out +
- cnts->offsets[i + offset]));
-free:
- kvfree(out);
- return ret;
-}
-
-static int mlx5_ib_get_hw_stats(struct ib_device *ibdev,
- struct rdma_hw_stats *stats,
- u8 port_num, int index)
-{
- struct mlx5_ib_dev *dev = to_mdev(ibdev);
- const struct mlx5_ib_counters *cnts = get_counters(dev, port_num - 1);
- struct mlx5_core_dev *mdev;
- int ret, num_counters;
- u8 mdev_port_num;
-
- if (!stats)
- return -EINVAL;
-
- num_counters = cnts->num_q_counters +
- cnts->num_cong_counters +
- cnts->num_ext_ppcnt_counters;
-
- /* q_counters are per IB device, query the master mdev */
- ret = mlx5_ib_query_q_counters(dev->mdev, cnts, stats, cnts->set_id);
- if (ret)
- return ret;
-
- if (MLX5_CAP_PCAM_FEATURE(dev->mdev, rx_icrc_encapsulated_counter)) {
- ret = mlx5_ib_query_ext_ppcnt_counters(dev, cnts, stats);
- if (ret)
- return ret;
- }
-
- if (MLX5_CAP_GEN(dev->mdev, cc_query_allowed)) {
- mdev = mlx5_ib_get_native_port_mdev(dev, port_num,
- &mdev_port_num);
- if (!mdev) {
- /* If port is not affiliated yet, its in down state
- * which doesn't have any counters yet, so it would be
- * zero. So no need to read from the HCA.
- */
- goto done;
- }
- ret = mlx5_lag_query_cong_counters(dev->mdev,
- stats->value +
- cnts->num_q_counters,
- cnts->num_cong_counters,
- cnts->offsets +
- cnts->num_q_counters);
-
- mlx5_ib_put_native_port_mdev(dev, port_num);
- if (ret)
- return ret;
- }
-
-done:
- return num_counters;
-}
-
-static struct rdma_hw_stats *
-mlx5_ib_counter_alloc_stats(struct rdma_counter *counter)
-{
- struct mlx5_ib_dev *dev = to_mdev(counter->device);
- const struct mlx5_ib_counters *cnts =
- get_counters(dev, counter->port - 1);
-
- return rdma_alloc_hw_stats_struct(cnts->names,
- cnts->num_q_counters +
- cnts->num_cong_counters +
- cnts->num_ext_ppcnt_counters,
- RDMA_HW_STATS_DEFAULT_LIFESPAN);
-}
-
-static int mlx5_ib_counter_update_stats(struct rdma_counter *counter)
-{
- struct mlx5_ib_dev *dev = to_mdev(counter->device);
- const struct mlx5_ib_counters *cnts =
- get_counters(dev, counter->port - 1);
-
- return mlx5_ib_query_q_counters(dev->mdev, cnts,
- counter->stats, counter->id);
-}
-
-static int mlx5_ib_counter_dealloc(struct rdma_counter *counter)
-{
- struct mlx5_ib_dev *dev = to_mdev(counter->device);
- u32 in[MLX5_ST_SZ_DW(dealloc_q_counter_in)] = {};
-
- if (!counter->id)
- return 0;
-
- MLX5_SET(dealloc_q_counter_in, in, opcode,
- MLX5_CMD_OP_DEALLOC_Q_COUNTER);
- MLX5_SET(dealloc_q_counter_in, in, counter_set_id, counter->id);
- return mlx5_cmd_exec_in(dev->mdev, dealloc_q_counter, in);
-}
-
-static int mlx5_ib_counter_bind_qp(struct rdma_counter *counter,
- struct ib_qp *qp)
-{
- struct mlx5_ib_dev *dev = to_mdev(qp->device);
- int err;
-
- if (!counter->id) {
- u32 out[MLX5_ST_SZ_DW(alloc_q_counter_out)] = {};
- u32 in[MLX5_ST_SZ_DW(alloc_q_counter_in)] = {};
-
- MLX5_SET(alloc_q_counter_in, in, opcode,
- MLX5_CMD_OP_ALLOC_Q_COUNTER);
- MLX5_SET(alloc_q_counter_in, in, uid, MLX5_SHARED_RESOURCE_UID);
- err = mlx5_cmd_exec_inout(dev->mdev, alloc_q_counter, in, out);
- if (err)
- return err;
- counter->id =
- MLX5_GET(alloc_q_counter_out, out, counter_set_id);
- }
-
- err = mlx5_ib_qp_set_counter(qp, counter);
- if (err)
- goto fail_set_counter;
-
- return 0;
-
-fail_set_counter:
- mlx5_ib_counter_dealloc(counter);
- counter->id = 0;
-
- return err;
-}
-
-static int mlx5_ib_counter_unbind_qp(struct ib_qp *qp)
-{
- return mlx5_ib_qp_set_counter(qp, NULL);
-}
-
static int mlx5_ib_rn_get_params(struct ib_device *device, u8 port_num,
enum rdma_netdev_t type,
struct rdma_netdev_alloc_params *params)
@@ -5802,23 +3369,6 @@
return mlx5_rdma_rn_get_params(to_mdev(device)->mdev, device, params);
}
-static void delay_drop_debugfs_cleanup(struct mlx5_ib_dev *dev)
-{
- if (!dev->delay_drop.dir_debugfs)
- return;
- debugfs_remove_recursive(dev->delay_drop.dir_debugfs);
- dev->delay_drop.dir_debugfs = NULL;
-}
-
-static void cancel_delay_drop(struct mlx5_ib_dev *dev)
-{
- if (!(dev->ib_dev.attrs.raw_packet_caps & IB_RAW_PACKET_CAP_DELAY_DROP))
- return;
-
- cancel_work_sync(&dev->delay_drop.delay_drop_work);
- delay_drop_debugfs_cleanup(dev);
-}
-
static ssize_t delay_drop_timeout_read(struct file *filp, char __user *buf,
size_t count, loff_t *pos)
{
@@ -5858,40 +3408,6 @@
.read = delay_drop_timeout_read,
};
-static void delay_drop_debugfs_init(struct mlx5_ib_dev *dev)
-{
- struct dentry *root;
-
- if (!mlx5_debugfs_root)
- return;
-
- root = debugfs_create_dir("delay_drop", dev->mdev->priv.dbg_root);
- dev->delay_drop.dir_debugfs = root;
-
- debugfs_create_atomic_t("num_timeout_events", 0400, root,
- &dev->delay_drop.events_cnt);
- debugfs_create_atomic_t("num_rqs", 0400, root,
- &dev->delay_drop.rqs_cnt);
- debugfs_create_file("timeout", 0600, root, &dev->delay_drop,
- &fops_delay_drop_timeout);
-}
-
-static void init_delay_drop(struct mlx5_ib_dev *dev)
-{
- if (!(dev->ib_dev.attrs.raw_packet_caps & IB_RAW_PACKET_CAP_DELAY_DROP))
- return;
-
- mutex_init(&dev->delay_drop.lock);
- dev->delay_drop.dev = dev;
- dev->delay_drop.activate = false;
- dev->delay_drop.timeout = MLX5_MAX_DELAY_DROP_TIMEOUT_MS * 1000;
- INIT_WORK(&dev->delay_drop.delay_drop_work, delay_drop_handler);
- atomic_set(&dev->delay_drop.rqs_cnt, 0);
- atomic_set(&dev->delay_drop.events_cnt, 0);
-
- delay_drop_debugfs_init(dev);
-}
-
static void mlx5_ib_unbind_slave_port(struct mlx5_ib_dev *ibdev,
struct mlx5_ib_multiport_info *mpi)
{
@@ -6385,90 +3901,32 @@
UVERBS_ATTR_FLAGS_IN(MLX5_IB_ATTR_CREATE_FLOW_ACTION_FLAGS,
enum mlx5_ib_uapi_flow_action_flags));
+ADD_UVERBS_ATTRIBUTES_SIMPLE(
+ mlx5_ib_query_context,
+ UVERBS_OBJECT_DEVICE,
+ UVERBS_METHOD_QUERY_CONTEXT,
+ UVERBS_ATTR_PTR_OUT(
+ MLX5_IB_ATTR_QUERY_CONTEXT_RESP_UCTX,
+ UVERBS_ATTR_STRUCT(struct mlx5_ib_alloc_ucontext_resp,
+ dump_fill_mkey),
+ UA_MANDATORY));
+
static const struct uapi_definition mlx5_ib_defs[] = {
UAPI_DEF_CHAIN(mlx5_ib_devx_defs),
UAPI_DEF_CHAIN(mlx5_ib_flow_defs),
UAPI_DEF_CHAIN(mlx5_ib_qos_defs),
+ UAPI_DEF_CHAIN(mlx5_ib_std_types_defs),
UAPI_DEF_CHAIN_OBJ_TREE(UVERBS_OBJECT_FLOW_ACTION,
&mlx5_ib_flow_action),
UAPI_DEF_CHAIN_OBJ_TREE(UVERBS_OBJECT_DM, &mlx5_ib_dm),
+ UAPI_DEF_CHAIN_OBJ_TREE(UVERBS_OBJECT_DEVICE, &mlx5_ib_query_context),
UAPI_DEF_CHAIN_OBJ_TREE_NAMED(MLX5_IB_OBJECT_VAR,
UAPI_DEF_IS_OBJ_SUPPORTED(var_is_supported)),
UAPI_DEF_CHAIN_OBJ_TREE_NAMED(MLX5_IB_OBJECT_UAR),
{}
};
-static int mlx5_ib_read_counters(struct ib_counters *counters,
- struct ib_counters_read_attr *read_attr,
- struct uverbs_attr_bundle *attrs)
-{
- struct mlx5_ib_mcounters *mcounters = to_mcounters(counters);
- struct mlx5_read_counters_attr mread_attr = {};
- struct mlx5_ib_flow_counters_desc *desc;
- int ret, i;
-
- mutex_lock(&mcounters->mcntrs_mutex);
- if (mcounters->cntrs_max_index > read_attr->ncounters) {
- ret = -EINVAL;
- goto err_bound;
- }
-
- mread_attr.out = kcalloc(mcounters->counters_num, sizeof(u64),
- GFP_KERNEL);
- if (!mread_attr.out) {
- ret = -ENOMEM;
- goto err_bound;
- }
-
- mread_attr.hw_cntrs_hndl = mcounters->hw_cntrs_hndl;
- mread_attr.flags = read_attr->flags;
- ret = mcounters->read_counters(counters->device, &mread_attr);
- if (ret)
- goto err_read;
-
- /* do the pass over the counters data array to assign according to the
- * descriptions and indexing pairs
- */
- desc = mcounters->counters_data;
- for (i = 0; i < mcounters->ncounters; i++)
- read_attr->counters_buff[desc[i].index] += mread_attr.out[desc[i].description];
-
-err_read:
- kfree(mread_attr.out);
-err_bound:
- mutex_unlock(&mcounters->mcntrs_mutex);
- return ret;
-}
-
-static int mlx5_ib_destroy_counters(struct ib_counters *counters)
-{
- struct mlx5_ib_mcounters *mcounters = to_mcounters(counters);
-
- counters_clear_description(counters);
- if (mcounters->hw_cntrs_hndl)
- mlx5_fc_destroy(to_mdev(counters->device)->mdev,
- mcounters->hw_cntrs_hndl);
-
- kfree(mcounters);
-
- return 0;
-}
-
-static struct ib_counters *mlx5_ib_create_counters(struct ib_device *device,
- struct uverbs_attr_bundle *attrs)
-{
- struct mlx5_ib_mcounters *mcounters;
-
- mcounters = kzalloc(sizeof(*mcounters), GFP_KERNEL);
- if (!mcounters)
- return ERR_PTR(-ENOMEM);
-
- mutex_init(&mcounters->mcntrs_mutex);
-
- return &mcounters->ibcntrs;
-}
-
static void mlx5_ib_stage_init_cleanup(struct mlx5_ib_dev *dev)
{
mlx5_ib_cleanup_multiport_master(dev);
@@ -6547,21 +4005,16 @@
return -ENOMEM;
}
-static int mlx5_ib_stage_flow_db_init(struct mlx5_ib_dev *dev)
+static int mlx5_ib_enable_driver(struct ib_device *dev)
{
- dev->flow_db = kzalloc(sizeof(*dev->flow_db), GFP_KERNEL);
+ struct mlx5_ib_dev *mdev = to_mdev(dev);
+ int ret;
- if (!dev->flow_db)
- return -ENOMEM;
+ ret = mlx5_ib_test_wc(mdev);
+ mlx5_ib_dbg(mdev, "Write-Combining %s",
+ mdev->wc_support ? "supported" : "not supported");
- mutex_init(&dev->flow_db->lock);
-
- return 0;
-}
-
-static void mlx5_ib_stage_flow_db_cleanup(struct mlx5_ib_dev *dev)
-{
- kfree(dev->flow_db);
+ return ret;
}
static const struct ib_device_ops mlx5_ib_dev_ops = {
@@ -6577,9 +4030,7 @@
.attach_mcast = mlx5_ib_mcg_attach,
.check_mr_status = mlx5_ib_check_mr_status,
.create_ah = mlx5_ib_create_ah,
- .create_counters = mlx5_ib_create_counters,
.create_cq = mlx5_ib_create_cq,
- .create_flow = mlx5_ib_create_flow,
.create_qp = mlx5_ib_create_qp,
.create_srq = mlx5_ib_create_srq,
.dealloc_pd = mlx5_ib_dealloc_pd,
@@ -6587,10 +4038,7 @@
.del_gid = mlx5_ib_del_gid,
.dereg_mr = mlx5_ib_dereg_mr,
.destroy_ah = mlx5_ib_destroy_ah,
- .destroy_counters = mlx5_ib_destroy_counters,
.destroy_cq = mlx5_ib_destroy_cq,
- .destroy_flow = mlx5_ib_destroy_flow,
- .destroy_flow_action = mlx5_ib_destroy_flow_action,
.destroy_qp = mlx5_ib_destroy_qp,
.destroy_srq = mlx5_ib_destroy_srq,
.detach_mcast = mlx5_ib_mcg_detach,
@@ -6598,8 +4046,6 @@
.drain_rq = mlx5_ib_drain_rq,
.drain_sq = mlx5_ib_drain_sq,
.enable_driver = mlx5_ib_enable_driver,
- .fill_res_entry = mlx5_ib_fill_res_entry,
- .fill_stat_entry = mlx5_ib_fill_stat_entry,
.get_dev_fw_str = get_dev_fw_str,
.get_dma_mr = mlx5_ib_get_dma_mr,
.get_link_layer = mlx5_ib_port_link_layer,
@@ -6623,24 +4069,20 @@
.query_pkey = mlx5_ib_query_pkey,
.query_qp = mlx5_ib_query_qp,
.query_srq = mlx5_ib_query_srq,
- .read_counters = mlx5_ib_read_counters,
+ .query_ucontext = mlx5_ib_query_ucontext,
.reg_user_mr = mlx5_ib_reg_user_mr,
.req_notify_cq = mlx5_ib_arm_cq,
.rereg_user_mr = mlx5_ib_rereg_user_mr,
.resize_cq = mlx5_ib_resize_cq,
INIT_RDMA_OBJ_SIZE(ib_ah, mlx5_ib_ah, ibah),
+ INIT_RDMA_OBJ_SIZE(ib_counters, mlx5_ib_mcounters, ibcntrs),
INIT_RDMA_OBJ_SIZE(ib_cq, mlx5_ib_cq, ibcq),
INIT_RDMA_OBJ_SIZE(ib_pd, mlx5_ib_pd, ibpd),
INIT_RDMA_OBJ_SIZE(ib_srq, mlx5_ib_srq, ibsrq),
INIT_RDMA_OBJ_SIZE(ib_ucontext, mlx5_ib_ucontext, ibucontext),
};
-static const struct ib_device_ops mlx5_ib_dev_flow_ipsec_ops = {
- .create_flow_action_esp = mlx5_ib_create_flow_action_esp,
- .modify_flow_action_esp = mlx5_ib_modify_flow_action_esp,
-};
-
static const struct ib_device_ops mlx5_ib_dev_ipoib_enhanced_ops = {
.rdma_netdev_get_params = mlx5_ib_rn_get_params,
};
@@ -6661,6 +4103,8 @@
static const struct ib_device_ops mlx5_ib_dev_xrc_ops = {
.alloc_xrcd = mlx5_ib_alloc_xrcd,
.dealloc_xrcd = mlx5_ib_dealloc_xrcd,
+
+ INIT_RDMA_OBJ_SIZE(ib_xrcd, mlx5_ib_xrcd, ibxrcd),
};
static const struct ib_device_ops mlx5_ib_dev_dm_ops = {
@@ -6769,9 +4213,6 @@
MLX5_GENERAL_OBJ_TYPES_CAP_SW_ICM)
ib_set_device_ops(&dev->ib_dev, &mlx5_ib_dev_dm_ops);
- if (mlx5_accel_ipsec_device_caps(dev->mdev) &
- MLX5_ACCEL_IPSEC_CAP_DEVICE)
- ib_set_device_ops(&dev->ib_dev, &mlx5_ib_dev_flow_ipsec_ops);
ib_set_device_ops(&dev->ib_dev, &mlx5_ib_dev_ops);
if (IS_ENABLED(CONFIG_INFINIBAND_USER_ACCESS))
@@ -6829,65 +4270,36 @@
.modify_wq = mlx5_ib_modify_wq,
};
-static int mlx5_ib_stage_common_roce_init(struct mlx5_ib_dev *dev)
-{
- u8 port_num;
-
- dev->ib_dev.uverbs_ex_cmd_mask |=
- (1ull << IB_USER_VERBS_EX_CMD_CREATE_WQ) |
- (1ull << IB_USER_VERBS_EX_CMD_MODIFY_WQ) |
- (1ull << IB_USER_VERBS_EX_CMD_DESTROY_WQ) |
- (1ull << IB_USER_VERBS_EX_CMD_CREATE_RWQ_IND_TBL) |
- (1ull << IB_USER_VERBS_EX_CMD_DESTROY_RWQ_IND_TBL);
- ib_set_device_ops(&dev->ib_dev, &mlx5_ib_dev_common_roce_ops);
-
- port_num = mlx5_core_native_port_num(dev->mdev) - 1;
-
- /* Register only for native ports */
- return mlx5_add_netdev_notifier(dev, port_num);
-}
-
-static void mlx5_ib_stage_common_roce_cleanup(struct mlx5_ib_dev *dev)
-{
- u8 port_num = mlx5_core_native_port_num(dev->mdev) - 1;
-
- mlx5_remove_netdev_notifier(dev, port_num);
-}
-
-static int mlx5_ib_stage_raw_eth_roce_init(struct mlx5_ib_dev *dev)
+static int mlx5_ib_roce_init(struct mlx5_ib_dev *dev)
{
struct mlx5_core_dev *mdev = dev->mdev;
enum rdma_link_layer ll;
int port_type_cap;
- int err = 0;
-
- port_type_cap = MLX5_CAP_GEN(mdev, port_type);
- ll = mlx5_port_type_cap_to_rdma_ll(port_type_cap);
-
- if (ll == IB_LINK_LAYER_ETHERNET)
- err = mlx5_ib_stage_common_roce_init(dev);
-
- return err;
-}
-
-static void mlx5_ib_stage_raw_eth_roce_cleanup(struct mlx5_ib_dev *dev)
-{
- mlx5_ib_stage_common_roce_cleanup(dev);
-}
-
-static int mlx5_ib_stage_roce_init(struct mlx5_ib_dev *dev)
-{
- struct mlx5_core_dev *mdev = dev->mdev;
- enum rdma_link_layer ll;
- int port_type_cap;
+ u8 port_num = 0;
int err;
port_type_cap = MLX5_CAP_GEN(mdev, port_type);
ll = mlx5_port_type_cap_to_rdma_ll(port_type_cap);
if (ll == IB_LINK_LAYER_ETHERNET) {
- err = mlx5_ib_stage_common_roce_init(dev);
- if (err)
+ dev->ib_dev.uverbs_ex_cmd_mask |=
+ (1ull << IB_USER_VERBS_EX_CMD_CREATE_WQ) |
+ (1ull << IB_USER_VERBS_EX_CMD_MODIFY_WQ) |
+ (1ull << IB_USER_VERBS_EX_CMD_DESTROY_WQ) |
+ (1ull << IB_USER_VERBS_EX_CMD_CREATE_RWQ_IND_TBL) |
+ (1ull << IB_USER_VERBS_EX_CMD_DESTROY_RWQ_IND_TBL);
+ ib_set_device_ops(&dev->ib_dev, &mlx5_ib_dev_common_roce_ops);
+
+ port_num = mlx5_core_native_port_num(dev->mdev) - 1;
+
+ /* Register only for native ports */
+ err = mlx5_add_netdev_notifier(dev, port_num);
+ if (err || dev->is_rep || !mlx5_is_roce_enabled(mdev))
+ /*
+ * We don't enable ETH interface for
+ * 1. IB representors
+ * 2. User disabled ROCE through devlink interface
+ */
return err;
err = mlx5_enable_eth(dev);
@@ -6897,73 +4309,29 @@
return 0;
cleanup:
- mlx5_ib_stage_common_roce_cleanup(dev);
-
+ mlx5_remove_netdev_notifier(dev, port_num);
return err;
}
-static void mlx5_ib_stage_roce_cleanup(struct mlx5_ib_dev *dev)
+static void mlx5_ib_roce_cleanup(struct mlx5_ib_dev *dev)
{
struct mlx5_core_dev *mdev = dev->mdev;
enum rdma_link_layer ll;
int port_type_cap;
+ u8 port_num;
port_type_cap = MLX5_CAP_GEN(mdev, port_type);
ll = mlx5_port_type_cap_to_rdma_ll(port_type_cap);
if (ll == IB_LINK_LAYER_ETHERNET) {
- mlx5_disable_eth(dev);
- mlx5_ib_stage_common_roce_cleanup(dev);
+ if (!dev->is_rep)
+ mlx5_disable_eth(dev);
+
+ port_num = mlx5_core_native_port_num(dev->mdev) - 1;
+ mlx5_remove_netdev_notifier(dev, port_num);
}
}
-static int mlx5_ib_stage_dev_res_init(struct mlx5_ib_dev *dev)
-{
- return create_dev_resources(&dev->devr);
-}
-
-static void mlx5_ib_stage_dev_res_cleanup(struct mlx5_ib_dev *dev)
-{
- destroy_dev_resources(&dev->devr);
-}
-
-static int mlx5_ib_stage_odp_init(struct mlx5_ib_dev *dev)
-{
- return mlx5_ib_odp_init_one(dev);
-}
-
-static void mlx5_ib_stage_odp_cleanup(struct mlx5_ib_dev *dev)
-{
- mlx5_ib_odp_cleanup_one(dev);
-}
-
-static const struct ib_device_ops mlx5_ib_dev_hw_stats_ops = {
- .alloc_hw_stats = mlx5_ib_alloc_hw_stats,
- .get_hw_stats = mlx5_ib_get_hw_stats,
- .counter_bind_qp = mlx5_ib_counter_bind_qp,
- .counter_unbind_qp = mlx5_ib_counter_unbind_qp,
- .counter_dealloc = mlx5_ib_counter_dealloc,
- .counter_alloc_stats = mlx5_ib_counter_alloc_stats,
- .counter_update_stats = mlx5_ib_counter_update_stats,
-};
-
-static int mlx5_ib_stage_counters_init(struct mlx5_ib_dev *dev)
-{
- if (MLX5_CAP_GEN(dev->mdev, max_qp_cnt)) {
- ib_set_device_ops(&dev->ib_dev, &mlx5_ib_dev_hw_stats_ops);
-
- return mlx5_ib_alloc_counters(dev);
- }
-
- return 0;
-}
-
-static void mlx5_ib_stage_counters_cleanup(struct mlx5_ib_dev *dev)
-{
- if (MLX5_CAP_GEN(dev->mdev, max_qp_cnt))
- mlx5_ib_dealloc_counters(dev);
-}
-
static int mlx5_ib_stage_cong_debugfs_init(struct mlx5_ib_dev *dev)
{
mlx5_ib_init_cong_debugfs(dev,
@@ -7023,7 +4391,18 @@
static void mlx5_ib_stage_pre_ib_reg_umr_cleanup(struct mlx5_ib_dev *dev)
{
- destroy_umrc_res(dev);
+ int err;
+
+ err = mlx5_mr_cache_cleanup(dev);
+ if (err)
+ mlx5_ib_warn(dev, "mr cache cleanup failed\n");
+
+ if (dev->umrc.qp)
+ mlx5_ib_destroy_qp(dev->umrc.qp, NULL);
+ if (dev->umrc.cq)
+ ib_free_cq(dev->umrc.cq);
+ if (dev->umrc.pd)
+ ib_dealloc_pd(dev->umrc.pd);
}
static void mlx5_ib_stage_ib_reg_cleanup(struct mlx5_ib_dev *dev)
@@ -7031,21 +4410,162 @@
ib_unregister_device(&dev->ib_dev);
}
+enum {
+ MAX_UMR_WR = 128,
+};
+
static int mlx5_ib_stage_post_ib_reg_umr_init(struct mlx5_ib_dev *dev)
{
- return create_umr_res(dev);
+ struct ib_qp_init_attr *init_attr = NULL;
+ struct ib_qp_attr *attr = NULL;
+ struct ib_pd *pd;
+ struct ib_cq *cq;
+ struct ib_qp *qp;
+ int ret;
+
+ attr = kzalloc(sizeof(*attr), GFP_KERNEL);
+ init_attr = kzalloc(sizeof(*init_attr), GFP_KERNEL);
+ if (!attr || !init_attr) {
+ ret = -ENOMEM;
+ goto error_0;
+ }
+
+ pd = ib_alloc_pd(&dev->ib_dev, 0);
+ if (IS_ERR(pd)) {
+ mlx5_ib_dbg(dev, "Couldn't create PD for sync UMR QP\n");
+ ret = PTR_ERR(pd);
+ goto error_0;
+ }
+
+ cq = ib_alloc_cq(&dev->ib_dev, NULL, 128, 0, IB_POLL_SOFTIRQ);
+ if (IS_ERR(cq)) {
+ mlx5_ib_dbg(dev, "Couldn't create CQ for sync UMR QP\n");
+ ret = PTR_ERR(cq);
+ goto error_2;
+ }
+
+ init_attr->send_cq = cq;
+ init_attr->recv_cq = cq;
+ init_attr->sq_sig_type = IB_SIGNAL_ALL_WR;
+ init_attr->cap.max_send_wr = MAX_UMR_WR;
+ init_attr->cap.max_send_sge = 1;
+ init_attr->qp_type = MLX5_IB_QPT_REG_UMR;
+ init_attr->port_num = 1;
+ qp = mlx5_ib_create_qp(pd, init_attr, NULL);
+ if (IS_ERR(qp)) {
+ mlx5_ib_dbg(dev, "Couldn't create sync UMR QP\n");
+ ret = PTR_ERR(qp);
+ goto error_3;
+ }
+ qp->device = &dev->ib_dev;
+ qp->real_qp = qp;
+ qp->uobject = NULL;
+ qp->qp_type = MLX5_IB_QPT_REG_UMR;
+ qp->send_cq = init_attr->send_cq;
+ qp->recv_cq = init_attr->recv_cq;
+
+ attr->qp_state = IB_QPS_INIT;
+ attr->port_num = 1;
+ ret = mlx5_ib_modify_qp(qp, attr, IB_QP_STATE | IB_QP_PKEY_INDEX |
+ IB_QP_PORT, NULL);
+ if (ret) {
+ mlx5_ib_dbg(dev, "Couldn't modify UMR QP\n");
+ goto error_4;
+ }
+
+ memset(attr, 0, sizeof(*attr));
+ attr->qp_state = IB_QPS_RTR;
+ attr->path_mtu = IB_MTU_256;
+
+ ret = mlx5_ib_modify_qp(qp, attr, IB_QP_STATE, NULL);
+ if (ret) {
+ mlx5_ib_dbg(dev, "Couldn't modify umr QP to rtr\n");
+ goto error_4;
+ }
+
+ memset(attr, 0, sizeof(*attr));
+ attr->qp_state = IB_QPS_RTS;
+ ret = mlx5_ib_modify_qp(qp, attr, IB_QP_STATE, NULL);
+ if (ret) {
+ mlx5_ib_dbg(dev, "Couldn't modify umr QP to rts\n");
+ goto error_4;
+ }
+
+ dev->umrc.qp = qp;
+ dev->umrc.cq = cq;
+ dev->umrc.pd = pd;
+
+ sema_init(&dev->umrc.sem, MAX_UMR_WR);
+ ret = mlx5_mr_cache_init(dev);
+ if (ret) {
+ mlx5_ib_warn(dev, "mr cache init failed %d\n", ret);
+ goto error_4;
+ }
+
+ kfree(attr);
+ kfree(init_attr);
+
+ return 0;
+
+error_4:
+ mlx5_ib_destroy_qp(qp, NULL);
+ dev->umrc.qp = NULL;
+
+error_3:
+ ib_free_cq(cq);
+ dev->umrc.cq = NULL;
+
+error_2:
+ ib_dealloc_pd(pd);
+ dev->umrc.pd = NULL;
+
+error_0:
+ kfree(attr);
+ kfree(init_attr);
+ return ret;
}
static int mlx5_ib_stage_delay_drop_init(struct mlx5_ib_dev *dev)
{
- init_delay_drop(dev);
+ struct dentry *root;
+ if (!(dev->ib_dev.attrs.raw_packet_caps & IB_RAW_PACKET_CAP_DELAY_DROP))
+ return 0;
+
+ mutex_init(&dev->delay_drop.lock);
+ dev->delay_drop.dev = dev;
+ dev->delay_drop.activate = false;
+ dev->delay_drop.timeout = MLX5_MAX_DELAY_DROP_TIMEOUT_MS * 1000;
+ INIT_WORK(&dev->delay_drop.delay_drop_work, delay_drop_handler);
+ atomic_set(&dev->delay_drop.rqs_cnt, 0);
+ atomic_set(&dev->delay_drop.events_cnt, 0);
+
+ if (!mlx5_debugfs_root)
+ return 0;
+
+ root = debugfs_create_dir("delay_drop", dev->mdev->priv.dbg_root);
+ dev->delay_drop.dir_debugfs = root;
+
+ debugfs_create_atomic_t("num_timeout_events", 0400, root,
+ &dev->delay_drop.events_cnt);
+ debugfs_create_atomic_t("num_rqs", 0400, root,
+ &dev->delay_drop.rqs_cnt);
+ debugfs_create_file("timeout", 0600, root, &dev->delay_drop,
+ &fops_delay_drop_timeout);
return 0;
}
static void mlx5_ib_stage_delay_drop_cleanup(struct mlx5_ib_dev *dev)
{
- cancel_delay_drop(dev);
+ if (!(dev->ib_dev.attrs.raw_packet_caps & IB_RAW_PACKET_CAP_DELAY_DROP))
+ return;
+
+ cancel_work_sync(&dev->delay_drop.delay_drop_work);
+ if (!dev->delay_drop.dir_debugfs)
+ return;
+
+ debugfs_remove_recursive(dev->delay_drop.dir_debugfs);
+ dev->delay_drop.dir_debugfs = NULL;
}
static int mlx5_ib_stage_dev_notifier_init(struct mlx5_ib_dev *dev)
@@ -7060,38 +4580,6 @@
mlx5_notifier_unregister(dev->mdev, &dev->mdev_events);
}
-static int mlx5_ib_stage_devx_init(struct mlx5_ib_dev *dev)
-{
- int uid;
-
- uid = mlx5_ib_devx_create(dev, false);
- if (uid > 0) {
- dev->devx_whitelist_uid = uid;
- mlx5_ib_devx_init_event_table(dev);
- }
-
- return 0;
-}
-static void mlx5_ib_stage_devx_cleanup(struct mlx5_ib_dev *dev)
-{
- if (dev->devx_whitelist_uid) {
- mlx5_ib_devx_cleanup_event_table(dev);
- mlx5_ib_devx_destroy(dev, dev->devx_whitelist_uid);
- }
-}
-
-int mlx5_ib_enable_driver(struct ib_device *dev)
-{
- struct mlx5_ib_dev *mdev = to_mdev(dev);
- int ret;
-
- ret = mlx5_ib_test_wc(mdev);
- mlx5_ib_dbg(mdev, "Write-Combining %s",
- mdev->wc_support ? "supported" : "not supported");
-
- return ret;
-}
-
void __mlx5_ib_remove(struct mlx5_ib_dev *dev,
const struct mlx5_ib_profile *profile,
int stage)
@@ -7139,9 +4627,9 @@
STAGE_CREATE(MLX5_IB_STAGE_INIT,
mlx5_ib_stage_init_init,
mlx5_ib_stage_init_cleanup),
- STAGE_CREATE(MLX5_IB_STAGE_FLOW_DB,
- mlx5_ib_stage_flow_db_init,
- mlx5_ib_stage_flow_db_cleanup),
+ STAGE_CREATE(MLX5_IB_STAGE_FS,
+ mlx5_ib_fs_init,
+ mlx5_ib_fs_cleanup),
STAGE_CREATE(MLX5_IB_STAGE_CAPS,
mlx5_ib_stage_caps_init,
mlx5_ib_stage_caps_cleanup),
@@ -7149,8 +4637,8 @@
mlx5_ib_stage_non_default_cb,
NULL),
STAGE_CREATE(MLX5_IB_STAGE_ROCE,
- mlx5_ib_stage_roce_init,
- mlx5_ib_stage_roce_cleanup),
+ mlx5_ib_roce_init,
+ mlx5_ib_roce_cleanup),
STAGE_CREATE(MLX5_IB_STAGE_QP,
mlx5_init_qp_table,
mlx5_cleanup_qp_table),
@@ -7158,17 +4646,17 @@
mlx5_init_srq_table,
mlx5_cleanup_srq_table),
STAGE_CREATE(MLX5_IB_STAGE_DEVICE_RESOURCES,
- mlx5_ib_stage_dev_res_init,
- mlx5_ib_stage_dev_res_cleanup),
+ mlx5_ib_dev_res_init,
+ mlx5_ib_dev_res_cleanup),
STAGE_CREATE(MLX5_IB_STAGE_DEVICE_NOTIFIER,
mlx5_ib_stage_dev_notifier_init,
mlx5_ib_stage_dev_notifier_cleanup),
STAGE_CREATE(MLX5_IB_STAGE_ODP,
- mlx5_ib_stage_odp_init,
- mlx5_ib_stage_odp_cleanup),
+ mlx5_ib_odp_init_one,
+ mlx5_ib_odp_cleanup_one),
STAGE_CREATE(MLX5_IB_STAGE_COUNTERS,
- mlx5_ib_stage_counters_init,
- mlx5_ib_stage_counters_cleanup),
+ mlx5_ib_counters_init,
+ mlx5_ib_counters_cleanup),
STAGE_CREATE(MLX5_IB_STAGE_CONG_DEBUGFS,
mlx5_ib_stage_cong_debugfs_init,
mlx5_ib_stage_cong_debugfs_cleanup),
@@ -7182,8 +4670,8 @@
NULL,
mlx5_ib_stage_pre_ib_reg_umr_cleanup),
STAGE_CREATE(MLX5_IB_STAGE_WHITELIST_UID,
- mlx5_ib_stage_devx_init,
- mlx5_ib_stage_devx_cleanup),
+ mlx5_ib_devx_init,
+ mlx5_ib_devx_cleanup),
STAGE_CREATE(MLX5_IB_STAGE_IB_REG,
mlx5_ib_stage_ib_reg_init,
mlx5_ib_stage_ib_reg_cleanup),
@@ -7193,15 +4681,18 @@
STAGE_CREATE(MLX5_IB_STAGE_DELAY_DROP,
mlx5_ib_stage_delay_drop_init,
mlx5_ib_stage_delay_drop_cleanup),
+ STAGE_CREATE(MLX5_IB_STAGE_RESTRACK,
+ mlx5_ib_restrack_init,
+ NULL),
};
const struct mlx5_ib_profile raw_eth_profile = {
STAGE_CREATE(MLX5_IB_STAGE_INIT,
mlx5_ib_stage_init_init,
mlx5_ib_stage_init_cleanup),
- STAGE_CREATE(MLX5_IB_STAGE_FLOW_DB,
- mlx5_ib_stage_flow_db_init,
- mlx5_ib_stage_flow_db_cleanup),
+ STAGE_CREATE(MLX5_IB_STAGE_FS,
+ mlx5_ib_fs_init,
+ mlx5_ib_fs_cleanup),
STAGE_CREATE(MLX5_IB_STAGE_CAPS,
mlx5_ib_stage_caps_init,
mlx5_ib_stage_caps_cleanup),
@@ -7209,8 +4700,8 @@
mlx5_ib_stage_raw_eth_non_default_cb,
NULL),
STAGE_CREATE(MLX5_IB_STAGE_ROCE,
- mlx5_ib_stage_raw_eth_roce_init,
- mlx5_ib_stage_raw_eth_roce_cleanup),
+ mlx5_ib_roce_init,
+ mlx5_ib_roce_cleanup),
STAGE_CREATE(MLX5_IB_STAGE_QP,
mlx5_init_qp_table,
mlx5_cleanup_qp_table),
@@ -7218,14 +4709,14 @@
mlx5_init_srq_table,
mlx5_cleanup_srq_table),
STAGE_CREATE(MLX5_IB_STAGE_DEVICE_RESOURCES,
- mlx5_ib_stage_dev_res_init,
- mlx5_ib_stage_dev_res_cleanup),
+ mlx5_ib_dev_res_init,
+ mlx5_ib_dev_res_cleanup),
STAGE_CREATE(MLX5_IB_STAGE_DEVICE_NOTIFIER,
mlx5_ib_stage_dev_notifier_init,
mlx5_ib_stage_dev_notifier_cleanup),
STAGE_CREATE(MLX5_IB_STAGE_COUNTERS,
- mlx5_ib_stage_counters_init,
- mlx5_ib_stage_counters_cleanup),
+ mlx5_ib_counters_init,
+ mlx5_ib_counters_cleanup),
STAGE_CREATE(MLX5_IB_STAGE_CONG_DEBUGFS,
mlx5_ib_stage_cong_debugfs_init,
mlx5_ib_stage_cong_debugfs_cleanup),
@@ -7239,14 +4730,17 @@
NULL,
mlx5_ib_stage_pre_ib_reg_umr_cleanup),
STAGE_CREATE(MLX5_IB_STAGE_WHITELIST_UID,
- mlx5_ib_stage_devx_init,
- mlx5_ib_stage_devx_cleanup),
+ mlx5_ib_devx_init,
+ mlx5_ib_devx_cleanup),
STAGE_CREATE(MLX5_IB_STAGE_IB_REG,
mlx5_ib_stage_ib_reg_init,
mlx5_ib_stage_ib_reg_cleanup),
STAGE_CREATE(MLX5_IB_STAGE_POST_IB_REG_UMR,
mlx5_ib_stage_post_ib_reg_umr_init,
NULL),
+ STAGE_CREATE(MLX5_IB_STAGE_RESTRACK,
+ mlx5_ib_restrack_init,
+ NULL),
};
static void *mlx5_ib_add_slave_port(struct mlx5_core_dev *mdev)
diff --git a/drivers/infiniband/hw/mlx5/mlx5_ib.h b/drivers/infiniband/hw/mlx5/mlx5_ib.h
index 5dbe3eb..5287fc8 100644
--- a/drivers/infiniband/hw/mlx5/mlx5_ib.h
+++ b/drivers/infiniband/hw/mlx5/mlx5_ib.h
@@ -1,33 +1,6 @@
+/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */
/*
- * Copyright (c) 2013-2015, Mellanox Technologies. All rights reserved.
- *
- * This software is available to you under a choice of one of two
- * licenses. You may choose to be licensed under the terms of the GNU
- * General Public License (GPL) Version 2, available from the file
- * COPYING in the main directory of this source tree, or the
- * OpenIB.org BSD license below:
- *
- * Redistribution and use in source and binary forms, with or
- * without modification, are permitted provided that the following
- * conditions are met:
- *
- * - Redistributions of source code must retain the above
- * copyright notice, this list of conditions and the following
- * disclaimer.
- *
- * - Redistributions in binary form must reproduce the above
- * copyright notice, this list of conditions and the following
- * disclaimer in the documentation and/or other materials
- * provided with the distribution.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
- * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
- * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
- * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
- * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
- * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
- * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
+ * Copyright (c) 2013-2020, Mellanox Technologies inc. All rights reserved.
*/
#ifndef MLX5_IB_H
@@ -730,8 +703,8 @@
struct mlx5_ib_resources {
struct ib_cq *c0;
- struct ib_xrcd *x0;
- struct ib_xrcd *x1;
+ u32 xrcdn0;
+ u32 xrcdn1;
struct ib_pd *p0;
struct ib_srq *s0;
struct ib_srq *s1;
@@ -832,7 +805,7 @@
enum mlx5_ib_stages {
MLX5_IB_STAGE_INIT,
- MLX5_IB_STAGE_FLOW_DB,
+ MLX5_IB_STAGE_FS,
MLX5_IB_STAGE_CAPS,
MLX5_IB_STAGE_NON_DEFAULT_CB,
MLX5_IB_STAGE_ROCE,
@@ -850,7 +823,7 @@
MLX5_IB_STAGE_IB_REG,
MLX5_IB_STAGE_POST_IB_REG_UMR,
MLX5_IB_STAGE_DELAY_DROP,
- MLX5_IB_STAGE_CLASS_ATTR,
+ MLX5_IB_STAGE_RESTRACK,
MLX5_IB_STAGE_MAX,
};
@@ -1078,11 +1051,6 @@
return container_of(core_qp, struct mlx5_ib_rwq, core_qp);
}
-static inline struct mlx5_ib_mr *to_mibmr(struct mlx5_core_mkey *mmkey)
-{
- return container_of(mmkey, struct mlx5_ib_mr, mmkey);
-}
-
static inline struct mlx5_ib_pd *to_mpd(struct ib_pd *ibpd)
{
return container_of(ibpd, struct mlx5_ib_pd, ibpd);
@@ -1210,7 +1178,7 @@
struct ib_pd *pd, struct ib_udata *udata);
int mlx5_ib_dereg_mr(struct ib_mr *ibmr, struct ib_udata *udata);
struct ib_mr *mlx5_ib_alloc_mr(struct ib_pd *pd, enum ib_mr_type mr_type,
- u32 max_num_sg, struct ib_udata *udata);
+ u32 max_num_sg);
struct ib_mr *mlx5_ib_alloc_mr_integrity(struct ib_pd *pd,
u32 max_num_sg,
u32 max_num_meta_sg);
@@ -1224,9 +1192,8 @@
const struct ib_wc *in_wc, const struct ib_grh *in_grh,
const struct ib_mad *in, struct ib_mad *out,
size_t *out_mad_size, u16 *out_mad_pkey_index);
-struct ib_xrcd *mlx5_ib_alloc_xrcd(struct ib_device *ibdev,
- struct ib_udata *udata);
-int mlx5_ib_dealloc_xrcd(struct ib_xrcd *xrcd, struct ib_udata *udata);
+int mlx5_ib_alloc_xrcd(struct ib_xrcd *xrcd, struct ib_udata *udata);
+void mlx5_ib_dealloc_xrcd(struct ib_xrcd *xrcd, struct ib_udata *udata);
int mlx5_ib_get_buf_offset(u64 addr, int page_shift, u32 *offset);
int mlx5_query_ext_port_caps(struct mlx5_ib_dev *dev, u8 port);
int mlx5_query_mad_ifc_smp_attr_node_info(struct ib_device *ibdev,
@@ -1375,46 +1342,12 @@
u8 *native_port_num);
void mlx5_ib_put_native_port_mdev(struct mlx5_ib_dev *dev,
u8 port_num);
-int mlx5_ib_fill_res_entry(struct sk_buff *msg,
- struct rdma_restrack_entry *res);
-int mlx5_ib_fill_stat_entry(struct sk_buff *msg,
- struct rdma_restrack_entry *res);
extern const struct uapi_definition mlx5_ib_devx_defs[];
extern const struct uapi_definition mlx5_ib_flow_defs[];
extern const struct uapi_definition mlx5_ib_qos_defs[];
+extern const struct uapi_definition mlx5_ib_std_types_defs[];
-#if IS_ENABLED(CONFIG_INFINIBAND_USER_ACCESS)
-int mlx5_ib_devx_create(struct mlx5_ib_dev *dev, bool is_user);
-void mlx5_ib_devx_destroy(struct mlx5_ib_dev *dev, u16 uid);
-void mlx5_ib_devx_init_event_table(struct mlx5_ib_dev *dev);
-void mlx5_ib_devx_cleanup_event_table(struct mlx5_ib_dev *dev);
-struct mlx5_ib_flow_handler *mlx5_ib_raw_fs_rule_add(
- struct mlx5_ib_dev *dev, struct mlx5_ib_flow_matcher *fs_matcher,
- struct mlx5_flow_context *flow_context,
- struct mlx5_flow_act *flow_act, u32 counter_id,
- void *cmd_in, int inlen, int dest_id, int dest_type);
-bool mlx5_ib_devx_is_flow_dest(void *obj, int *dest_id, int *dest_type);
-bool mlx5_ib_devx_is_flow_counter(void *obj, u32 offset, u32 *counter_id);
-void mlx5_ib_destroy_flow_action_raw(struct mlx5_ib_flow_action *maction);
-#else
-static inline int
-mlx5_ib_devx_create(struct mlx5_ib_dev *dev,
- bool is_user) { return -EOPNOTSUPP; }
-static inline void mlx5_ib_devx_destroy(struct mlx5_ib_dev *dev, u16 uid) {}
-static inline void mlx5_ib_devx_init_event_table(struct mlx5_ib_dev *dev) {}
-static inline void mlx5_ib_devx_cleanup_event_table(struct mlx5_ib_dev *dev) {}
-static inline bool mlx5_ib_devx_is_flow_dest(void *obj, int *dest_id,
- int *dest_type)
-{
- return false;
-}
-static inline void
-mlx5_ib_destroy_flow_action_raw(struct mlx5_ib_flow_action *maction)
-{
- return;
-};
-#endif
static inline void init_query_mad(struct ib_smp *mad)
{
mad->base_version = 1;
@@ -1423,15 +1356,6 @@
mad->method = IB_MGMT_METHOD_GET;
}
-static inline u8 convert_access(int acc)
-{
- return (acc & IB_ACCESS_REMOTE_ATOMIC ? MLX5_PERM_ATOMIC : 0) |
- (acc & IB_ACCESS_REMOTE_WRITE ? MLX5_PERM_REMOTE_WRITE : 0) |
- (acc & IB_ACCESS_REMOTE_READ ? MLX5_PERM_REMOTE_READ : 0) |
- (acc & IB_ACCESS_LOCAL_WRITE ? MLX5_PERM_LOCAL_WRITE : 0) |
- MLX5_PERM_LOCAL_READ;
-}
-
static inline int is_qp1(enum ib_qp_type qp_type)
{
return qp_type == MLX5_IB_QPT_HW_GSI;
@@ -1518,9 +1442,6 @@
struct mlx5_bfreg_info *bfregi, u32 bfregn,
bool dyn_bfreg);
-int mlx5_ib_qp_set_counter(struct ib_qp *qp, struct rdma_counter *counter);
-u16 mlx5_ib_get_counters_id(struct mlx5_ib_dev *dev, u8 port_num);
-
static inline bool mlx5_ib_can_use_umr(struct mlx5_ib_dev *dev,
bool do_modify_atomic, int access_flags)
{
@@ -1533,14 +1454,18 @@
return false;
if (access_flags & IB_ACCESS_RELAXED_ORDERING &&
- (MLX5_CAP_GEN(dev->mdev, relaxed_ordering_write) ||
- MLX5_CAP_GEN(dev->mdev, relaxed_ordering_read)))
+ MLX5_CAP_GEN(dev->mdev, relaxed_ordering_write) &&
+ !MLX5_CAP_GEN(dev->mdev, relaxed_ordering_write_umr))
+ return false;
+
+ if (access_flags & IB_ACCESS_RELAXED_ORDERING &&
+ MLX5_CAP_GEN(dev->mdev, relaxed_ordering_read) &&
+ !MLX5_CAP_GEN(dev->mdev, relaxed_ordering_read_umr))
return false;
return true;
}
-int mlx5_ib_enable_driver(struct ib_device *dev);
int mlx5_ib_test_wc(struct mlx5_ib_dev *dev);
static inline bool mlx5_ib_lag_should_assign_affinity(struct mlx5_ib_dev *dev)
diff --git a/drivers/infiniband/hw/mlx5/mr.c b/drivers/infiniband/hw/mlx5/mr.c
index 4468307..3e6f2f9 100644
--- a/drivers/infiniband/hw/mlx5/mr.c
+++ b/drivers/infiniband/hw/mlx5/mr.c
@@ -1961,7 +1961,7 @@
}
struct ib_mr *mlx5_ib_alloc_mr(struct ib_pd *pd, enum ib_mr_type mr_type,
- u32 max_num_sg, struct ib_udata *udata)
+ u32 max_num_sg)
{
return __mlx5_ib_alloc_mr(pd, mr_type, max_num_sg, 0);
}
diff --git a/drivers/infiniband/hw/mlx5/odp.c b/drivers/infiniband/hw/mlx5/odp.c
index 77dca1e..cfd7efa 100644
--- a/drivers/infiniband/hw/mlx5/odp.c
+++ b/drivers/infiniband/hw/mlx5/odp.c
@@ -816,6 +816,7 @@
{
struct ib_umem_odp *odp = to_ib_umem_odp(mr->umem);
+ lockdep_assert_held(&mr->dev->odp_srcu);
if (unlikely(io_virt < mr->mmkey.iova))
return -EFAULT;
@@ -929,11 +930,6 @@
if (ret < 0)
goto srcu_unlock;
- /*
- * When prefetching a page, page fault is generated
- * in order to bring the page to the main memory.
- * In the current flow, page faults are being counted.
- */
mlx5_update_odp_stats(mr, faults, ret);
npages += ret;
@@ -1770,13 +1766,26 @@
{
struct prefetch_mr_work *work =
container_of(w, struct prefetch_mr_work, work);
+ struct mlx5_ib_dev *dev;
u32 bytes_mapped = 0;
+ int srcu_key;
+ int ret;
u32 i;
- for (i = 0; i < work->num_sge; ++i)
- pagefault_mr(work->frags[i].mr, work->frags[i].io_virt,
- work->frags[i].length, &bytes_mapped,
- work->pf_flags);
+ /* We rely on IB/core that work is executed if we have num_sge != 0 only. */
+ WARN_ON(!work->num_sge);
+ dev = work->frags[0].mr->dev;
+ /* SRCU should be held when calling to mlx5_odp_populate_xlt() */
+ srcu_key = srcu_read_lock(&dev->odp_srcu);
+ for (i = 0; i < work->num_sge; ++i) {
+ ret = pagefault_mr(work->frags[i].mr, work->frags[i].io_virt,
+ work->frags[i].length, &bytes_mapped,
+ work->pf_flags);
+ if (ret <= 0)
+ continue;
+ mlx5_update_odp_stats(work->frags[i].mr, prefetch, ret);
+ }
+ srcu_read_unlock(&dev->odp_srcu, srcu_key);
destroy_prefetch_work(work);
}
@@ -1832,6 +1841,7 @@
&bytes_mapped, pf_flags);
if (ret < 0)
goto out;
+ mlx5_update_odp_stats(mr, prefetch, ret);
}
ret = 0;
diff --git a/drivers/infiniband/hw/mlx5/qp.c b/drivers/infiniband/hw/mlx5/qp.c
index 1225b8d..59fce5f 100644
--- a/drivers/infiniband/hw/mlx5/qp.c
+++ b/drivers/infiniband/hw/mlx5/qp.c
@@ -38,6 +38,7 @@
#include <linux/mlx5/fs.h>
#include "mlx5_ib.h"
#include "ib_rep.h"
+#include "counters.h"
#include "cmd.h"
#include "qp.h"
#include "wr.h"
@@ -2031,15 +2032,15 @@
switch (init_attr->qp_type) {
case IB_QPT_XRC_INI:
MLX5_SET(qpc, qpc, cqn_rcv, to_mcq(devr->c0)->mcq.cqn);
- MLX5_SET(qpc, qpc, xrcd, to_mxrcd(devr->x1)->xrcdn);
+ MLX5_SET(qpc, qpc, xrcd, devr->xrcdn1);
MLX5_SET(qpc, qpc, srqn_rmpn_xrqn, to_msrq(devr->s0)->msrq.srqn);
break;
default:
if (init_attr->srq) {
- MLX5_SET(qpc, qpc, xrcd, to_mxrcd(devr->x0)->xrcdn);
+ MLX5_SET(qpc, qpc, xrcd, devr->xrcdn0);
MLX5_SET(qpc, qpc, srqn_rmpn_xrqn, to_msrq(init_attr->srq)->msrq.srqn);
} else {
- MLX5_SET(qpc, qpc, xrcd, to_mxrcd(devr->x1)->xrcdn);
+ MLX5_SET(qpc, qpc, xrcd, devr->xrcdn1);
MLX5_SET(qpc, qpc, srqn_rmpn_xrqn, to_msrq(devr->s1)->msrq.srqn);
}
}
@@ -2178,11 +2179,11 @@
MLX5_SET(qpc, qpc, no_sq, 1);
if (attr->srq) {
- MLX5_SET(qpc, qpc, xrcd, to_mxrcd(devr->x0)->xrcdn);
+ MLX5_SET(qpc, qpc, xrcd, devr->xrcdn0);
MLX5_SET(qpc, qpc, srqn_rmpn_xrqn,
to_msrq(attr->srq)->msrq.srqn);
} else {
- MLX5_SET(qpc, qpc, xrcd, to_mxrcd(devr->x1)->xrcdn);
+ MLX5_SET(qpc, qpc, xrcd, devr->xrcdn1);
MLX5_SET(qpc, qpc, srqn_rmpn_xrqn,
to_msrq(devr->s1)->msrq.srqn);
}
@@ -3554,7 +3555,7 @@
switch (raw_qp_param->operation) {
case MLX5_CMD_OP_RST2INIT_QP:
rq_state = MLX5_RQC_STATE_RDY;
- sq_state = MLX5_SQC_STATE_RDY;
+ sq_state = MLX5_SQC_STATE_RST;
break;
case MLX5_CMD_OP_2ERR_QP:
rq_state = MLX5_RQC_STATE_ERR;
@@ -3566,13 +3567,11 @@
break;
case MLX5_CMD_OP_RTR2RTS_QP:
case MLX5_CMD_OP_RTS2RTS_QP:
- if (raw_qp_param->set_mask ==
- MLX5_RAW_QP_RATE_LIMIT) {
- modify_rq = 0;
- sq_state = sq->state;
- } else {
- return raw_qp_param->set_mask ? -EINVAL : 0;
- }
+ if (raw_qp_param->set_mask & ~MLX5_RAW_QP_RATE_LIMIT)
+ return -EINVAL;
+
+ modify_rq = 0;
+ sq_state = MLX5_SQC_STATE_RDY;
break;
case MLX5_CMD_OP_INIT2INIT_QP:
case MLX5_CMD_OP_INIT2RTR_QP:
@@ -4114,9 +4113,9 @@
struct mlx5_ib_qp *qp = to_mqp(ibqp);
struct mlx5_ib_dev *dev = to_mdev(ibqp->device);
enum ib_qp_state cur_state, new_state;
- int err = 0;
int required = IB_QP_STATE;
void *dctc;
+ int err;
if (!(attr_mask & IB_QP_STATE))
return -EINVAL;
@@ -4208,11 +4207,9 @@
mlx5_ib_warn(dev, "Modify DCT: Invalid transition from %d to %d\n", cur_state, new_state);
return -EINVAL;
}
- if (err)
- qp->state = IB_QPS_ERR;
- else
- qp->state = new_state;
- return err;
+
+ qp->state = new_state;
+ return 0;
}
int mlx5_ib_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
@@ -4450,7 +4447,7 @@
[MLX5_SQ_STATE_NA] = IB_QPS_RESET,
},
[MLX5_RQC_STATE_RDY] = {
- [MLX5_SQC_STATE_RST] = MLX5_QP_STATE_BAD,
+ [MLX5_SQC_STATE_RST] = MLX5_QP_STATE,
[MLX5_SQC_STATE_RDY] = MLX5_QP_STATE,
[MLX5_SQC_STATE_ERR] = IB_QPS_SQE,
[MLX5_SQ_STATE_NA] = MLX5_QP_STATE,
@@ -4462,7 +4459,7 @@
[MLX5_SQ_STATE_NA] = IB_QPS_ERR,
},
[MLX5_RQ_STATE_NA] = {
- [MLX5_SQC_STATE_RST] = IB_QPS_RESET,
+ [MLX5_SQC_STATE_RST] = MLX5_QP_STATE,
[MLX5_SQC_STATE_RDY] = MLX5_QP_STATE,
[MLX5_SQC_STATE_ERR] = MLX5_QP_STATE,
[MLX5_SQ_STATE_NA] = MLX5_QP_STATE_BAD,
@@ -4708,41 +4705,23 @@
return err;
}
-struct ib_xrcd *mlx5_ib_alloc_xrcd(struct ib_device *ibdev,
- struct ib_udata *udata)
+int mlx5_ib_alloc_xrcd(struct ib_xrcd *ibxrcd, struct ib_udata *udata)
{
- struct mlx5_ib_dev *dev = to_mdev(ibdev);
- struct mlx5_ib_xrcd *xrcd;
- int err;
+ struct mlx5_ib_dev *dev = to_mdev(ibxrcd->device);
+ struct mlx5_ib_xrcd *xrcd = to_mxrcd(ibxrcd);
if (!MLX5_CAP_GEN(dev->mdev, xrc))
- return ERR_PTR(-ENOSYS);
+ return -EOPNOTSUPP;
- xrcd = kmalloc(sizeof(*xrcd), GFP_KERNEL);
- if (!xrcd)
- return ERR_PTR(-ENOMEM);
-
- err = mlx5_cmd_xrcd_alloc(dev->mdev, &xrcd->xrcdn, 0);
- if (err) {
- kfree(xrcd);
- return ERR_PTR(-ENOMEM);
- }
-
- return &xrcd->ibxrcd;
+ return mlx5_cmd_xrcd_alloc(dev->mdev, &xrcd->xrcdn, 0);
}
-int mlx5_ib_dealloc_xrcd(struct ib_xrcd *xrcd, struct ib_udata *udata)
+void mlx5_ib_dealloc_xrcd(struct ib_xrcd *xrcd, struct ib_udata *udata)
{
struct mlx5_ib_dev *dev = to_mdev(xrcd->device);
u32 xrcdn = to_mxrcd(xrcd)->xrcdn;
- int err;
- err = mlx5_cmd_xrcd_dealloc(dev->mdev, xrcdn, 0);
- if (err)
- mlx5_ib_warn(dev, "failed to dealloc xrcdn 0x%x\n", xrcdn);
-
- kfree(xrcd);
- return 0;
+ mlx5_cmd_xrcd_dealloc(dev->mdev, xrcdn, 0);
}
static void mlx5_ib_wq_event(struct mlx5_core_qp *core_qp, int type)
diff --git a/drivers/infiniband/hw/mlx5/qp.h b/drivers/infiniband/hw/mlx5/qp.h
index 82ea2b9..ba899df 100644
--- a/drivers/infiniband/hw/mlx5/qp.h
+++ b/drivers/infiniband/hw/mlx5/qp.h
@@ -43,4 +43,5 @@
int mlx5_core_xrcd_alloc(struct mlx5_ib_dev *dev, u32 *xrcdn);
int mlx5_core_xrcd_dealloc(struct mlx5_ib_dev *dev, u32 xrcdn);
+int mlx5_ib_qp_set_counter(struct ib_qp *qp, struct rdma_counter *counter);
#endif /* _MLX5_IB_QP_H */
diff --git a/drivers/infiniband/hw/mlx5/restrack.c b/drivers/infiniband/hw/mlx5/restrack.c
index 8f6c04f..887270d 100644
--- a/drivers/infiniband/hw/mlx5/restrack.c
+++ b/drivers/infiniband/hw/mlx5/restrack.c
@@ -1,17 +1,85 @@
// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
/*
- * Copyright (c) 2019, Mellanox Technologies inc. All rights reserved.
+ * Copyright (c) 2019-2020, Mellanox Technologies Ltd. All rights reserved.
*/
#include <uapi/rdma/rdma_netlink.h>
+#include <linux/mlx5/rsc_dump.h>
#include <rdma/ib_umem_odp.h>
#include <rdma/restrack.h>
#include "mlx5_ib.h"
+#include "restrack.h"
-static int fill_stat_mr_entry(struct sk_buff *msg,
- struct rdma_restrack_entry *res)
+#define MAX_DUMP_SIZE 1024
+
+static int dump_rsc(struct mlx5_core_dev *dev, enum mlx5_sgmt_type type,
+ int index, void *data, int *data_len)
{
- struct ib_mr *ibmr = container_of(res, struct ib_mr, res);
+ struct mlx5_core_dev *mdev = dev;
+ struct mlx5_rsc_dump_cmd *cmd;
+ struct mlx5_rsc_key key = {};
+ struct page *page;
+ int offset = 0;
+ int err = 0;
+ int cmd_err;
+ int size;
+
+ page = alloc_page(GFP_KERNEL);
+ if (!page)
+ return -ENOMEM;
+
+ key.size = PAGE_SIZE;
+ key.rsc = type;
+ key.index1 = index;
+ key.num_of_obj1 = 1;
+
+ cmd = mlx5_rsc_dump_cmd_create(mdev, &key);
+ if (IS_ERR(cmd)) {
+ err = PTR_ERR(cmd);
+ goto free_page;
+ }
+
+ do {
+ cmd_err = mlx5_rsc_dump_next(mdev, cmd, page, &size);
+ if (cmd_err < 0 || size + offset > MAX_DUMP_SIZE) {
+ err = cmd_err;
+ goto destroy_cmd;
+ }
+ memcpy(data + offset, page_address(page), size);
+ offset += size;
+ } while (cmd_err > 0);
+ *data_len = offset;
+
+destroy_cmd:
+ mlx5_rsc_dump_cmd_destroy(cmd);
+free_page:
+ __free_page(page);
+ return err;
+}
+
+static int fill_res_raw(struct sk_buff *msg, struct mlx5_ib_dev *dev,
+ enum mlx5_sgmt_type type, u32 key)
+{
+ int len = 0;
+ void *data;
+ int err;
+
+ data = kzalloc(MAX_DUMP_SIZE, GFP_KERNEL);
+ if (!data)
+ return -ENOMEM;
+
+ err = dump_rsc(dev->mdev, type, key, data, &len);
+ if (err)
+ goto out;
+
+ err = nla_put(msg, RDMA_NLDEV_ATTR_RES_RAW, len, data);
+out:
+ kfree(data);
+ return err;
+}
+
+static int fill_stat_mr_entry(struct sk_buff *msg, struct ib_mr *ibmr)
+{
struct mlx5_ib_mr *mr = to_mmr(ibmr);
struct nlattr *table_attr;
@@ -31,6 +99,9 @@
msg, "page_invalidations",
atomic64_read(&mr->odp_stats.invalidations)))
goto err_table;
+ if (rdma_nl_stat_hwcounter_entry(msg, "page_prefetch",
+ atomic64_read(&mr->odp_stats.prefetch)))
+ goto err_table;
nla_nest_end(msg, table_attr);
return 0;
@@ -41,10 +112,16 @@
return -EMSGSIZE;
}
-static int fill_res_mr_entry(struct sk_buff *msg,
- struct rdma_restrack_entry *res)
+static int fill_res_mr_entry_raw(struct sk_buff *msg, struct ib_mr *ibmr)
{
- struct ib_mr *ibmr = container_of(res, struct ib_mr, res);
+ struct mlx5_ib_mr *mr = to_mmr(ibmr);
+
+ return fill_res_raw(msg, mr->dev, MLX5_SGMT_TYPE_PRM_QUERY_MKEY,
+ mlx5_mkey_to_idx(mr->mmkey.key));
+}
+
+static int fill_res_mr_entry(struct sk_buff *msg, struct ib_mr *ibmr)
+{
struct mlx5_ib_mr *mr = to_mmr(ibmr);
struct nlattr *table_attr;
@@ -71,20 +148,32 @@
return -EMSGSIZE;
}
-int mlx5_ib_fill_res_entry(struct sk_buff *msg,
- struct rdma_restrack_entry *res)
+static int fill_res_cq_entry_raw(struct sk_buff *msg, struct ib_cq *ibcq)
{
- if (res->type == RDMA_RESTRACK_MR)
- return fill_res_mr_entry(msg, res);
+ struct mlx5_ib_dev *dev = to_mdev(ibcq->device);
+ struct mlx5_ib_cq *cq = to_mcq(ibcq);
- return 0;
+ return fill_res_raw(msg, dev, MLX5_SGMT_TYPE_PRM_QUERY_CQ, cq->mcq.cqn);
}
-int mlx5_ib_fill_stat_entry(struct sk_buff *msg,
- struct rdma_restrack_entry *res)
+static int fill_res_qp_entry_raw(struct sk_buff *msg, struct ib_qp *ibqp)
{
- if (res->type == RDMA_RESTRACK_MR)
- return fill_stat_mr_entry(msg, res);
+ struct mlx5_ib_dev *dev = to_mdev(ibqp->device);
+ return fill_res_raw(msg, dev, MLX5_SGMT_TYPE_PRM_QUERY_QP,
+ ibqp->qp_num);
+}
+
+static const struct ib_device_ops restrack_ops = {
+ .fill_res_cq_entry_raw = fill_res_cq_entry_raw,
+ .fill_res_mr_entry = fill_res_mr_entry,
+ .fill_res_mr_entry_raw = fill_res_mr_entry_raw,
+ .fill_res_qp_entry_raw = fill_res_qp_entry_raw,
+ .fill_stat_mr_entry = fill_stat_mr_entry,
+};
+
+int mlx5_ib_restrack_init(struct mlx5_ib_dev *dev)
+{
+ ib_set_device_ops(&dev->ib_dev, &restrack_ops);
return 0;
}
diff --git a/drivers/infiniband/hw/mlx5/restrack.h b/drivers/infiniband/hw/mlx5/restrack.h
new file mode 100644
index 0000000..e8d8127
--- /dev/null
+++ b/drivers/infiniband/hw/mlx5/restrack.h
@@ -0,0 +1,13 @@
+/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */
+/*
+ * Copyright (c) 2013-2020, Mellanox Technologies Ltd. All rights reserved.
+ */
+
+#ifndef _MLX5_IB_RESTRACK_H
+#define _MLX5_IB_RESTRACK_H
+
+#include "mlx5_ib.h"
+
+int mlx5_ib_restrack_init(struct mlx5_ib_dev *dev);
+
+#endif /* _MLX5_IB_RESTRACK_H */
diff --git a/drivers/infiniband/hw/mlx5/srq.c b/drivers/infiniband/hw/mlx5/srq.c
index 6d1ff13..7e10cbc 100644
--- a/drivers/infiniband/hw/mlx5/srq.c
+++ b/drivers/infiniband/hw/mlx5/srq.c
@@ -274,10 +274,10 @@
if (srq->wq_sig)
in.flags |= MLX5_SRQ_FLAG_WQ_SIG;
- if (init_attr->srq_type == IB_SRQT_XRC)
+ if (init_attr->srq_type == IB_SRQT_XRC && init_attr->ext.xrc.xrcd)
in.xrcd = to_mxrcd(init_attr->ext.xrc.xrcd)->xrcdn;
else
- in.xrcd = to_mxrcd(dev->devr.x0)->xrcdn;
+ in.xrcd = dev->devr.xrcdn0;
if (init_attr->srq_type == IB_SRQT_TM) {
in.tm_log_list_size =
diff --git a/drivers/infiniband/hw/mlx5/std_types.c b/drivers/infiniband/hw/mlx5/std_types.c
new file mode 100644
index 0000000..16145fd
--- /dev/null
+++ b/drivers/infiniband/hw/mlx5/std_types.c
@@ -0,0 +1,45 @@
+// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
+/*
+ * Copyright (c) 2020, Mellanox Technologies inc. All rights reserved.
+ */
+
+#include <rdma/uverbs_ioctl.h>
+#include <rdma/mlx5_user_ioctl_cmds.h>
+#include <rdma/mlx5_user_ioctl_verbs.h>
+#include <linux/mlx5/driver.h>
+#include "mlx5_ib.h"
+
+#define UVERBS_MODULE_NAME mlx5_ib
+#include <rdma/uverbs_named_ioctl.h>
+
+static int UVERBS_HANDLER(MLX5_IB_METHOD_PD_QUERY)(
+ struct uverbs_attr_bundle *attrs)
+{
+ struct ib_pd *pd =
+ uverbs_attr_get_obj(attrs, MLX5_IB_ATTR_QUERY_PD_HANDLE);
+ struct mlx5_ib_pd *mpd = to_mpd(pd);
+
+ return uverbs_copy_to(attrs, MLX5_IB_ATTR_QUERY_PD_RESP_PDN,
+ &mpd->pdn, sizeof(mpd->pdn));
+}
+
+DECLARE_UVERBS_NAMED_METHOD(
+ MLX5_IB_METHOD_PD_QUERY,
+ UVERBS_ATTR_IDR(MLX5_IB_ATTR_QUERY_PD_HANDLE,
+ UVERBS_OBJECT_PD,
+ UVERBS_ACCESS_READ,
+ UA_MANDATORY),
+ UVERBS_ATTR_PTR_OUT(MLX5_IB_ATTR_QUERY_PD_RESP_PDN,
+ UVERBS_ATTR_TYPE(u32),
+ UA_MANDATORY));
+
+ADD_UVERBS_METHODS(mlx5_ib_pd,
+ UVERBS_OBJECT_PD,
+ &UVERBS_METHOD(MLX5_IB_METHOD_PD_QUERY));
+
+const struct uapi_definition mlx5_ib_std_types_defs[] = {
+ UAPI_DEF_CHAIN_OBJ_TREE(
+ UVERBS_OBJECT_PD,
+ &mlx5_ib_pd),
+ {},
+};
diff --git a/drivers/infiniband/hw/mlx5/wr.c b/drivers/infiniband/hw/mlx5/wr.c
index bc35dbe..4388097 100644
--- a/drivers/infiniband/hw/mlx5/wr.c
+++ b/drivers/infiniband/hw/mlx5/wr.c
@@ -263,7 +263,9 @@
return cpu_to_be64(result);
}
-static __be64 get_umr_update_access_mask(int atomic)
+static __be64 get_umr_update_access_mask(int atomic,
+ int relaxed_ordering_write,
+ int relaxed_ordering_read)
{
u64 result;
@@ -275,6 +277,12 @@
if (atomic)
result |= MLX5_MKEY_MASK_A;
+ if (relaxed_ordering_write)
+ result |= MLX5_MKEY_MASK_RELAXED_ORDERING_WRITE;
+
+ if (relaxed_ordering_read)
+ result |= MLX5_MKEY_MASK_RELAXED_ORDERING_READ;
+
return cpu_to_be64(result);
}
@@ -289,17 +297,28 @@
static int umr_check_mkey_mask(struct mlx5_ib_dev *dev, u64 mask)
{
- if ((mask & MLX5_MKEY_MASK_PAGE_SIZE &&
- MLX5_CAP_GEN(dev->mdev, umr_modify_entity_size_disabled)) ||
- (mask & MLX5_MKEY_MASK_A &&
- MLX5_CAP_GEN(dev->mdev, umr_modify_atomic_disabled)))
+ if (mask & MLX5_MKEY_MASK_PAGE_SIZE &&
+ MLX5_CAP_GEN(dev->mdev, umr_modify_entity_size_disabled))
return -EPERM;
+
+ if (mask & MLX5_MKEY_MASK_A &&
+ MLX5_CAP_GEN(dev->mdev, umr_modify_atomic_disabled))
+ return -EPERM;
+
+ if (mask & MLX5_MKEY_MASK_RELAXED_ORDERING_WRITE &&
+ !MLX5_CAP_GEN(dev->mdev, relaxed_ordering_write_umr))
+ return -EPERM;
+
+ if (mask & MLX5_MKEY_MASK_RELAXED_ORDERING_READ &&
+ !MLX5_CAP_GEN(dev->mdev, relaxed_ordering_read_umr))
+ return -EPERM;
+
return 0;
}
static int set_reg_umr_segment(struct mlx5_ib_dev *dev,
struct mlx5_wqe_umr_ctrl_seg *umr,
- const struct ib_send_wr *wr, int atomic)
+ const struct ib_send_wr *wr)
{
const struct mlx5_umr_wr *umrwr = umr_wr(wr);
@@ -325,7 +344,10 @@
if (wr->send_flags & MLX5_IB_SEND_UMR_UPDATE_TRANSLATION)
umr->mkey_mask |= get_umr_update_translation_mask();
if (wr->send_flags & MLX5_IB_SEND_UMR_UPDATE_PD_ACCESS) {
- umr->mkey_mask |= get_umr_update_access_mask(atomic);
+ umr->mkey_mask |= get_umr_update_access_mask(
+ !!(MLX5_CAP_GEN(dev->mdev, atomic)),
+ !!(MLX5_CAP_GEN(dev->mdev, relaxed_ordering_write_umr)),
+ !!(MLX5_CAP_GEN(dev->mdev, relaxed_ordering_read_umr)));
umr->mkey_mask |= get_umr_update_pd_mask();
}
if (wr->send_flags & MLX5_IB_SEND_UMR_ENABLE_MR)
@@ -383,20 +405,31 @@
memset(seg, 0, sizeof(*seg));
if (wr->send_flags & MLX5_IB_SEND_UMR_DISABLE_MR)
- seg->status = MLX5_MKEY_STATUS_FREE;
+ MLX5_SET(mkc, seg, free, 1);
- seg->flags = convert_access(umrwr->access_flags);
+ MLX5_SET(mkc, seg, a,
+ !!(umrwr->access_flags & IB_ACCESS_REMOTE_ATOMIC));
+ MLX5_SET(mkc, seg, rw,
+ !!(umrwr->access_flags & IB_ACCESS_REMOTE_WRITE));
+ MLX5_SET(mkc, seg, rr, !!(umrwr->access_flags & IB_ACCESS_REMOTE_READ));
+ MLX5_SET(mkc, seg, lw, !!(umrwr->access_flags & IB_ACCESS_LOCAL_WRITE));
+ MLX5_SET(mkc, seg, lr, 1);
+ MLX5_SET(mkc, seg, relaxed_ordering_write,
+ !!(umrwr->access_flags & IB_ACCESS_RELAXED_ORDERING));
+ MLX5_SET(mkc, seg, relaxed_ordering_read,
+ !!(umrwr->access_flags & IB_ACCESS_RELAXED_ORDERING));
+
if (umrwr->pd)
- seg->flags_pd = cpu_to_be32(to_mpd(umrwr->pd)->pdn);
+ MLX5_SET(mkc, seg, pd, to_mpd(umrwr->pd)->pdn);
if (wr->send_flags & MLX5_IB_SEND_UMR_UPDATE_TRANSLATION &&
!umrwr->length)
- seg->flags_pd |= cpu_to_be32(MLX5_MKEY_LEN64);
+ MLX5_SET(mkc, seg, length64, 1);
- seg->start_addr = cpu_to_be64(umrwr->virt_addr);
- seg->len = cpu_to_be64(umrwr->length);
- seg->log2_page_size = umrwr->page_shift;
- seg->qpn_mkey7_0 = cpu_to_be32(0xffffff00 |
- mlx5_mkey_variant(umrwr->mkey));
+ MLX5_SET64(mkc, seg, start_addr, umrwr->virt_addr);
+ MLX5_SET64(mkc, seg, len, umrwr->length);
+ MLX5_SET(mkc, seg, log_page_size, umrwr->page_shift);
+ MLX5_SET(mkc, seg, qpn, 0xffffff);
+ MLX5_SET(mkc, seg, mkey_7_0, mlx5_mkey_variant(umrwr->mkey));
}
static void set_reg_data_seg(struct mlx5_wqe_data_seg *dseg,
@@ -1224,8 +1257,7 @@
qp->sq.wr_data[idx] = MLX5_IB_WR_UMR;
(*ctrl)->imm = cpu_to_be32(umr_wr(wr)->mkey);
- err = set_reg_umr_segment(dev, *seg, wr,
- !!(MLX5_CAP_GEN(dev->mdev, atomic)));
+ err = set_reg_umr_segment(dev, *seg, wr);
if (unlikely(err))
goto out;
*seg += sizeof(struct mlx5_wqe_umr_ctrl_seg);
diff --git a/drivers/infiniband/hw/ocrdma/ocrdma_verbs.c b/drivers/infiniband/hw/ocrdma/ocrdma_verbs.c
index d11c743..6cdbec1 100644
--- a/drivers/infiniband/hw/ocrdma/ocrdma_verbs.c
+++ b/drivers/infiniband/hw/ocrdma/ocrdma_verbs.c
@@ -2901,7 +2901,7 @@
}
struct ib_mr *ocrdma_alloc_mr(struct ib_pd *ibpd, enum ib_mr_type mr_type,
- u32 max_num_sg, struct ib_udata *udata)
+ u32 max_num_sg)
{
int status;
struct ocrdma_mr *mr;
diff --git a/drivers/infiniband/hw/ocrdma/ocrdma_verbs.h b/drivers/infiniband/hw/ocrdma/ocrdma_verbs.h
index 3a50108..df8e3b9 100644
--- a/drivers/infiniband/hw/ocrdma/ocrdma_verbs.h
+++ b/drivers/infiniband/hw/ocrdma/ocrdma_verbs.h
@@ -101,7 +101,7 @@
struct ib_mr *ocrdma_reg_user_mr(struct ib_pd *, u64 start, u64 length,
u64 virt, int acc, struct ib_udata *);
struct ib_mr *ocrdma_alloc_mr(struct ib_pd *pd, enum ib_mr_type mr_type,
- u32 max_num_sg, struct ib_udata *udata);
+ u32 max_num_sg);
int ocrdma_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg, int sg_nents,
unsigned int *sg_offset);
diff --git a/drivers/infiniband/hw/qedr/main.c b/drivers/infiniband/hw/qedr/main.c
index b1de8d6..d85f992 100644
--- a/drivers/infiniband/hw/qedr/main.c
+++ b/drivers/infiniband/hw/qedr/main.c
@@ -110,7 +110,6 @@
if (err)
return err;
- immutable->pkey_tbl_len = 1;
immutable->gid_tbl_len = 1;
immutable->core_cap_flags = RDMA_CORE_PORT_IWARP;
immutable->max_mad_size = 0;
@@ -179,6 +178,7 @@
static const struct ib_device_ops qedr_roce_dev_ops = {
.get_port_immutable = qedr_roce_port_immutable,
+ .query_pkey = qedr_query_pkey,
};
static void qedr_roce_register_device(struct qedr_dev *dev)
@@ -221,7 +221,6 @@
.post_srq_recv = qedr_post_srq_recv,
.process_mad = qedr_process_mad,
.query_device = qedr_query_device,
- .query_pkey = qedr_query_pkey,
.query_port = qedr_query_port,
.query_qp = qedr_query_qp,
.query_srq = qedr_query_srq,
diff --git a/drivers/infiniband/hw/qedr/qedr.h b/drivers/infiniband/hw/qedr/qedr.h
index fdf90ec..4602921 100644
--- a/drivers/infiniband/hw/qedr/qedr.h
+++ b/drivers/infiniband/hw/qedr/qedr.h
@@ -235,6 +235,7 @@
u32 dpi_size;
u16 dpi;
bool db_rec;
+ u8 edpm_mode;
};
union db_prod32 {
@@ -344,10 +345,10 @@
u32 wqe_prod;
u32 sge_prod;
u32 wr_prod_cnt;
- u32 wr_cons_cnt;
+ atomic_t wr_cons_cnt;
u32 num_elems;
- u32 *virt_prod_pair_addr;
+ struct rdma_srq_producers *virt_prod_pair_addr;
dma_addr_t phy_prod_pair_addr;
};
diff --git a/drivers/infiniband/hw/qedr/verbs.c b/drivers/infiniband/hw/qedr/verbs.c
index f5aa85a..4ce4e2e 100644
--- a/drivers/infiniband/hw/qedr/verbs.c
+++ b/drivers/infiniband/hw/qedr/verbs.c
@@ -239,7 +239,6 @@
attr->ip_gids = true;
if (rdma_protocol_iwarp(&dev->ibdev, 1)) {
attr->gid_tbl_len = 1;
- attr->pkey_tbl_len = 1;
} else {
attr->gid_tbl_len = QEDR_MAX_SGID;
attr->pkey_tbl_len = QEDR_ROCE_PKEY_TABLE_LEN;
@@ -275,7 +274,8 @@
DP_ERR(dev, "Problem copying data from user space\n");
return -EFAULT;
}
-
+ ctx->edpm_mode = !!(ureq.context_flags &
+ QEDR_ALLOC_UCTX_EDPM_MODE);
ctx->db_rec = !!(ureq.context_flags & QEDR_ALLOC_UCTX_DB_REC);
}
@@ -316,11 +316,15 @@
uresp.dpm_flags = QEDR_DPM_TYPE_IWARP_LEGACY;
else
uresp.dpm_flags = QEDR_DPM_TYPE_ROCE_ENHANCED |
- QEDR_DPM_TYPE_ROCE_LEGACY;
+ QEDR_DPM_TYPE_ROCE_LEGACY |
+ QEDR_DPM_TYPE_ROCE_EDPM_MODE;
- uresp.dpm_flags |= QEDR_DPM_SIZES_SET;
- uresp.ldpm_limit_size = QEDR_LDPM_MAX_SIZE;
- uresp.edpm_trans_size = QEDR_EDPM_TRANS_SIZE;
+ if (ureq.context_flags & QEDR_SUPPORT_DPM_SIZES) {
+ uresp.dpm_flags |= QEDR_DPM_SIZES_SET;
+ uresp.ldpm_limit_size = QEDR_LDPM_MAX_SIZE;
+ uresp.edpm_trans_size = QEDR_EDPM_TRANS_SIZE;
+ uresp.edpm_limit_size = QEDR_EDPM_MAX_SIZE;
+ }
uresp.wids_enabled = 1;
uresp.wid_count = oparams.wid_count;
@@ -1754,7 +1758,7 @@
struct qed_rdma_create_qp_out_params out_params;
struct qedr_pd *pd = get_qedr_pd(ibpd);
struct qedr_create_qp_uresp uresp;
- struct qedr_ucontext *ctx = NULL;
+ struct qedr_ucontext *ctx = pd ? pd->uctx : NULL;
struct qedr_create_qp_ureq ureq;
int alloc_and_init = rdma_protocol_roce(&dev->ibdev, 1);
int rc = -EINVAL;
@@ -1792,6 +1796,9 @@
in_params.rq_pbl_ptr = qp->urq.pbl_tbl->pa;
}
+ if (ctx)
+ SET_FIELD(in_params.flags, QED_ROCE_EDPM_MODE, ctx->edpm_mode);
+
qp->qed_qp = dev->ops->rdma_create_qp(dev->rdma_ctx,
&in_params, &out_params);
@@ -3004,7 +3011,7 @@
}
struct ib_mr *qedr_alloc_mr(struct ib_pd *ibpd, enum ib_mr_type mr_type,
- u32 max_num_sg, struct ib_udata *udata)
+ u32 max_num_sg)
{
struct qedr_mr *mr;
@@ -3687,7 +3694,7 @@
* count and consumer count and subtract it from max
* work request supported so that we get elements left.
*/
- used = hw_srq->wr_prod_cnt - hw_srq->wr_cons_cnt;
+ used = hw_srq->wr_prod_cnt - (u32)atomic_read(&hw_srq->wr_cons_cnt);
return hw_srq->max_wr - used;
}
@@ -3702,7 +3709,6 @@
unsigned long flags;
int status = 0;
u32 num_sge;
- u32 offset;
spin_lock_irqsave(&srq->lock, flags);
@@ -3715,7 +3721,8 @@
if (!qedr_srq_elem_left(hw_srq) ||
wr->num_sge > srq->hw_srq.max_sges) {
DP_ERR(dev, "Can't post WR (%d,%d) || (%d > %d)\n",
- hw_srq->wr_prod_cnt, hw_srq->wr_cons_cnt,
+ hw_srq->wr_prod_cnt,
+ atomic_read(&hw_srq->wr_cons_cnt),
wr->num_sge, srq->hw_srq.max_sges);
status = -ENOMEM;
*bad_wr = wr;
@@ -3749,22 +3756,20 @@
hw_srq->sge_prod++;
}
- /* Flush WQE and SGE information before
+ /* Update WQE and SGE information before
* updating producer.
*/
- wmb();
+ dma_wmb();
/* SRQ producer is 8 bytes. Need to update SGE producer index
* in first 4 bytes and need to update WQE producer in
* next 4 bytes.
*/
- *srq->hw_srq.virt_prod_pair_addr = hw_srq->sge_prod;
- offset = offsetof(struct rdma_srq_producers, wqe_prod);
- *((u8 *)srq->hw_srq.virt_prod_pair_addr + offset) =
- hw_srq->wqe_prod;
+ srq->hw_srq.virt_prod_pair_addr->sge_prod = hw_srq->sge_prod;
+ /* Make sure sge producer is updated first */
+ dma_wmb();
+ srq->hw_srq.virt_prod_pair_addr->wqe_prod = hw_srq->wqe_prod;
- /* Flush producer after updating it. */
- wmb();
wr = wr->next;
}
@@ -4183,7 +4188,7 @@
} else {
__process_resp_one(dev, qp, cq, wc, resp, wr_id);
}
- srq->hw_srq.wr_cons_cnt++;
+ atomic_inc(&srq->hw_srq.wr_cons_cnt);
return 1;
}
diff --git a/drivers/infiniband/hw/qedr/verbs.h b/drivers/infiniband/hw/qedr/verbs.h
index 5e02387..39dd628 100644
--- a/drivers/infiniband/hw/qedr/verbs.h
+++ b/drivers/infiniband/hw/qedr/verbs.h
@@ -84,7 +84,7 @@
int sg_nents, unsigned int *sg_offset);
struct ib_mr *qedr_alloc_mr(struct ib_pd *pd, enum ib_mr_type mr_type,
- u32 max_num_sg, struct ib_udata *udata);
+ u32 max_num_sg);
int qedr_poll_cq(struct ib_cq *, int num_entries, struct ib_wc *wc);
int qedr_post_send(struct ib_qp *, const struct ib_send_wr *,
const struct ib_send_wr **bad_wr);
diff --git a/drivers/infiniband/hw/usnic/usnic_fwd.c b/drivers/infiniband/hw/usnic/usnic_fwd.c
index 7875883..398c4c0 100644
--- a/drivers/infiniband/hw/usnic/usnic_fwd.c
+++ b/drivers/infiniband/hw/usnic/usnic_fwd.c
@@ -214,7 +214,7 @@
if (!flow)
return ERR_PTR(-ENOMEM);
- tlv = pci_alloc_consistent(pdev, tlv_size, &tlv_pa);
+ tlv = dma_alloc_coherent(&pdev->dev, tlv_size, &tlv_pa, GFP_ATOMIC);
if (!tlv) {
usnic_err("Failed to allocate memory\n");
status = -ENOMEM;
@@ -258,7 +258,7 @@
out_free_tlv:
spin_unlock(&ufdev->lock);
- pci_free_consistent(pdev, tlv_size, tlv, tlv_pa);
+ dma_free_coherent(&pdev->dev, tlv_size, tlv, tlv_pa);
if (!status)
return flow;
out_free_flow:
diff --git a/drivers/infiniband/hw/vmw_pvrdma/pvrdma_mr.c b/drivers/infiniband/hw/vmw_pvrdma/pvrdma_mr.c
index b039f1f..77a010e68 100644
--- a/drivers/infiniband/hw/vmw_pvrdma/pvrdma_mr.c
+++ b/drivers/infiniband/hw/vmw_pvrdma/pvrdma_mr.c
@@ -202,7 +202,7 @@
* @return: ib_mr pointer on success, otherwise returns an errno.
*/
struct ib_mr *pvrdma_alloc_mr(struct ib_pd *pd, enum ib_mr_type mr_type,
- u32 max_num_sg, struct ib_udata *udata)
+ u32 max_num_sg)
{
struct pvrdma_dev *dev = to_vdev(pd->device);
struct pvrdma_user_mr *mr;
diff --git a/drivers/infiniband/hw/vmw_pvrdma/pvrdma_verbs.h b/drivers/infiniband/hw/vmw_pvrdma/pvrdma_verbs.h
index 2677022..699b208 100644
--- a/drivers/infiniband/hw/vmw_pvrdma/pvrdma_verbs.h
+++ b/drivers/infiniband/hw/vmw_pvrdma/pvrdma_verbs.h
@@ -406,7 +406,7 @@
struct ib_udata *udata);
int pvrdma_dereg_mr(struct ib_mr *mr, struct ib_udata *udata);
struct ib_mr *pvrdma_alloc_mr(struct ib_pd *pd, enum ib_mr_type mr_type,
- u32 max_num_sg, struct ib_udata *udata);
+ u32 max_num_sg);
int pvrdma_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg,
int sg_nents, unsigned int *sg_offset);
int pvrdma_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr,
diff --git a/drivers/infiniband/sw/rdmavt/ah.c b/drivers/infiniband/sw/rdmavt/ah.c
index 40480ad..75a04b1 100644
--- a/drivers/infiniband/sw/rdmavt/ah.c
+++ b/drivers/infiniband/sw/rdmavt/ah.c
@@ -90,8 +90,7 @@
/**
* rvt_create_ah - create an address handle
* @ibah: the IB address handle
- * @ah_attr: the attributes of the AH
- * @create_flags: create address handle flags (see enum rdma_create_ah_flags)
+ * @init_attr: the attributes of the AH
* @udata: pointer to user's input output buffer information.
*
* This may be called from interrupt context.
diff --git a/drivers/infiniband/sw/rdmavt/mr.c b/drivers/infiniband/sw/rdmavt/mr.c
index 60864e5..2f7c25f 100644
--- a/drivers/infiniband/sw/rdmavt/mr.c
+++ b/drivers/infiniband/sw/rdmavt/mr.c
@@ -576,7 +576,7 @@
* Return: the memory region on success, otherwise return an errno.
*/
struct ib_mr *rvt_alloc_mr(struct ib_pd *pd, enum ib_mr_type mr_type,
- u32 max_num_sg, struct ib_udata *udata)
+ u32 max_num_sg)
{
struct rvt_mr *mr;
diff --git a/drivers/infiniband/sw/rdmavt/mr.h b/drivers/infiniband/sw/rdmavt/mr.h
index 780fc63..b3aba35 100644
--- a/drivers/infiniband/sw/rdmavt/mr.h
+++ b/drivers/infiniband/sw/rdmavt/mr.h
@@ -71,7 +71,7 @@
struct ib_udata *udata);
int rvt_dereg_mr(struct ib_mr *ibmr, struct ib_udata *udata);
struct ib_mr *rvt_alloc_mr(struct ib_pd *pd, enum ib_mr_type mr_type,
- u32 max_num_sg, struct ib_udata *udata);
+ u32 max_num_sg);
int rvt_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg,
int sg_nents, unsigned int *sg_offset);
diff --git a/drivers/infiniband/sw/rxe/rxe.c b/drivers/infiniband/sw/rxe/rxe.c
index 5642eef..907203a 100644
--- a/drivers/infiniband/sw/rxe/rxe.c
+++ b/drivers/infiniband/sw/rxe/rxe.c
@@ -40,14 +40,6 @@
MODULE_DESCRIPTION("Soft RDMA transport");
MODULE_LICENSE("Dual BSD/GPL");
-/* free resources for all ports on a device */
-static void rxe_cleanup_ports(struct rxe_dev *rxe)
-{
- kfree(rxe->port.pkey_tbl);
- rxe->port.pkey_tbl = NULL;
-
-}
-
/* free resources for a rxe device all objects created for this device must
* have been destroyed
*/
@@ -66,8 +58,6 @@
rxe_pool_cleanup(&rxe->mc_grp_pool);
rxe_pool_cleanup(&rxe->mc_elem_pool);
- rxe_cleanup_ports(rxe);
-
if (rxe->tfm)
crypto_free_shash(rxe->tfm);
}
@@ -111,7 +101,7 @@
}
/* initialize port attributes */
-static int rxe_init_port_param(struct rxe_port *port)
+static void rxe_init_port_param(struct rxe_port *port)
{
port->attr.state = IB_PORT_DOWN;
port->attr.max_mtu = IB_MTU_4096;
@@ -134,35 +124,19 @@
port->attr.phys_state = RXE_PORT_PHYS_STATE;
port->mtu_cap = ib_mtu_enum_to_int(IB_MTU_256);
port->subnet_prefix = cpu_to_be64(RXE_PORT_SUBNET_PREFIX);
-
- return 0;
}
/* initialize port state, note IB convention that HCA ports are always
* numbered from 1
*/
-static int rxe_init_ports(struct rxe_dev *rxe)
+static void rxe_init_ports(struct rxe_dev *rxe)
{
struct rxe_port *port = &rxe->port;
rxe_init_port_param(port);
-
- if (!port->attr.pkey_tbl_len || !port->attr.gid_tbl_len)
- return -EINVAL;
-
- port->pkey_tbl = kcalloc(port->attr.pkey_tbl_len,
- sizeof(*port->pkey_tbl), GFP_KERNEL);
-
- if (!port->pkey_tbl)
- return -ENOMEM;
-
- port->pkey_tbl[0] = 0xffff;
addrconf_addr_eui48((unsigned char *)&port->port_guid,
rxe->ndev->dev_addr);
-
spin_lock_init(&port->port_lock);
-
- return 0;
}
/* init pools of managed objects */
@@ -252,13 +226,11 @@
/* init default device parameters */
rxe_init_device_param(rxe);
- err = rxe_init_ports(rxe);
- if (err)
- goto err1;
+ rxe_init_ports(rxe);
err = rxe_init_pools(rxe);
if (err)
- goto err2;
+ return err;
/* init pending mmap list */
spin_lock_init(&rxe->mmap_offset_lock);
@@ -268,11 +240,6 @@
mutex_init(&rxe->usdev_lock);
return 0;
-
-err2:
- rxe_cleanup_ports(rxe);
-err1:
- return err;
}
void rxe_set_mtu(struct rxe_dev *rxe, unsigned int ndev_mtu)
diff --git a/drivers/infiniband/sw/rxe/rxe_loc.h b/drivers/infiniband/sw/rxe/rxe_loc.h
index 775c23b..39dc3bf 100644
--- a/drivers/infiniband/sw/rxe/rxe_loc.h
+++ b/drivers/infiniband/sw/rxe/rxe_loc.h
@@ -103,8 +103,8 @@
from_mem_obj,
};
-int rxe_mem_init_dma(struct rxe_pd *pd,
- int access, struct rxe_mem *mem);
+void rxe_mem_init_dma(struct rxe_pd *pd,
+ int access, struct rxe_mem *mem);
int rxe_mem_init_user(struct rxe_pd *pd, u64 start,
u64 length, u64 iova, int access, struct ib_udata *udata,
@@ -132,9 +132,6 @@
int mem_check_range(struct rxe_mem *mem, u64 iova, size_t length);
-int rxe_mem_map_pages(struct rxe_dev *rxe, struct rxe_mem *mem,
- u64 *page, int num_pages, u64 iova);
-
void rxe_mem_cleanup(struct rxe_pool_entry *arg);
int advance_dma_data(struct rxe_dma_info *dma, unsigned int length);
@@ -145,7 +142,6 @@
struct sk_buff *rxe_init_packet(struct rxe_dev *rxe, struct rxe_av *av,
int paylen, struct rxe_pkt_info *pkt);
int rxe_prepare(struct rxe_pkt_info *pkt, struct sk_buff *skb, u32 *crc);
-enum rdma_link_layer rxe_link_layer(struct rxe_dev *rxe, unsigned int port_num);
const char *rxe_parent_name(struct rxe_dev *rxe, unsigned int port_num);
struct device *rxe_dma_device(struct rxe_dev *rxe);
int rxe_mcast_add(struct rxe_dev *rxe, union ib_gid *mgid);
diff --git a/drivers/infiniband/sw/rxe/rxe_mr.c b/drivers/infiniband/sw/rxe/rxe_mr.c
index e83c7b5..cdd811a 100644
--- a/drivers/infiniband/sw/rxe/rxe_mr.c
+++ b/drivers/infiniband/sw/rxe/rxe_mr.c
@@ -144,8 +144,8 @@
return -ENOMEM;
}
-int rxe_mem_init_dma(struct rxe_pd *pd,
- int access, struct rxe_mem *mem)
+void rxe_mem_init_dma(struct rxe_pd *pd,
+ int access, struct rxe_mem *mem)
{
rxe_mem_init(access, mem);
@@ -153,8 +153,6 @@
mem->access = access;
mem->state = RXE_MEM_STATE_VALID;
mem->type = RXE_MEM_TYPE_DMA;
-
- return 0;
}
int rxe_mem_init_user(struct rxe_pd *pd, u64 start,
@@ -587,47 +585,3 @@
return mem;
}
-
-int rxe_mem_map_pages(struct rxe_dev *rxe, struct rxe_mem *mem,
- u64 *page, int num_pages, u64 iova)
-{
- int i;
- int num_buf;
- int err;
- struct rxe_map **map;
- struct rxe_phys_buf *buf;
- int page_size;
-
- if (num_pages > mem->max_buf) {
- err = -EINVAL;
- goto err1;
- }
-
- num_buf = 0;
- page_size = 1 << mem->page_shift;
- map = mem->map;
- buf = map[0]->buf;
-
- for (i = 0; i < num_pages; i++) {
- buf->addr = *page++;
- buf->size = page_size;
- buf++;
- num_buf++;
-
- if (num_buf == RXE_BUF_PER_MAP) {
- map++;
- buf = map[0]->buf;
- num_buf = 0;
- }
- }
-
- mem->iova = iova;
- mem->va = iova;
- mem->length = num_pages << mem->page_shift;
- mem->state = RXE_MEM_STATE_VALID;
-
- return 0;
-
-err1:
- return err;
-}
diff --git a/drivers/infiniband/sw/rxe/rxe_net.c b/drivers/infiniband/sw/rxe/rxe_net.c
index 312c2fc..0c38086 100644
--- a/drivers/infiniband/sw/rxe/rxe_net.c
+++ b/drivers/infiniband/sw/rxe/rxe_net.c
@@ -520,11 +520,6 @@
return rxe->ndev->name;
}
-enum rdma_link_layer rxe_link_layer(struct rxe_dev *rxe, unsigned int port_num)
-{
- return IB_LINK_LAYER_ETHERNET;
-}
-
int rxe_net_add(const char *ibdev_name, struct net_device *ndev)
{
int err;
diff --git a/drivers/infiniband/sw/rxe/rxe_param.h b/drivers/infiniband/sw/rxe/rxe_param.h
index 99e9d8b..2f381ae 100644
--- a/drivers/infiniband/sw/rxe/rxe_param.h
+++ b/drivers/infiniband/sw/rxe/rxe_param.h
@@ -100,7 +100,7 @@
RXE_MAX_SRQ_SGE = 27,
RXE_MIN_SRQ_SGE = 1,
RXE_MAX_FMR_PAGE_LIST_LEN = 512,
- RXE_MAX_PKEYS = 64,
+ RXE_MAX_PKEYS = 1,
RXE_LOCAL_CA_ACK_DELAY = 15,
RXE_MAX_UCONTEXT = 512,
@@ -148,7 +148,7 @@
RXE_PORT_INIT_TYPE_REPLY = 0,
RXE_PORT_ACTIVE_WIDTH = IB_WIDTH_1X,
RXE_PORT_ACTIVE_SPEED = 1,
- RXE_PORT_PKEY_TBL_LEN = 64,
+ RXE_PORT_PKEY_TBL_LEN = 1,
RXE_PORT_PHYS_STATE = IB_PORT_PHYS_STATE_POLLING,
RXE_PORT_SUBNET_PREFIX = 0xfe80000000000000ULL,
};
diff --git a/drivers/infiniband/sw/rxe/rxe_recv.c b/drivers/infiniband/sw/rxe/rxe_recv.c
index 831ad57..7e123d3 100644
--- a/drivers/infiniband/sw/rxe/rxe_recv.c
+++ b/drivers/infiniband/sw/rxe/rxe_recv.c
@@ -101,36 +101,15 @@
static int check_keys(struct rxe_dev *rxe, struct rxe_pkt_info *pkt,
u32 qpn, struct rxe_qp *qp)
{
- int i;
- int found_pkey = 0;
struct rxe_port *port = &rxe->port;
u16 pkey = bth_pkey(pkt);
pkt->pkey_index = 0;
- if (qpn == 1) {
- for (i = 0; i < port->attr.pkey_tbl_len; i++) {
- if (pkey_match(pkey, port->pkey_tbl[i])) {
- pkt->pkey_index = i;
- found_pkey = 1;
- break;
- }
- }
-
- if (!found_pkey) {
- pr_warn_ratelimited("bad pkey = 0x%x\n", pkey);
- set_bad_pkey_cntr(port);
- goto err1;
- }
- } else {
- if (unlikely(!pkey_match(pkey,
- port->pkey_tbl[qp->attr.pkey_index]
- ))) {
- pr_warn_ratelimited("bad pkey = 0x%0x\n", pkey);
- set_bad_pkey_cntr(port);
- goto err1;
- }
- pkt->pkey_index = qp->attr.pkey_index;
+ if (!pkey_match(pkey, IB_DEFAULT_PKEY_FULL)) {
+ pr_warn_ratelimited("bad pkey = 0x%x\n", pkey);
+ set_bad_pkey_cntr(port);
+ goto err1;
}
if ((qp_type(qp) == IB_QPT_UD || qp_type(qp) == IB_QPT_GSI) &&
@@ -330,10 +309,14 @@
static int rxe_match_dgid(struct rxe_dev *rxe, struct sk_buff *skb)
{
+ struct rxe_pkt_info *pkt = SKB_TO_PKT(skb);
const struct ib_gid_attr *gid_attr;
union ib_gid dgid;
union ib_gid *pdgid;
+ if (pkt->mask & RXE_LOOPBACK_MASK)
+ return 0;
+
if (skb->protocol == htons(ETH_P_IP)) {
ipv6_addr_set_v4mapped(ip_hdr(skb)->daddr,
(struct in6_addr *)&dgid);
@@ -366,7 +349,7 @@
if (unlikely(skb->len < pkt->offset + RXE_BTH_BYTES))
goto drop;
- if (unlikely(rxe_match_dgid(rxe, skb) < 0)) {
+ if (rxe_match_dgid(rxe, skb) < 0) {
pr_warn_ratelimited("failed matching dgid\n");
goto drop;
}
diff --git a/drivers/infiniband/sw/rxe/rxe_req.c b/drivers/infiniband/sw/rxe/rxe_req.c
index e503117..34df2b5 100644
--- a/drivers/infiniband/sw/rxe/rxe_req.c
+++ b/drivers/infiniband/sw/rxe/rxe_req.c
@@ -381,7 +381,6 @@
struct rxe_pkt_info *pkt)
{
struct rxe_dev *rxe = to_rdev(qp->ibqp.device);
- struct rxe_port *port = &rxe->port;
struct sk_buff *skb;
struct rxe_send_wr *ibwr = &wqe->wr;
struct rxe_av *av;
@@ -419,9 +418,7 @@
(pkt->mask & (RXE_WRITE_MASK | RXE_IMMDT_MASK)) ==
(RXE_WRITE_MASK | RXE_IMMDT_MASK));
- pkey = (qp_type(qp) == IB_QPT_GSI) ?
- port->pkey_tbl[ibwr->wr.ud.pkey_index] :
- port->pkey_tbl[qp->attr.pkey_index];
+ pkey = IB_DEFAULT_PKEY_FULL;
qp_num = (pkt->mask & RXE_DETH_MASK) ? ibwr->wr.ud.remote_qpn :
qp->attr.dest_qp_num;
diff --git a/drivers/infiniband/sw/rxe/rxe_verbs.c b/drivers/infiniband/sw/rxe/rxe_verbs.c
index b8a22af..bb61e53 100644
--- a/drivers/infiniband/sw/rxe/rxe_verbs.c
+++ b/drivers/infiniband/sw/rxe/rxe_verbs.c
@@ -83,22 +83,11 @@
static int rxe_query_pkey(struct ib_device *device,
u8 port_num, u16 index, u16 *pkey)
{
- struct rxe_dev *rxe = to_rdev(device);
- struct rxe_port *port;
+ if (index > 0)
+ return -EINVAL;
- port = &rxe->port;
-
- if (unlikely(index >= port->attr.pkey_tbl_len)) {
- dev_warn(device->dev.parent, "invalid index = %d\n",
- index);
- goto err1;
- }
-
- *pkey = port->pkey_tbl[index];
+ *pkey = IB_DEFAULT_PKEY_FULL;
return 0;
-
-err1:
- return -EINVAL;
}
static int rxe_modify_device(struct ib_device *dev,
@@ -141,9 +130,7 @@
static enum rdma_link_layer rxe_get_link_layer(struct ib_device *dev,
u8 port_num)
{
- struct rxe_dev *rxe = to_rdev(dev);
-
- return rxe_link_layer(rxe, port_num);
+ return IB_LINK_LAYER_ETHERNET;
}
static int rxe_alloc_ucontext(struct ib_ucontext *uctx, struct ib_udata *udata)
@@ -684,6 +671,7 @@
unsigned int mask;
unsigned int length = 0;
int i;
+ struct ib_send_wr *next;
while (wr) {
mask = wr_opcode_mask(wr->opcode, qp);
@@ -700,6 +688,8 @@
break;
}
+ next = wr->next;
+
length = 0;
for (i = 0; i < wr->num_sge; i++)
length += wr->sg_list[i].length;
@@ -710,7 +700,7 @@
*bad_wr = wr;
break;
}
- wr = wr->next;
+ wr = next;
}
rxe_run_task(&qp->req.task, 1);
@@ -901,30 +891,16 @@
struct rxe_dev *rxe = to_rdev(ibpd->device);
struct rxe_pd *pd = to_rpd(ibpd);
struct rxe_mem *mr;
- int err;
mr = rxe_alloc(&rxe->mr_pool);
- if (!mr) {
- err = -ENOMEM;
- goto err1;
- }
+ if (!mr)
+ return ERR_PTR(-ENOMEM);
rxe_add_index(mr);
-
rxe_add_ref(pd);
-
- err = rxe_mem_init_dma(pd, access, mr);
- if (err)
- goto err2;
+ rxe_mem_init_dma(pd, access, mr);
return &mr->ibmr;
-
-err2:
- rxe_drop_ref(pd);
- rxe_drop_index(mr);
- rxe_drop_ref(mr);
-err1:
- return ERR_PTR(err);
}
static struct ib_mr *rxe_reg_user_mr(struct ib_pd *ibpd,
@@ -975,7 +951,7 @@
}
static struct ib_mr *rxe_alloc_mr(struct ib_pd *ibpd, enum ib_mr_type mr_type,
- u32 max_num_sg, struct ib_udata *udata)
+ u32 max_num_sg)
{
struct rxe_dev *rxe = to_rdev(ibpd->device);
struct rxe_pd *pd = to_rpd(ibpd);
diff --git a/drivers/infiniband/sw/rxe/rxe_verbs.h b/drivers/infiniband/sw/rxe/rxe_verbs.h
index 92de39c..c664c7f 100644
--- a/drivers/infiniband/sw/rxe/rxe_verbs.h
+++ b/drivers/infiniband/sw/rxe/rxe_verbs.h
@@ -371,7 +371,6 @@
struct rxe_port {
struct ib_port_attr attr;
- u16 *pkey_tbl;
__be64 port_guid;
__be64 subnet_prefix;
spinlock_t port_lock; /* guard port */
diff --git a/drivers/infiniband/sw/siw/siw_main.c b/drivers/infiniband/sw/siw/siw_main.c
index ed60c9e4..d862bec 100644
--- a/drivers/infiniband/sw/siw/siw_main.c
+++ b/drivers/infiniband/sw/siw/siw_main.c
@@ -289,7 +289,6 @@
.post_srq_recv = siw_post_srq_recv,
.query_device = siw_query_device,
.query_gid = siw_query_gid,
- .query_pkey = siw_query_pkey,
.query_port = siw_query_port,
.query_qp = siw_query_qp,
.query_srq = siw_query_srq,
diff --git a/drivers/infiniband/sw/siw/siw_verbs.c b/drivers/infiniband/sw/siw/siw_verbs.c
index 987e2ba..adafa1b 100644
--- a/drivers/infiniband/sw/siw/siw_verbs.c
+++ b/drivers/infiniband/sw/siw/siw_verbs.c
@@ -176,7 +176,6 @@
attr->active_mtu = ib_mtu_int_to_enum(sdev->netdev->mtu);
attr->phys_state = sdev->state == IB_PORT_ACTIVE ?
IB_PORT_PHYS_STATE_LINK_UP : IB_PORT_PHYS_STATE_DISABLED;
- attr->pkey_tbl_len = 1;
attr->port_cap_flags = IB_PORT_CM_SUP | IB_PORT_DEVICE_MGMT_SUP;
attr->state = sdev->state;
/*
@@ -204,20 +203,12 @@
if (rv)
return rv;
- port_immutable->pkey_tbl_len = attr.pkey_tbl_len;
port_immutable->gid_tbl_len = attr.gid_tbl_len;
port_immutable->core_cap_flags = RDMA_CORE_PORT_IWARP;
return 0;
}
-int siw_query_pkey(struct ib_device *base_dev, u8 port, u16 idx, u16 *pkey)
-{
- /* Report the default pkey */
- *pkey = 0xffff;
- return 0;
-}
-
int siw_query_gid(struct ib_device *base_dev, u8 port, int idx,
union ib_gid *gid)
{
@@ -1373,7 +1364,7 @@
}
struct ib_mr *siw_alloc_mr(struct ib_pd *pd, enum ib_mr_type mr_type,
- u32 max_sge, struct ib_udata *udata)
+ u32 max_sge)
{
struct siw_device *sdev = to_siw_dev(pd->device);
struct siw_mr *mr = NULL;
diff --git a/drivers/infiniband/sw/siw/siw_verbs.h b/drivers/infiniband/sw/siw/siw_verbs.h
index 1a73198..d957227 100644
--- a/drivers/infiniband/sw/siw/siw_verbs.h
+++ b/drivers/infiniband/sw/siw/siw_verbs.h
@@ -46,7 +46,6 @@
struct ib_udata *udata);
int siw_query_port(struct ib_device *base_dev, u8 port,
struct ib_port_attr *attr);
-int siw_query_pkey(struct ib_device *base_dev, u8 port, u16 idx, u16 *pkey);
int siw_query_gid(struct ib_device *base_dev, u8 port, int idx,
union ib_gid *gid);
int siw_alloc_pd(struct ib_pd *base_pd, struct ib_udata *udata);
@@ -69,7 +68,7 @@
struct ib_mr *siw_reg_user_mr(struct ib_pd *base_pd, u64 start, u64 len,
u64 rnic_va, int rights, struct ib_udata *udata);
struct ib_mr *siw_alloc_mr(struct ib_pd *base_pd, enum ib_mr_type mr_type,
- u32 max_sge, struct ib_udata *udata);
+ u32 max_sge);
struct ib_mr *siw_get_dma_mr(struct ib_pd *base_pd, int rights);
int siw_map_mr_sg(struct ib_mr *base_mr, struct scatterlist *sl, int num_sle,
unsigned int *sg_off);
diff --git a/drivers/infiniband/ulp/ipoib/ipoib.h b/drivers/infiniband/ulp/ipoib/ipoib.h
index 9a3379c..3440dc4 100644
--- a/drivers/infiniband/ulp/ipoib/ipoib.h
+++ b/drivers/infiniband/ulp/ipoib/ipoib.h
@@ -515,7 +515,7 @@
int ipoib_ib_dev_open_default(struct net_device *dev);
int ipoib_ib_dev_open(struct net_device *dev);
-int ipoib_ib_dev_stop(struct net_device *dev);
+void ipoib_ib_dev_stop(struct net_device *dev);
void ipoib_ib_dev_up(struct net_device *dev);
void ipoib_ib_dev_down(struct net_device *dev);
int ipoib_ib_dev_stop_default(struct net_device *dev);
@@ -527,7 +527,7 @@
void ipoib_mcast_restart_task(struct work_struct *work);
void ipoib_mcast_start_thread(struct net_device *dev);
-int ipoib_mcast_stop_thread(struct net_device *dev);
+void ipoib_mcast_stop_thread(struct net_device *dev);
void ipoib_mcast_dev_down(struct net_device *dev);
void ipoib_mcast_dev_flush(struct net_device *dev);
diff --git a/drivers/infiniband/ulp/ipoib/ipoib_ib.c b/drivers/infiniband/ulp/ipoib/ipoib_ib.c
index da3c531..494f413 100644
--- a/drivers/infiniband/ulp/ipoib/ipoib_ib.c
+++ b/drivers/infiniband/ulp/ipoib/ipoib_ib.c
@@ -670,13 +670,12 @@
return rc;
}
-static void __ipoib_reap_ah(struct net_device *dev)
+static void ipoib_reap_dead_ahs(struct ipoib_dev_priv *priv)
{
- struct ipoib_dev_priv *priv = ipoib_priv(dev);
struct ipoib_ah *ah, *tah;
unsigned long flags;
- netif_tx_lock_bh(dev);
+ netif_tx_lock_bh(priv->dev);
spin_lock_irqsave(&priv->lock, flags);
list_for_each_entry_safe(ah, tah, &priv->dead_ahs, list)
@@ -687,37 +686,37 @@
}
spin_unlock_irqrestore(&priv->lock, flags);
- netif_tx_unlock_bh(dev);
+ netif_tx_unlock_bh(priv->dev);
}
void ipoib_reap_ah(struct work_struct *work)
{
struct ipoib_dev_priv *priv =
container_of(work, struct ipoib_dev_priv, ah_reap_task.work);
- struct net_device *dev = priv->dev;
- __ipoib_reap_ah(dev);
+ ipoib_reap_dead_ahs(priv);
if (!test_bit(IPOIB_STOP_REAPER, &priv->flags))
queue_delayed_work(priv->wq, &priv->ah_reap_task,
round_jiffies_relative(HZ));
}
-static void ipoib_flush_ah(struct net_device *dev)
+static void ipoib_start_ah_reaper(struct ipoib_dev_priv *priv)
{
- struct ipoib_dev_priv *priv = ipoib_priv(dev);
-
- cancel_delayed_work(&priv->ah_reap_task);
- flush_workqueue(priv->wq);
- ipoib_reap_ah(&priv->ah_reap_task.work);
+ clear_bit(IPOIB_STOP_REAPER, &priv->flags);
+ queue_delayed_work(priv->wq, &priv->ah_reap_task,
+ round_jiffies_relative(HZ));
}
-static void ipoib_stop_ah(struct net_device *dev)
+static void ipoib_stop_ah_reaper(struct ipoib_dev_priv *priv)
{
- struct ipoib_dev_priv *priv = ipoib_priv(dev);
-
set_bit(IPOIB_STOP_REAPER, &priv->flags);
- ipoib_flush_ah(dev);
+ cancel_delayed_work(&priv->ah_reap_task);
+ /*
+ * After ipoib_stop_ah_reaper() we always go through
+ * ipoib_reap_dead_ahs() which ensures the work is really stopped and
+ * does a final flush out of the dead_ah's list
+ */
}
static int recvs_pending(struct net_device *dev)
@@ -846,18 +845,6 @@
return 0;
}
-int ipoib_ib_dev_stop(struct net_device *dev)
-{
- struct ipoib_dev_priv *priv = ipoib_priv(dev);
-
- priv->rn_ops->ndo_stop(dev);
-
- clear_bit(IPOIB_FLAG_INITIALIZED, &priv->flags);
- ipoib_flush_ah(dev);
-
- return 0;
-}
-
int ipoib_ib_dev_open_default(struct net_device *dev)
{
struct ipoib_dev_priv *priv = ipoib_priv(dev);
@@ -901,10 +888,7 @@
return -1;
}
- clear_bit(IPOIB_STOP_REAPER, &priv->flags);
- queue_delayed_work(priv->wq, &priv->ah_reap_task,
- round_jiffies_relative(HZ));
-
+ ipoib_start_ah_reaper(priv);
if (priv->rn_ops->ndo_open(dev)) {
pr_warn("%s: Failed to open dev\n", dev->name);
goto dev_stop;
@@ -915,13 +899,20 @@
return 0;
dev_stop:
- set_bit(IPOIB_STOP_REAPER, &priv->flags);
- cancel_delayed_work(&priv->ah_reap_task);
- set_bit(IPOIB_FLAG_INITIALIZED, &priv->flags);
- ipoib_ib_dev_stop(dev);
+ ipoib_stop_ah_reaper(priv);
return -1;
}
+void ipoib_ib_dev_stop(struct net_device *dev)
+{
+ struct ipoib_dev_priv *priv = ipoib_priv(dev);
+
+ priv->rn_ops->ndo_stop(dev);
+
+ clear_bit(IPOIB_FLAG_INITIALIZED, &priv->flags);
+ ipoib_stop_ah_reaper(priv);
+}
+
void ipoib_pkey_dev_check_presence(struct net_device *dev)
{
struct ipoib_dev_priv *priv = ipoib_priv(dev);
@@ -1232,7 +1223,7 @@
ipoib_mcast_dev_flush(dev);
if (oper_up)
set_bit(IPOIB_FLAG_OPER_UP, &priv->flags);
- ipoib_flush_ah(dev);
+ ipoib_reap_dead_ahs(priv);
}
if (level >= IPOIB_FLUSH_NORMAL)
@@ -1307,7 +1298,7 @@
* the neighbor garbage collection is stopped and reaped.
* That should all be done now, so make a final ah flush.
*/
- ipoib_stop_ah(dev);
+ ipoib_reap_dead_ahs(priv);
clear_bit(IPOIB_PKEY_ASSIGNED, &priv->flags);
diff --git a/drivers/infiniband/ulp/ipoib/ipoib_main.c b/drivers/infiniband/ulp/ipoib/ipoib_main.c
index 3cfb682..752581a 100644
--- a/drivers/infiniband/ulp/ipoib/ipoib_main.c
+++ b/drivers/infiniband/ulp/ipoib/ipoib_main.c
@@ -1892,8 +1892,15 @@
priv->max_ib_mtu = ppriv->max_ib_mtu;
set_bit(IPOIB_FLAG_SUBINTERFACE, &priv->flags);
- memcpy(priv->dev->dev_addr, ppriv->dev->dev_addr, INFINIBAND_ALEN);
- memcpy(&priv->local_gid, &ppriv->local_gid, sizeof(priv->local_gid));
+ if (memchr_inv(priv->dev->dev_addr, 0, INFINIBAND_ALEN))
+ memcpy(&priv->local_gid, priv->dev->dev_addr + 4,
+ sizeof(priv->local_gid));
+ else {
+ memcpy(priv->dev->dev_addr, ppriv->dev->dev_addr,
+ INFINIBAND_ALEN);
+ memcpy(&priv->local_gid, &ppriv->local_gid,
+ sizeof(priv->local_gid));
+ }
}
static int ipoib_ndo_init(struct net_device *ndev)
@@ -1976,6 +1983,8 @@
/* no more works over the priv->wq */
if (priv->wq) {
+ /* See ipoib_mcast_carrier_on_task() */
+ WARN_ON(test_bit(IPOIB_FLAG_OPER_UP, &priv->flags));
flush_workqueue(priv->wq);
destroy_workqueue(priv->wq);
priv->wq = NULL;
diff --git a/drivers/infiniband/ulp/ipoib/ipoib_multicast.c b/drivers/infiniband/ulp/ipoib/ipoib_multicast.c
index 9bfa514..86e4ed6 100644
--- a/drivers/infiniband/ulp/ipoib/ipoib_multicast.c
+++ b/drivers/infiniband/ulp/ipoib/ipoib_multicast.c
@@ -680,15 +680,13 @@
spin_unlock_irqrestore(&priv->lock, flags);
}
-int ipoib_mcast_stop_thread(struct net_device *dev)
+void ipoib_mcast_stop_thread(struct net_device *dev)
{
struct ipoib_dev_priv *priv = ipoib_priv(dev);
ipoib_dbg_mcast(priv, "stopping multicast thread\n");
cancel_delayed_work_sync(&priv->mcast_task);
-
- return 0;
}
static int ipoib_mcast_leave(struct net_device *dev, struct ipoib_mcast *mcast)
diff --git a/drivers/infiniband/ulp/iser/iscsi_iser.h b/drivers/infiniband/ulp/iser/iscsi_iser.h
index 1d77c7f..78ee944 100644
--- a/drivers/infiniband/ulp/iser/iscsi_iser.h
+++ b/drivers/infiniband/ulp/iser/iscsi_iser.h
@@ -300,18 +300,6 @@
struct iscsi_iser_task;
/**
- * struct iser_comp - iSER completion context
- *
- * @cq: completion queue
- * @active_qps: Number of active QPs attached
- * to completion context
- */
-struct iser_comp {
- struct ib_cq *cq;
- int active_qps;
-};
-
-/**
* struct iser_device - iSER device handle
*
* @ib_device: RDMA device
@@ -320,9 +308,6 @@
* @event_handler: IB events handle routine
* @ig_list: entry in devices list
* @refcount: Reference counter, dominated by open iser connections
- * @comps_used: Number of completion contexts used, Min between online
- * cpus and device max completion vectors
- * @comps: Dinamically allocated array of completion handlers
*/
struct iser_device {
struct ib_device *ib_device;
@@ -330,8 +315,6 @@
struct ib_event_handler event_handler;
struct list_head ig_list;
int refcount;
- int comps_used;
- struct iser_comp *comps;
};
/**
@@ -353,6 +336,7 @@
* @list: entry in connection fastreg pool
* @rsc: data buffer registration resources
* @sig_protected: is region protected indicator
+ * @all_list: first and last list members
*/
struct iser_fr_desc {
struct list_head list;
@@ -367,6 +351,7 @@
* @list: list of fastreg descriptors
* @lock: protects fastreg pool
* @size: size of the pool
+ * @all_list: first and last list members
*/
struct iser_fr_pool {
struct list_head list;
@@ -380,11 +365,12 @@
*
* @cma_id: rdma_cm connection maneger handle
* @qp: Connection Queue-pair
+ * @cq: Connection completion queue
+ * @cq_size: The number of max outstanding completions
* @post_recv_buf_count: post receive counter
* @sig_count: send work request signal count
* @rx_wr: receive work request for batch posts
* @device: reference to iser device
- * @comp: iser completion context
* @fr_pool: connection fast registration poool
* @pi_support: Indicate device T10-PI support
* @reg_cqe: completion handler
@@ -392,11 +378,12 @@
struct ib_conn {
struct rdma_cm_id *cma_id;
struct ib_qp *qp;
+ struct ib_cq *cq;
+ u32 cq_size;
int post_recv_buf_count;
u8 sig_count;
struct ib_recv_wr rx_wr[ISER_MIN_POSTED_RX];
struct iser_device *device;
- struct iser_comp *comp;
struct iser_fr_pool fr_pool;
bool pi_support;
struct ib_cqe reg_cqe;
diff --git a/drivers/infiniband/ulp/iser/iser_verbs.c b/drivers/infiniband/ulp/iser/iser_verbs.c
index c1f44c4..699e075 100644
--- a/drivers/infiniband/ulp/iser/iser_verbs.c
+++ b/drivers/infiniband/ulp/iser/iser_verbs.c
@@ -68,59 +68,23 @@
static int iser_create_device_ib_res(struct iser_device *device)
{
struct ib_device *ib_dev = device->ib_device;
- int i, max_cqe;
if (!(ib_dev->attrs.device_cap_flags & IB_DEVICE_MEM_MGT_EXTENSIONS)) {
iser_err("IB device does not support memory registrations\n");
return -1;
}
- device->comps_used = min_t(int, num_online_cpus(),
- ib_dev->num_comp_vectors);
-
- device->comps = kcalloc(device->comps_used, sizeof(*device->comps),
- GFP_KERNEL);
- if (!device->comps)
- goto comps_err;
-
- max_cqe = min(ISER_MAX_CQ_LEN, ib_dev->attrs.max_cqe);
-
- iser_info("using %d CQs, device %s supports %d vectors max_cqe %d\n",
- device->comps_used, dev_name(&ib_dev->dev),
- ib_dev->num_comp_vectors, max_cqe);
-
device->pd = ib_alloc_pd(ib_dev,
iser_always_reg ? 0 : IB_PD_UNSAFE_GLOBAL_RKEY);
if (IS_ERR(device->pd))
goto pd_err;
- for (i = 0; i < device->comps_used; i++) {
- struct iser_comp *comp = &device->comps[i];
-
- comp->cq = ib_alloc_cq(ib_dev, comp, max_cqe, i,
- IB_POLL_SOFTIRQ);
- if (IS_ERR(comp->cq)) {
- comp->cq = NULL;
- goto cq_err;
- }
- }
-
INIT_IB_EVENT_HANDLER(&device->event_handler, ib_dev,
iser_event_handler);
ib_register_event_handler(&device->event_handler);
return 0;
-cq_err:
- for (i = 0; i < device->comps_used; i++) {
- struct iser_comp *comp = &device->comps[i];
-
- if (comp->cq)
- ib_free_cq(comp->cq);
- }
- ib_dealloc_pd(device->pd);
pd_err:
- kfree(device->comps);
-comps_err:
iser_err("failed to allocate an IB resource\n");
return -1;
}
@@ -131,20 +95,9 @@
*/
static void iser_free_device_ib_res(struct iser_device *device)
{
- int i;
-
- for (i = 0; i < device->comps_used; i++) {
- struct iser_comp *comp = &device->comps[i];
-
- ib_free_cq(comp->cq);
- comp->cq = NULL;
- }
-
ib_unregister_event_handler(&device->event_handler);
ib_dealloc_pd(device->pd);
- kfree(device->comps);
- device->comps = NULL;
device->pd = NULL;
}
@@ -287,70 +240,57 @@
struct ib_device *ib_dev;
struct ib_qp_init_attr init_attr;
int ret = -ENOMEM;
- int index, min_index = 0;
+ unsigned int max_send_wr, cq_size;
BUG_ON(ib_conn->device == NULL);
device = ib_conn->device;
ib_dev = device->ib_device;
- memset(&init_attr, 0, sizeof init_attr);
+ if (ib_conn->pi_support)
+ max_send_wr = ISER_QP_SIG_MAX_REQ_DTOS + 1;
+ else
+ max_send_wr = ISER_QP_MAX_REQ_DTOS + 1;
+ max_send_wr = min_t(unsigned int, max_send_wr,
+ (unsigned int)ib_dev->attrs.max_qp_wr);
- mutex_lock(&ig.connlist_mutex);
- /* select the CQ with the minimal number of usages */
- for (index = 0; index < device->comps_used; index++) {
- if (device->comps[index].active_qps <
- device->comps[min_index].active_qps)
- min_index = index;
+ cq_size = max_send_wr + ISER_QP_MAX_RECV_DTOS;
+ ib_conn->cq = ib_cq_pool_get(ib_dev, cq_size, -1, IB_POLL_SOFTIRQ);
+ if (IS_ERR(ib_conn->cq)) {
+ ret = PTR_ERR(ib_conn->cq);
+ goto cq_err;
}
- ib_conn->comp = &device->comps[min_index];
- ib_conn->comp->active_qps++;
- mutex_unlock(&ig.connlist_mutex);
- iser_info("cq index %d used for ib_conn %p\n", min_index, ib_conn);
+ ib_conn->cq_size = cq_size;
+
+ memset(&init_attr, 0, sizeof(init_attr));
init_attr.event_handler = iser_qp_event_callback;
init_attr.qp_context = (void *)ib_conn;
- init_attr.send_cq = ib_conn->comp->cq;
- init_attr.recv_cq = ib_conn->comp->cq;
+ init_attr.send_cq = ib_conn->cq;
+ init_attr.recv_cq = ib_conn->cq;
init_attr.cap.max_recv_wr = ISER_QP_MAX_RECV_DTOS;
init_attr.cap.max_send_sge = 2;
init_attr.cap.max_recv_sge = 1;
init_attr.sq_sig_type = IB_SIGNAL_REQ_WR;
init_attr.qp_type = IB_QPT_RC;
- if (ib_conn->pi_support) {
- init_attr.cap.max_send_wr = ISER_QP_SIG_MAX_REQ_DTOS + 1;
+ init_attr.cap.max_send_wr = max_send_wr;
+ if (ib_conn->pi_support)
init_attr.create_flags |= IB_QP_CREATE_INTEGRITY_EN;
- iser_conn->max_cmds =
- ISER_GET_MAX_XMIT_CMDS(ISER_QP_SIG_MAX_REQ_DTOS);
- } else {
- if (ib_dev->attrs.max_qp_wr > ISER_QP_MAX_REQ_DTOS) {
- init_attr.cap.max_send_wr = ISER_QP_MAX_REQ_DTOS + 1;
- iser_conn->max_cmds =
- ISER_GET_MAX_XMIT_CMDS(ISER_QP_MAX_REQ_DTOS);
- } else {
- init_attr.cap.max_send_wr = ib_dev->attrs.max_qp_wr;
- iser_conn->max_cmds =
- ISER_GET_MAX_XMIT_CMDS(ib_dev->attrs.max_qp_wr);
- iser_dbg("device %s supports max_send_wr %d\n",
- dev_name(&device->ib_device->dev),
- ib_dev->attrs.max_qp_wr);
- }
- }
+ iser_conn->max_cmds = ISER_GET_MAX_XMIT_CMDS(max_send_wr - 1);
ret = rdma_create_qp(ib_conn->cma_id, device->pd, &init_attr);
if (ret)
goto out_err;
ib_conn->qp = ib_conn->cma_id->qp;
- iser_info("setting conn %p cma_id %p qp %p\n",
+ iser_info("setting conn %p cma_id %p qp %p max_send_wr %d\n",
ib_conn, ib_conn->cma_id,
- ib_conn->cma_id->qp);
+ ib_conn->cma_id->qp, max_send_wr);
return ret;
out_err:
- mutex_lock(&ig.connlist_mutex);
- ib_conn->comp->active_qps--;
- mutex_unlock(&ig.connlist_mutex);
+ ib_cq_pool_put(ib_conn->cq, ib_conn->cq_size);
+cq_err:
iser_err("unable to alloc mem or create resource, err %d\n", ret);
return ret;
@@ -462,10 +402,8 @@
iser_conn, ib_conn->cma_id, ib_conn->qp);
if (ib_conn->qp != NULL) {
- mutex_lock(&ig.connlist_mutex);
- ib_conn->comp->active_qps--;
- mutex_unlock(&ig.connlist_mutex);
rdma_destroy_qp(ib_conn->cma_id);
+ ib_cq_pool_put(ib_conn->cq, ib_conn->cq_size);
ib_conn->qp = NULL;
}
diff --git a/drivers/infiniband/ulp/isert/ib_isert.c b/drivers/infiniband/ulp/isert/ib_isert.c
index b7df38e..61e2f7f 100644
--- a/drivers/infiniband/ulp/isert/ib_isert.c
+++ b/drivers/infiniband/ulp/isert/ib_isert.c
@@ -24,13 +24,6 @@
#include "ib_isert.h"
-#define ISERT_MAX_CONN 8
-#define ISER_MAX_RX_CQ_LEN (ISERT_QP_MAX_RECV_DTOS * ISERT_MAX_CONN)
-#define ISER_MAX_TX_CQ_LEN \
- ((ISERT_QP_MAX_REQ_DTOS + ISCSI_DEF_XMIT_CMDS_MAX) * ISERT_MAX_CONN)
-#define ISER_MAX_CQ_LEN (ISER_MAX_RX_CQ_LEN + ISER_MAX_TX_CQ_LEN + \
- ISERT_MAX_CONN)
-
static int isert_debug_level;
module_param_named(debug_level, isert_debug_level, int, 0644);
MODULE_PARM_DESC(debug_level, "Enable debug tracing if > 0 (default:0)");
@@ -82,53 +75,34 @@
}
}
-static struct isert_comp *
-isert_comp_get(struct isert_conn *isert_conn)
-{
- struct isert_device *device = isert_conn->device;
- struct isert_comp *comp;
- int i, min = 0;
-
- mutex_lock(&device_list_mutex);
- for (i = 0; i < device->comps_used; i++)
- if (device->comps[i].active_qps <
- device->comps[min].active_qps)
- min = i;
- comp = &device->comps[min];
- comp->active_qps++;
- mutex_unlock(&device_list_mutex);
-
- isert_info("conn %p, using comp %p min_index: %d\n",
- isert_conn, comp, min);
-
- return comp;
-}
-
-static void
-isert_comp_put(struct isert_comp *comp)
-{
- mutex_lock(&device_list_mutex);
- comp->active_qps--;
- mutex_unlock(&device_list_mutex);
-}
-
static struct ib_qp *
isert_create_qp(struct isert_conn *isert_conn,
- struct isert_comp *comp,
struct rdma_cm_id *cma_id)
{
+ u32 cq_size = ISERT_QP_MAX_REQ_DTOS + ISERT_QP_MAX_RECV_DTOS + 2;
struct isert_device *device = isert_conn->device;
+ struct ib_device *ib_dev = device->ib_device;
struct ib_qp_init_attr attr;
- int ret;
+ int ret, factor;
+
+ isert_conn->cq = ib_cq_pool_get(ib_dev, cq_size, -1, IB_POLL_WORKQUEUE);
+ if (IS_ERR(isert_conn->cq)) {
+ isert_err("Unable to allocate cq\n");
+ ret = PTR_ERR(isert_conn->cq);
+ return ERR_PTR(ret);
+ }
+ isert_conn->cq_size = cq_size;
memset(&attr, 0, sizeof(struct ib_qp_init_attr));
attr.event_handler = isert_qp_event_callback;
attr.qp_context = isert_conn;
- attr.send_cq = comp->cq;
- attr.recv_cq = comp->cq;
+ attr.send_cq = isert_conn->cq;
+ attr.recv_cq = isert_conn->cq;
attr.cap.max_send_wr = ISERT_QP_MAX_REQ_DTOS + 1;
attr.cap.max_recv_wr = ISERT_QP_MAX_RECV_DTOS + 1;
- attr.cap.max_rdma_ctxs = ISCSI_DEF_XMIT_CMDS_MAX;
+ factor = rdma_rw_mr_factor(device->ib_device, cma_id->port_num,
+ ISCSI_ISER_MAX_SG_TABLESIZE);
+ attr.cap.max_rdma_ctxs = ISCSI_DEF_XMIT_CMDS_MAX * factor;
attr.cap.max_send_sge = device->ib_device->attrs.max_send_sge;
attr.cap.max_recv_sge = 1;
attr.sq_sig_type = IB_SIGNAL_REQ_WR;
@@ -139,6 +113,8 @@
ret = rdma_create_qp(cma_id, device->pd, &attr);
if (ret) {
isert_err("rdma_create_qp failed for cma_id %d\n", ret);
+ ib_cq_pool_put(isert_conn->cq, isert_conn->cq_size);
+
return ERR_PTR(ret);
}
@@ -146,25 +122,6 @@
}
static int
-isert_conn_setup_qp(struct isert_conn *isert_conn, struct rdma_cm_id *cma_id)
-{
- struct isert_comp *comp;
- int ret;
-
- comp = isert_comp_get(isert_conn);
- isert_conn->qp = isert_create_qp(isert_conn, comp, cma_id);
- if (IS_ERR(isert_conn->qp)) {
- ret = PTR_ERR(isert_conn->qp);
- goto err;
- }
-
- return 0;
-err:
- isert_comp_put(comp);
- return ret;
-}
-
-static int
isert_alloc_rx_descriptors(struct isert_conn *isert_conn)
{
struct isert_device *device = isert_conn->device;
@@ -231,61 +188,6 @@
isert_conn->rx_descs = NULL;
}
-static void
-isert_free_comps(struct isert_device *device)
-{
- int i;
-
- for (i = 0; i < device->comps_used; i++) {
- struct isert_comp *comp = &device->comps[i];
-
- if (comp->cq)
- ib_free_cq(comp->cq);
- }
- kfree(device->comps);
-}
-
-static int
-isert_alloc_comps(struct isert_device *device)
-{
- int i, max_cqe, ret = 0;
-
- device->comps_used = min(ISERT_MAX_CQ, min_t(int, num_online_cpus(),
- device->ib_device->num_comp_vectors));
-
- isert_info("Using %d CQs, %s supports %d vectors support "
- "pi_capable %d\n",
- device->comps_used, dev_name(&device->ib_device->dev),
- device->ib_device->num_comp_vectors,
- device->pi_capable);
-
- device->comps = kcalloc(device->comps_used, sizeof(struct isert_comp),
- GFP_KERNEL);
- if (!device->comps)
- return -ENOMEM;
-
- max_cqe = min(ISER_MAX_CQ_LEN, device->ib_device->attrs.max_cqe);
-
- for (i = 0; i < device->comps_used; i++) {
- struct isert_comp *comp = &device->comps[i];
-
- comp->device = device;
- comp->cq = ib_alloc_cq(device->ib_device, comp, max_cqe, i,
- IB_POLL_WORKQUEUE);
- if (IS_ERR(comp->cq)) {
- isert_err("Unable to allocate cq\n");
- ret = PTR_ERR(comp->cq);
- comp->cq = NULL;
- goto out_cq;
- }
- }
-
- return 0;
-out_cq:
- isert_free_comps(device);
- return ret;
-}
-
static int
isert_create_device_ib_res(struct isert_device *device)
{
@@ -296,16 +198,12 @@
ib_dev->attrs.max_send_sge, ib_dev->attrs.max_recv_sge);
isert_dbg("devattr->max_sge_rd: %d\n", ib_dev->attrs.max_sge_rd);
- ret = isert_alloc_comps(device);
- if (ret)
- goto out;
-
device->pd = ib_alloc_pd(ib_dev, 0);
if (IS_ERR(device->pd)) {
ret = PTR_ERR(device->pd);
isert_err("failed to allocate pd, device %p, ret=%d\n",
device, ret);
- goto out_cq;
+ return ret;
}
/* Check signature cap */
@@ -313,13 +211,6 @@
IB_DEVICE_INTEGRITY_HANDOVER ? true : false;
return 0;
-
-out_cq:
- isert_free_comps(device);
-out:
- if (ret > 0)
- ret = -EINVAL;
- return ret;
}
static void
@@ -328,7 +219,6 @@
isert_info("device %p\n", device);
ib_dealloc_pd(device->pd);
- isert_free_comps(device);
}
static void
@@ -490,6 +380,13 @@
}
}
+static void
+isert_destroy_qp(struct isert_conn *isert_conn)
+{
+ ib_destroy_qp(isert_conn->qp);
+ ib_cq_pool_put(isert_conn->cq, isert_conn->cq_size);
+}
+
static int
isert_connect_request(struct rdma_cm_id *cma_id, struct rdma_cm_event *event)
{
@@ -530,17 +427,19 @@
isert_set_nego_params(isert_conn, &event->param.conn);
- ret = isert_conn_setup_qp(isert_conn, cma_id);
- if (ret)
+ isert_conn->qp = isert_create_qp(isert_conn, cma_id);
+ if (IS_ERR(isert_conn->qp)) {
+ ret = PTR_ERR(isert_conn->qp);
goto out_conn_dev;
+ }
ret = isert_login_post_recv(isert_conn);
if (ret)
- goto out_conn_dev;
+ goto out_destroy_qp;
ret = isert_rdma_accept(isert_conn);
if (ret)
- goto out_conn_dev;
+ goto out_destroy_qp;
mutex_lock(&isert_np->mutex);
list_add_tail(&isert_conn->node, &isert_np->accepted);
@@ -548,6 +447,8 @@
return 0;
+out_destroy_qp:
+ isert_destroy_qp(isert_conn);
out_conn_dev:
isert_device_put(device);
out_rsp_dma_map:
@@ -572,12 +473,8 @@
!isert_conn->dev_removed)
rdma_destroy_id(isert_conn->cm_id);
- if (isert_conn->qp) {
- struct isert_comp *comp = isert_conn->qp->recv_cq->cq_context;
-
- isert_comp_put(comp);
- ib_destroy_qp(isert_conn->qp);
- }
+ if (isert_conn->qp)
+ isert_destroy_qp(isert_conn);
if (isert_conn->login_req_buf)
isert_free_login_buf(isert_conn);
diff --git a/drivers/infiniband/ulp/isert/ib_isert.h b/drivers/infiniband/ulp/isert/ib_isert.h
index 3b296ba..c55f7d9 100644
--- a/drivers/infiniband/ulp/isert/ib_isert.h
+++ b/drivers/infiniband/ulp/isert/ib_isert.h
@@ -63,7 +63,8 @@
(ISER_RX_PAYLOAD_SIZE + sizeof(u64) + sizeof(struct ib_sge) + \
sizeof(struct ib_cqe) + sizeof(bool)))
-#define ISCSI_ISER_SG_TABLESIZE 256
+/* Maximum support is 16MB I/O size */
+#define ISCSI_ISER_MAX_SG_TABLESIZE 4096
enum isert_desc_type {
ISCSI_TX_CONTROL,
@@ -155,6 +156,8 @@
struct iser_tx_desc login_tx_desc;
struct rdma_cm_id *cm_id;
struct ib_qp *qp;
+ struct ib_cq *cq;
+ u32 cq_size;
struct isert_device *device;
struct mutex mutex;
struct kref kref;
@@ -165,22 +168,6 @@
bool dev_removed;
};
-#define ISERT_MAX_CQ 64
-
-/**
- * struct isert_comp - iSER completion context
- *
- * @device: pointer to device handle
- * @cq: completion queue
- * @active_qps: Number of active QPs attached
- * to completion context
- */
-struct isert_comp {
- struct isert_device *device;
- struct ib_cq *cq;
- int active_qps;
-};
-
struct isert_device {
bool pi_capable;
int refcount;
diff --git a/drivers/infiniband/ulp/opa_vnic/opa_vnic_encap.h b/drivers/infiniband/ulp/opa_vnic/opa_vnic_encap.h
index d324312..f645198 100644
--- a/drivers/infiniband/ulp/opa_vnic/opa_vnic_encap.h
+++ b/drivers/infiniband/ulp/opa_vnic/opa_vnic_encap.h
@@ -118,12 +118,17 @@
* struct opa_vesw_info - OPA vnic switch information
* @fabric_id: 10-bit fabric id
* @vesw_id: 12-bit virtual ethernet switch id
+ * @rsvd0: reserved bytes
* @def_port_mask: bitmask of default ports
+ * @rsvd1: reserved bytes
* @pkey: partition key
+ * @rsvd2: reserved bytes
* @u_mcast_dlid: unknown multicast dlid
* @u_ucast_dlid: array of unknown unicast dlids
+ * @rsvd3: reserved bytes
* @rc: routing control
* @eth_mtu: Ethernet MTU
+ * @rsvd4: reserved bytes
*/
struct opa_vesw_info {
__be16 fabric_id;
@@ -150,12 +155,14 @@
* struct opa_per_veswport_info - OPA vnic per port information
* @port_num: port number
* @eth_link_status: current ethernet link state
+ * @rsvd0: reserved bytes
* @base_mac_addr: base mac address
* @config_state: configured port state
* @oper_state: operational port state
* @max_mac_tbl_ent: max number of mac table entries
* @max_smac_ent: max smac entries in mac table
* @mac_tbl_digest: mac table digest
+ * @rsvd1: reserved bytes
* @encap_slid: base slid for the port
* @pcp_to_sc_uc: sc by pcp index for unicast ethernet packets
* @pcp_to_vl_uc: vl by pcp index for unicast ethernet packets
@@ -165,8 +172,10 @@
* @non_vlan_vl_uc: vl for non-vlan unicast ethernet packets
* @non_vlan_sc_mc: sc for non-vlan multicast ethernet packets
* @non_vlan_vl_mc: vl for non-vlan multicast ethernet packets
+ * @rsvd2: reserved bytes
* @uc_macs_gen_count: generation count for unicast macs list
* @mc_macs_gen_count: generation count for multicast macs list
+ * @rsvd3: reserved bytes
*/
struct opa_per_veswport_info {
__be32 port_num;
@@ -294,6 +303,7 @@
* @rx_512_1023: received packet length is >=512 and < 1023 bytes
* @rx_1024_1518: received packet length is >=1024 and < 1518 bytes
* @rx_1519_max: received packet length >= 1519 bytes
+ * @reserved: reserved bytes
*
* All the above are counters of corresponding conditions.
*/
@@ -347,16 +357,26 @@
* @veswport_num: virtual ethernet switch port number
* @tx_errors: transmit errors
* @rx_errors: receive errors
+ * @rsvd0: reserved bytes
* @tx_smac_filt: smac filter errors
+ * @rsvd1: reserved bytes
+ * @rsvd2: reserved bytes
+ * @rsvd3: reserved bytes
* @tx_dlid_zero: transmit packets with invalid dlid
+ * @rsvd4: reserved bytes
* @tx_logic: other transmit errors
+ * @rsvd5: reserved bytes
* @tx_drop_state: packet tansmission in non-forward port state
* @rx_bad_veswid: received packet with invalid vesw id
+ * @rsvd6: reserved bytes
* @rx_runt: received ethernet packet with length < 64 bytes
* @rx_oversize: received ethernet packet with length > MTU size
+ * @rsvd7: reserved bytes
* @rx_eth_down: received packets when interface is down
* @rx_drop_state: received packets in non-forwarding port state
* @rx_logic: other receive errors
+ * @rsvd8: reserved bytes
+ * @rsvd9: reserved bytes
*
* All the above are counters of corresponding error conditions.
*/
@@ -447,6 +467,7 @@
* struct opa_vnic_vema_mad - Generic VEMA MAD
* @mad_hdr: Generic MAD header
* @rmpp_hdr: RMPP header for vendor specific MADs
+ * @reserved: reserved bytes
* @oui: Unique org identifier
* @data: MAD data
*/
@@ -467,6 +488,7 @@
* @trap_num: Trap number
* @toggle_count: Notice toggle bit and count value
* @issuer_lid: Trap issuer's lid
+ * @reserved: reserved bytes
* @issuer_gid: Issuer GID (only if Report method)
* @raw_data: Trap message body
*/
@@ -487,6 +509,7 @@
* struct opa_vnic_vema_mad_trap - Generic VEMA MAD Trap
* @mad_hdr: Generic MAD header
* @rmpp_hdr: RMPP header for vendor specific MADs
+ * @reserved: reserved bytes
* @oui: Unique org identifier
* @notice: Notice structure
*/
diff --git a/drivers/infiniband/ulp/rtrs/rtrs-clt.c b/drivers/infiniband/ulp/rtrs/rtrs-clt.c
index 564388a..776e892 100644
--- a/drivers/infiniband/ulp/rtrs/rtrs-clt.c
+++ b/drivers/infiniband/ulp/rtrs/rtrs-clt.c
@@ -12,6 +12,7 @@
#include <linux/module.h>
#include <linux/rculist.h>
+#include <linux/random.h>
#include "rtrs-clt.h"
#include "rtrs-log.h"
@@ -23,6 +24,12 @@
* leads to "false positives" failed reconnect attempts
*/
#define RTRS_RECONNECT_BACKOFF 1000
+/*
+ * Wait for additional random time between 0 and 8 seconds
+ * before starting to reconnect to avoid clients reconnecting
+ * all at once in case of a major network outage
+ */
+#define RTRS_RECONNECT_SEED 8
MODULE_DESCRIPTION("RDMA Transport Client");
MODULE_LICENSE("GPL");
@@ -306,7 +313,8 @@
*/
delay_ms = clt->reconnect_delay_sec * 1000;
queue_delayed_work(rtrs_wq, &sess->reconnect_dwork,
- msecs_to_jiffies(delay_ms));
+ msecs_to_jiffies(delay_ms +
+ prandom_u32() % RTRS_RECONNECT_SEED));
} else {
/*
* Error can happen just on establishing new connection,
@@ -2503,7 +2511,9 @@
sess->stats->reconnects.fail_cnt++;
delay_ms = clt->reconnect_delay_sec * 1000;
queue_delayed_work(rtrs_wq, &sess->reconnect_dwork,
- msecs_to_jiffies(delay_ms));
+ msecs_to_jiffies(delay_ms +
+ prandom_u32() %
+ RTRS_RECONNECT_SEED));
}
}
@@ -2972,7 +2982,7 @@
pr_err("Failed to create rtrs-client dev class\n");
return PTR_ERR(rtrs_clt_dev_class);
}
- rtrs_wq = alloc_workqueue("rtrs_client_wq", WQ_MEM_RECLAIM, 0);
+ rtrs_wq = alloc_workqueue("rtrs_client_wq", 0, 0);
if (!rtrs_wq) {
class_destroy(rtrs_clt_dev_class);
return -ENOMEM;
diff --git a/drivers/infiniband/ulp/rtrs/rtrs-srv.c b/drivers/infiniband/ulp/rtrs/rtrs-srv.c
index 0d9241f..a219bd1 100644
--- a/drivers/infiniband/ulp/rtrs/rtrs-srv.c
+++ b/drivers/infiniband/ulp/rtrs/rtrs-srv.c
@@ -2150,7 +2150,7 @@
err = PTR_ERR(rtrs_dev_class);
goto out_chunk_pool;
}
- rtrs_wq = alloc_workqueue("rtrs_server_wq", WQ_MEM_RECLAIM, 0);
+ rtrs_wq = alloc_workqueue("rtrs_server_wq", 0, 0);
if (!rtrs_wq) {
err = -ENOMEM;
goto out_dev_class;
diff --git a/drivers/infiniband/ulp/srpt/ib_srpt.c b/drivers/infiniband/ulp/srpt/ib_srpt.c
index ef7fcd3..0065eb1 100644
--- a/drivers/infiniband/ulp/srpt/ib_srpt.c
+++ b/drivers/infiniband/ulp/srpt/ib_srpt.c
@@ -869,7 +869,7 @@
static void srpt_zerolength_write_done(struct ib_cq *cq, struct ib_wc *wc)
{
- struct srpt_rdma_ch *ch = cq->cq_context;
+ struct srpt_rdma_ch *ch = wc->qp->qp_context;
pr_debug("%s-%d wc->status %d\n", ch->sess_name, ch->qp->qp_num,
wc->status);
@@ -1322,7 +1322,7 @@
*/
static void srpt_rdma_read_done(struct ib_cq *cq, struct ib_wc *wc)
{
- struct srpt_rdma_ch *ch = cq->cq_context;
+ struct srpt_rdma_ch *ch = wc->qp->qp_context;
struct srpt_send_ioctx *ioctx =
container_of(wc->wr_cqe, struct srpt_send_ioctx, rdma_cqe);
@@ -1683,7 +1683,7 @@
static void srpt_recv_done(struct ib_cq *cq, struct ib_wc *wc)
{
- struct srpt_rdma_ch *ch = cq->cq_context;
+ struct srpt_rdma_ch *ch = wc->qp->qp_context;
struct srpt_recv_ioctx *ioctx =
container_of(wc->wr_cqe, struct srpt_recv_ioctx, ioctx.cqe);
@@ -1744,7 +1744,7 @@
*/
static void srpt_send_done(struct ib_cq *cq, struct ib_wc *wc)
{
- struct srpt_rdma_ch *ch = cq->cq_context;
+ struct srpt_rdma_ch *ch = wc->qp->qp_context;
struct srpt_send_ioctx *ioctx =
container_of(wc->wr_cqe, struct srpt_send_ioctx, ioctx.cqe);
enum srpt_command_state state;
@@ -1791,7 +1791,7 @@
goto out;
retry:
- ch->cq = ib_alloc_cq_any(sdev->device, ch, ch->rq_size + sq_size,
+ ch->cq = ib_cq_pool_get(sdev->device, ch->rq_size + sq_size, -1,
IB_POLL_WORKQUEUE);
if (IS_ERR(ch->cq)) {
ret = PTR_ERR(ch->cq);
@@ -1799,6 +1799,7 @@
ch->rq_size + sq_size, ret);
goto out;
}
+ ch->cq_size = ch->rq_size + sq_size;
qp_init->qp_context = (void *)ch;
qp_init->event_handler
@@ -1843,7 +1844,7 @@
if (retry) {
pr_debug("failed to create queue pair with sq_size = %d (%d) - retrying\n",
sq_size, ret);
- ib_free_cq(ch->cq);
+ ib_cq_pool_put(ch->cq, ch->cq_size);
sq_size = max(sq_size / 2, MIN_SRPT_SQ_SIZE);
goto retry;
} else {
@@ -1869,14 +1870,14 @@
err_destroy_cq:
ch->qp = NULL;
- ib_free_cq(ch->cq);
+ ib_cq_pool_put(ch->cq, ch->cq_size);
goto out;
}
static void srpt_destroy_ch_ib(struct srpt_rdma_ch *ch)
{
ib_destroy_qp(ch->qp);
- ib_free_cq(ch->cq);
+ ib_cq_pool_put(ch->cq, ch->cq_size);
}
/**
@@ -2156,9 +2157,6 @@
WARN_ON_ONCE(irqs_disabled());
- if (WARN_ON(!sdev || !req))
- return -EINVAL;
-
it_iu_len = be32_to_cpu(req->req_it_iu_len);
pr_info("Received SRP_LOGIN_REQ with i_port_id %pI6, t_port_id %pI6 and it_iu_len %d on port %d (guid=%pI6); pkey %#04x\n",
diff --git a/drivers/infiniband/ulp/srpt/ib_srpt.h b/drivers/infiniband/ulp/srpt/ib_srpt.h
index f31c349..41435a6 100644
--- a/drivers/infiniband/ulp/srpt/ib_srpt.h
+++ b/drivers/infiniband/ulp/srpt/ib_srpt.h
@@ -300,6 +300,7 @@
} rdma_cm;
};
struct ib_cq *cq;
+ u32 cq_size;
struct ib_cqe zw_cqe;
struct rcu_head rcu;
struct kref kref;
diff --git a/drivers/macintosh/adb-iop.c b/drivers/macintosh/adb-iop.c
index fca3164..f3d1a46 100644
--- a/drivers/macintosh/adb-iop.c
+++ b/drivers/macintosh/adb-iop.c
@@ -7,10 +7,6 @@
* 1999-07-01 (jmt) - First implementation for new driver architecture.
*
* 1999-07-31 (jmt) - First working version.
- *
- * TODO:
- *
- * o Implement SRQ handling.
*/
#include <linux/types.h>
@@ -18,24 +14,17 @@
#include <linux/mm.h>
#include <linux/delay.h>
#include <linux/init.h>
-#include <linux/proc_fs.h>
#include <asm/macintosh.h>
#include <asm/macints.h>
#include <asm/mac_iop.h>
-#include <asm/mac_oss.h>
#include <asm/adb_iop.h>
#include <linux/adb.h>
-/*#define DEBUG_ADB_IOP*/
-
static struct adb_request *current_req;
static struct adb_request *last_req;
-#if 0
-static unsigned char reply_buff[16];
-static unsigned char *reply_ptr;
-#endif
+static unsigned int autopoll_devs;
static enum adb_iop_state {
idle,
@@ -62,13 +51,19 @@
.reset_bus = adb_iop_reset_bus
};
-static void adb_iop_end_req(struct adb_request *req, int state)
+static void adb_iop_done(void)
{
+ struct adb_request *req = current_req;
+
+ adb_iop_state = idle;
+
req->complete = 1;
current_req = req->next;
if (req->done)
(*req->done)(req);
- adb_iop_state = state;
+
+ if (adb_iop_state == idle)
+ adb_iop_start();
}
/*
@@ -79,15 +74,14 @@
static void adb_iop_complete(struct iop_msg *msg)
{
- struct adb_request *req;
unsigned long flags;
local_irq_save(flags);
- req = current_req;
- if ((adb_iop_state == sending) && req && req->reply_expected) {
+ if (current_req->reply_expected)
adb_iop_state = awaiting_reply;
- }
+ else
+ adb_iop_done();
local_irq_restore(flags);
}
@@ -102,52 +96,36 @@
static void adb_iop_listen(struct iop_msg *msg)
{
struct adb_iopmsg *amsg = (struct adb_iopmsg *)msg->message;
- struct adb_request *req;
unsigned long flags;
-#ifdef DEBUG_ADB_IOP
- int i;
-#endif
+ bool req_done = false;
local_irq_save(flags);
- req = current_req;
+ /* Handle a timeout. Timeout packets seem to occur even after
+ * we've gotten a valid reply to a TALK, presumably because of
+ * autopolling.
+ */
-#ifdef DEBUG_ADB_IOP
- printk("adb_iop_listen %p: rcvd packet, %d bytes: %02X %02X", req,
- (uint)amsg->count + 2, (uint)amsg->flags, (uint)amsg->cmd);
- for (i = 0; i < amsg->count; i++)
- printk(" %02X", (uint)amsg->data[i]);
- printk("\n");
-#endif
+ if (amsg->flags & ADB_IOP_EXPLICIT) {
+ if (adb_iop_state == awaiting_reply) {
+ struct adb_request *req = current_req;
- /* Handle a timeout. Timeout packets seem to occur even after */
- /* we've gotten a valid reply to a TALK, so I'm assuming that */
- /* a "timeout" is actually more like an "end-of-data" signal. */
- /* We need to send back a timeout packet to the IOP to shut */
- /* it up, plus complete the current request, if any. */
-
- if (amsg->flags & ADB_IOP_TIMEOUT) {
- msg->reply[0] = ADB_IOP_TIMEOUT | ADB_IOP_AUTOPOLL;
- msg->reply[1] = 0;
- msg->reply[2] = 0;
- if (req && (adb_iop_state != idle)) {
- adb_iop_end_req(req, idle);
- }
- } else {
- /* TODO: is it possible for more than one chunk of data */
- /* to arrive before the timeout? If so we need to */
- /* use reply_ptr here like the other drivers do. */
- if ((adb_iop_state == awaiting_reply) &&
- (amsg->flags & ADB_IOP_EXPLICIT)) {
req->reply_len = amsg->count + 1;
memcpy(req->reply, &amsg->cmd, req->reply_len);
- } else {
- adb_input(&amsg->cmd, amsg->count + 1,
- amsg->flags & ADB_IOP_AUTOPOLL);
+
+ req_done = true;
}
- memcpy(msg->reply, msg->message, IOP_MSG_LEN);
+ } else if (!(amsg->flags & ADB_IOP_TIMEOUT)) {
+ adb_input(&amsg->cmd, amsg->count + 1,
+ amsg->flags & ADB_IOP_AUTOPOLL);
}
+
+ msg->reply[0] = autopoll_devs ? ADB_IOP_AUTOPOLL : 0;
iop_complete_message(msg);
+
+ if (req_done)
+ adb_iop_done();
+
local_irq_restore(flags);
}
@@ -160,63 +138,50 @@
static void adb_iop_start(void)
{
- unsigned long flags;
struct adb_request *req;
struct adb_iopmsg amsg;
-#ifdef DEBUG_ADB_IOP
- int i;
-#endif
/* get the packet to send */
req = current_req;
if (!req)
return;
- local_irq_save(flags);
-
-#ifdef DEBUG_ADB_IOP
- printk("adb_iop_start %p: sending packet, %d bytes:", req, req->nbytes);
- for (i = 0; i < req->nbytes; i++)
- printk(" %02X", (uint)req->data[i]);
- printk("\n");
-#endif
-
- /* The IOP takes MacII-style packets, so */
- /* strip the initial ADB_PACKET byte. */
-
+ /* The IOP takes MacII-style packets, so strip the initial
+ * ADB_PACKET byte.
+ */
amsg.flags = ADB_IOP_EXPLICIT;
amsg.count = req->nbytes - 2;
- /* amsg.data immediately follows amsg.cmd, effectively making */
- /* amsg.cmd a pointer to the beginning of a full ADB packet. */
+ /* amsg.data immediately follows amsg.cmd, effectively making
+ * &amsg.cmd a pointer to the beginning of a full ADB packet.
+ */
memcpy(&amsg.cmd, req->data + 1, req->nbytes - 1);
req->sent = 1;
adb_iop_state = sending;
- local_irq_restore(flags);
- /* Now send it. The IOP manager will call adb_iop_complete */
- /* when the packet has been sent. */
-
+ /* Now send it. The IOP manager will call adb_iop_complete
+ * when the message has been sent.
+ */
iop_send_message(ADB_IOP, ADB_CHAN, req, sizeof(amsg), (__u8 *)&amsg,
adb_iop_complete);
}
-int adb_iop_probe(void)
+static int adb_iop_probe(void)
{
if (!iop_ism_present)
return -ENODEV;
return 0;
}
-int adb_iop_init(void)
+static int adb_iop_init(void)
{
pr_info("adb: IOP ISM driver v0.4 for Unified ADB\n");
iop_listen(ADB_IOP, ADB_CHAN, adb_iop_listen, "ADB");
return 0;
}
-int adb_iop_send_request(struct adb_request *req, int sync)
+static int adb_iop_send_request(struct adb_request *req, int sync)
{
int err;
@@ -240,14 +205,14 @@
return -EINVAL;
}
- local_irq_save(flags);
-
req->next = NULL;
req->sent = 0;
req->complete = 0;
req->reply_len = 0;
- if (current_req != 0) {
+ local_irq_save(flags);
+
+ if (current_req) {
last_req->next = req;
last_req = req;
} else {
@@ -255,39 +220,58 @@
last_req = req;
}
+ if (adb_iop_state == idle)
+ adb_iop_start();
+
local_irq_restore(flags);
- if (adb_iop_state == idle)
- adb_iop_start();
return 0;
}
-int adb_iop_autopoll(int devs)
+static void adb_iop_set_ap_complete(struct iop_msg *msg)
{
- /* TODO: how do we enable/disable autopoll? */
+ struct adb_iopmsg *amsg = (struct adb_iopmsg *)msg->message;
+
+ autopoll_devs = (amsg->data[1] << 8) | amsg->data[0];
+}
+
+static int adb_iop_autopoll(int devs)
+{
+ struct adb_iopmsg amsg;
+ unsigned long flags;
+ unsigned int mask = (unsigned int)devs & 0xFFFE;
+
+ local_irq_save(flags);
+
+ amsg.flags = ADB_IOP_SET_AUTOPOLL | (mask ? ADB_IOP_AUTOPOLL : 0);
+ amsg.count = 2;
+ amsg.cmd = 0;
+ amsg.data[0] = mask & 0xFF;
+ amsg.data[1] = (mask >> 8) & 0xFF;
+
+ iop_send_message(ADB_IOP, ADB_CHAN, NULL, sizeof(amsg), (__u8 *)&amsg,
+ adb_iop_set_ap_complete);
+
+ local_irq_restore(flags);
+
return 0;
}
-void adb_iop_poll(void)
+static void adb_iop_poll(void)
{
- if (adb_iop_state == idle)
- adb_iop_start();
iop_ism_irq_poll(ADB_IOP);
}
-int adb_iop_reset_bus(void)
+static int adb_iop_reset_bus(void)
{
- struct adb_request req = {
- .reply_expected = 0,
- .nbytes = 2,
- .data = { ADB_PACKET, 0 },
- };
+ struct adb_request req;
- adb_iop_write(&req);
- while (!req.complete) {
- adb_iop_poll();
- schedule();
- }
+ /* Command = 0, Address = ignored */
+ adb_request(&req, NULL, ADBREQ_NOSEND, 1, ADB_BUSRESET);
+ adb_iop_send_request(&req, 1);
+
+ /* Don't want any more requests during the Global Reset low time. */
+ mdelay(3);
return 0;
}
diff --git a/drivers/macintosh/adb.c b/drivers/macintosh/adb.c
index e49d1f2..73b3961 100644
--- a/drivers/macintosh/adb.c
+++ b/drivers/macintosh/adb.c
@@ -163,7 +163,7 @@
* See if anybody actually moved. This is suggested
* by HW TechNote 01:
*
- * http://developer.apple.com/technotes/hw/hw_01.html
+ * https://developer.apple.com/technotes/hw/hw_01.html
*/
adb_request(&req, NULL, ADBREQ_SYNC | ADBREQ_REPLY, 1,
(highFree << 4) | 0xf);
diff --git a/drivers/macintosh/therm_adt746x.c b/drivers/macintosh/therm_adt746x.c
index 8f7725d..7e21843 100644
--- a/drivers/macintosh/therm_adt746x.c
+++ b/drivers/macintosh/therm_adt746x.c
@@ -5,8 +5,8 @@
* Copyright (C) 2003, 2004 Colin Leroy, Rasmus Rohde, Benjamin Herrenschmidt
*
* Documentation from 115254175ADT7467_pra.pdf and 3686221171167ADT7460_b.pdf
- * http://www.onsemi.com/PowerSolutions/product.do?id=ADT7467
- * http://www.onsemi.com/PowerSolutions/product.do?id=ADT7460
+ * https://www.onsemi.com/PowerSolutions/product.do?id=ADT7467
+ * https://www.onsemi.com/PowerSolutions/product.do?id=ADT7460
*
*/
diff --git a/drivers/macintosh/via-macii.c b/drivers/macintosh/via-macii.c
index ac824d7..060e03f 100644
--- a/drivers/macintosh/via-macii.c
+++ b/drivers/macintosh/via-macii.c
@@ -77,6 +77,12 @@
#define ST_ODD 0x20 /* ADB state: odd data byte */
#define ST_IDLE 0x30 /* ADB state: idle, nothing to send */
+/* ADB command byte structure */
+#define ADDR_MASK 0xF0
+#define CMD_MASK 0x0F
+#define OP_MASK 0x0C
+#define TALK 0x0C
+
static int macii_init_via(void);
static void macii_start(void);
static irqreturn_t macii_interrupt(int irq, void *arg);
@@ -104,21 +110,22 @@
idle,
sending,
reading,
- read_done,
} macii_state;
static struct adb_request *current_req; /* first request struct in the queue */
static struct adb_request *last_req; /* last request struct in the queue */
static unsigned char reply_buf[16]; /* storage for autopolled replies */
static unsigned char *reply_ptr; /* next byte in reply_buf or req->reply */
-static int reading_reply; /* store reply in reply_buf else req->reply */
+static bool reading_reply; /* store reply in reply_buf else req->reply */
static int data_index; /* index of the next byte to send from req->data */
static int reply_len; /* number of bytes received in reply_buf or req->reply */
static int status; /* VIA's ADB status bits captured upon interrupt */
-static int last_status; /* status bits as at previous interrupt */
-static int srq_asserted; /* have to poll for the device that asserted it */
-static int command_byte; /* the most recent command byte transmitted */
-static int autopoll_devs; /* bits set are device addresses to be polled */
+static bool bus_timeout; /* no data was sent by the device */
+static bool srq_asserted; /* have to poll for the device that asserted it */
+static u8 last_cmd; /* the most recent command byte transmitted */
+static u8 last_talk_cmd; /* the most recent Talk command byte transmitted */
+static u8 last_poll_cmd; /* the most recent Talk R0 command byte transmitted */
+static unsigned int autopoll_devs; /* bits set are device addresses to poll */
/* Check for MacII style ADB */
static int macii_probe(void)
@@ -133,7 +140,7 @@
}
/* Initialize the driver */
-int macii_init(void)
+static int macii_init(void)
{
unsigned long flags;
int err;
@@ -165,7 +172,6 @@
/* Set up state: idle */
via[B] |= ST_IDLE;
- last_status = via[B] & (ST_MASK | CTLR_IRQ);
/* Shift register on input */
via[ACR] = (via[ACR] & ~SR_CTRL) | SR_EXT;
@@ -179,35 +185,49 @@
/* Send an ADB poll (Talk Register 0 command prepended to the request queue) */
static void macii_queue_poll(void)
{
- /* No point polling the active device as it will never assert SRQ, so
- * poll the next device in the autopoll list. This could leave us
- * stuck in a polling loop if an unprobed device is asserting SRQ.
- * In theory, that could only happen if a device was plugged in after
- * probing started. Unplugging it again will break the cycle.
- * (Simply polling the next higher device often ends up polling almost
- * every device (after wrapping around), which takes too long.)
- */
- int device_mask;
- int next_device;
static struct adb_request req;
+ unsigned char poll_command;
+ unsigned int poll_addr;
+ /* This only polls devices in the autopoll list, which assumes that
+ * unprobed devices never assert SRQ. That could happen if a device was
+ * plugged in after the adb bus scan. Unplugging it again will resolve
+ * the problem. This behaviour is similar to MacOS.
+ */
if (!autopoll_devs)
return;
- device_mask = (1 << (((command_byte & 0xF0) >> 4) + 1)) - 1;
- if (autopoll_devs & ~device_mask)
- next_device = ffs(autopoll_devs & ~device_mask) - 1;
- else
- next_device = ffs(autopoll_devs) - 1;
+ /* The device most recently polled may not be the best device to poll
+ * right now. Some other device(s) may have signalled SRQ (the active
+ * device won't do that). Or the autopoll list may have been changed.
+ * Try polling the next higher address.
+ */
+ poll_addr = (last_poll_cmd & ADDR_MASK) >> 4;
+ if ((srq_asserted && last_cmd == last_poll_cmd) ||
+ !(autopoll_devs & (1 << poll_addr))) {
+ unsigned int higher_devs;
- adb_request(&req, NULL, ADBREQ_NOSEND, 1, ADB_READREG(next_device, 0));
+ higher_devs = autopoll_devs & -(1 << (poll_addr + 1));
+ poll_addr = ffs(higher_devs ? higher_devs : autopoll_devs) - 1;
+ }
+
+ /* Send a Talk Register 0 command */
+ poll_command = ADB_READREG(poll_addr, 0);
+
+ /* No need to repeat this Talk command. The transceiver will do that
+ * as long as it is idle.
+ */
+ if (poll_command == last_cmd)
+ return;
+
+ adb_request(&req, NULL, ADBREQ_NOSEND, 1, poll_command);
req.sent = 0;
req.complete = 0;
req.reply_len = 0;
req.next = current_req;
- if (current_req != NULL) {
+ if (WARN_ON(current_req)) {
current_req = &req;
} else {
current_req = &req;
@@ -266,40 +286,22 @@
/* Start auto-polling */
static int macii_autopoll(int devs)
{
- static struct adb_request req;
unsigned long flags;
- int err = 0;
-
- /* bit 1 == device 1, and so on. */
- autopoll_devs = devs & 0xFFFE;
-
- if (!autopoll_devs)
- return 0;
local_irq_save(flags);
- if (current_req == NULL) {
- /* Send a Talk Reg 0. The controller will repeatedly transmit
- * this as long as it is idle.
- */
- adb_request(&req, NULL, ADBREQ_NOSEND, 1,
- ADB_READREG(ffs(autopoll_devs) - 1, 0));
- err = macii_write(&req);
+ /* bit 1 == device 1, and so on. */
+ autopoll_devs = (unsigned int)devs & 0xFFFE;
+
+ if (!current_req) {
+ macii_queue_poll();
+ if (current_req && macii_state == idle)
+ macii_start();
}
local_irq_restore(flags);
- return err;
-}
-static inline int need_autopoll(void)
-{
- /* Was the last command Talk Reg 0
- * and is the target on the autopoll list?
- */
- if ((command_byte & 0x0F) == 0x0C &&
- ((1 << ((command_byte & 0xF0) >> 4)) & autopoll_devs))
- return 0;
- return 1;
+ return 0;
}
/* Prod the chip without interrupts */
@@ -311,7 +313,7 @@
/* Reset the bus */
static int macii_reset_bus(void)
{
- static struct adb_request req;
+ struct adb_request req;
/* Command = 0, Address = ignored */
adb_request(&req, NULL, ADBREQ_NOSEND, 1, ADB_BUSRESET);
@@ -335,8 +337,6 @@
* And req->nbytes is the number of bytes of real data plus one.
*/
- /* store command byte */
- command_byte = req->data[1];
/* Output mode */
via[ACR] |= SR_OUT;
/* Load data */
@@ -346,6 +346,9 @@
macii_state = sending;
data_index = 2;
+
+ bus_timeout = false;
+ srq_asserted = false;
}
/*
@@ -354,15 +357,17 @@
* generating shift register interrupts (SR_INT) for us. This means there has
* to be activity on the ADB bus. The chip will poll to achieve this.
*
- * The basic ADB state machine was left unchanged from the original MacII code
- * by Alan Cox, which was based on the CUDA driver for PowerMac.
- * The syntax of the ADB status lines is totally different on MacII,
- * though. MacII uses the states Command -> Even -> Odd -> Even ->...-> Idle
- * for sending and Idle -> Even -> Odd -> Even ->...-> Idle for receiving.
- * Start and end of a receive packet are signalled by asserting /IRQ on the
- * interrupt line (/IRQ means the CTLR_IRQ bit in port B; not to be confused
- * with the VIA shift register interrupt. /IRQ never actually interrupts the
- * processor, it's just an ordinary input.)
+ * The VIA Port B output signalling works as follows. After the ADB transceiver
+ * sees a transition on the PB4 and PB5 lines it will crank over the VIA shift
+ * register which eventually raises the SR_INT interrupt. The PB4/PB5 outputs
+ * are toggled with each byte as the ADB transaction progresses.
+ *
+ * Request with no reply expected (and empty transceiver buffer):
+ * CMD -> IDLE
+ * Request with expected reply packet (or with buffered autopoll packet):
+ * CMD -> EVEN -> ODD -> EVEN -> ... -> IDLE
+ * Unsolicited packet:
+ * IDLE -> EVEN -> ODD -> EVEN -> ... -> IDLE
*/
static irqreturn_t macii_interrupt(int irq, void *arg)
{
@@ -382,31 +387,31 @@
}
}
- last_status = status;
status = via[B] & (ST_MASK | CTLR_IRQ);
switch (macii_state) {
case idle:
- if (reading_reply) {
- reply_ptr = current_req->reply;
- } else {
- WARN_ON(current_req);
- reply_ptr = reply_buf;
- }
+ WARN_ON((status & ST_MASK) != ST_IDLE);
+
+ reply_ptr = reply_buf;
+ reading_reply = false;
+
+ bus_timeout = false;
+ srq_asserted = false;
x = via[SR];
- if ((status & CTLR_IRQ) && (x == 0xFF)) {
- /* Bus timeout without SRQ sequence:
- * data is "FF" while CTLR_IRQ is "H"
+ if (!(status & CTLR_IRQ)) {
+ /* /CTLR_IRQ asserted in idle state means we must
+ * read an autopoll reply from the transceiver buffer.
*/
- reply_len = 0;
- srq_asserted = 0;
- macii_state = read_done;
- } else {
macii_state = reading;
*reply_ptr = x;
reply_len = 1;
+ } else {
+ /* bus timeout */
+ reply_len = 0;
+ break;
}
/* set ADB state = even for first data byte */
@@ -415,41 +420,83 @@
case sending:
req = current_req;
- if (data_index >= req->nbytes) {
+
+ if (status == (ST_CMD | CTLR_IRQ)) {
+ /* /CTLR_IRQ de-asserted after the command byte means
+ * the host can continue with the transaction.
+ */
+
+ /* Store command byte */
+ last_cmd = req->data[1];
+ if ((last_cmd & OP_MASK) == TALK) {
+ last_talk_cmd = last_cmd;
+ if ((last_cmd & CMD_MASK) == ADB_READREG(0, 0))
+ last_poll_cmd = last_cmd;
+ }
+ }
+
+ if (status == ST_CMD) {
+ /* /CTLR_IRQ asserted after the command byte means we
+ * must read an autopoll reply. The first byte was
+ * lost because the shift register was an output.
+ */
+ macii_state = reading;
+
+ reading_reply = false;
+ reply_ptr = reply_buf;
+ *reply_ptr = last_talk_cmd;
+ reply_len = 1;
+
+ /* reset to shift in */
+ via[ACR] &= ~SR_OUT;
+ x = via[SR];
+ } else if (data_index >= req->nbytes) {
req->sent = 1;
- macii_state = idle;
if (req->reply_expected) {
- reading_reply = 1;
- } else {
+ macii_state = reading;
+
+ reading_reply = true;
+ reply_ptr = req->reply;
+ *reply_ptr = req->data[1];
+ reply_len = 1;
+
+ via[ACR] &= ~SR_OUT;
+ x = via[SR];
+ } else if ((req->data[1] & OP_MASK) == TALK) {
+ macii_state = reading;
+
+ reading_reply = false;
+ reply_ptr = reply_buf;
+ *reply_ptr = req->data[1];
+ reply_len = 1;
+
+ via[ACR] &= ~SR_OUT;
+ x = via[SR];
+
req->complete = 1;
current_req = req->next;
if (req->done)
(*req->done)(req);
+ } else {
+ macii_state = idle;
- if (current_req)
- macii_start();
- else if (need_autopoll())
- macii_autopoll(autopoll_devs);
- }
-
- if (macii_state == idle) {
- /* reset to shift in */
- via[ACR] &= ~SR_OUT;
- x = via[SR];
- /* set ADB state idle - might get SRQ */
- via[B] = (via[B] & ~ST_MASK) | ST_IDLE;
+ req->complete = 1;
+ current_req = req->next;
+ if (req->done)
+ (*req->done)(req);
+ break;
}
} else {
via[SR] = req->data[data_index++];
+ }
- if ((via[B] & ST_MASK) == ST_CMD) {
- /* just sent the command byte, set to EVEN */
- via[B] = (via[B] & ~ST_MASK) | ST_EVEN;
- } else {
- /* invert state bits, toggle ODD/EVEN */
- via[B] ^= ST_MASK;
- }
+ if ((via[B] & ST_MASK) == ST_CMD) {
+ /* just sent the command byte, set to EVEN */
+ via[B] = (via[B] & ~ST_MASK) | ST_EVEN;
+ } else {
+ /* invert state bits, toggle ODD/EVEN */
+ via[B] ^= ST_MASK;
}
break;
@@ -458,33 +505,35 @@
WARN_ON((status & ST_MASK) == ST_CMD ||
(status & ST_MASK) == ST_IDLE);
- /* Bus timeout with SRQ sequence:
- * data is "XX FF" while CTLR_IRQ is "L L"
- * End of packet without SRQ sequence:
- * data is "XX...YY 00" while CTLR_IRQ is "L...H L"
- * End of packet SRQ sequence:
- * data is "XX...YY 00" while CTLR_IRQ is "L...L L"
- * (where XX is the first response byte and
- * YY is the last byte of valid response data.)
- */
-
- srq_asserted = 0;
if (!(status & CTLR_IRQ)) {
- if (x == 0xFF) {
- if (!(last_status & CTLR_IRQ)) {
- macii_state = read_done;
+ if (status == ST_EVEN && reply_len == 1) {
+ bus_timeout = true;
+ } else if (status == ST_ODD && reply_len == 2) {
+ srq_asserted = true;
+ } else {
+ macii_state = idle;
+
+ if (bus_timeout)
reply_len = 0;
- srq_asserted = 1;
+
+ if (reading_reply) {
+ struct adb_request *req = current_req;
+
+ req->reply_len = reply_len;
+
+ req->complete = 1;
+ current_req = req->next;
+ if (req->done)
+ (*req->done)(req);
+ } else if (reply_len && autopoll_devs &&
+ reply_buf[0] == last_poll_cmd) {
+ adb_input(reply_buf, reply_len, 1);
}
- } else if (x == 0x00) {
- macii_state = read_done;
- if (!(last_status & CTLR_IRQ))
- srq_asserted = 1;
+ break;
}
}
- if (macii_state == reading &&
- reply_len < ARRAY_SIZE(reply_buf)) {
+ if (reply_len < ARRAY_SIZE(reply_buf)) {
reply_ptr++;
*reply_ptr = x;
reply_len++;
@@ -494,37 +543,22 @@
via[B] ^= ST_MASK;
break;
- case read_done:
- x = via[SR];
+ default:
+ break;
+ }
- if (reading_reply) {
- reading_reply = 0;
- req = current_req;
- req->reply_len = reply_len;
- req->complete = 1;
- current_req = req->next;
- if (req->done)
- (*req->done)(req);
- } else if (reply_len && autopoll_devs)
- adb_input(reply_buf, reply_len, 0);
-
- macii_state = idle;
-
- /* SRQ seen before, initiate poll now */
- if (srq_asserted)
+ if (macii_state == idle) {
+ if (!current_req)
macii_queue_poll();
if (current_req)
macii_start();
- else if (need_autopoll())
- macii_autopoll(autopoll_devs);
- if (macii_state == idle)
+ if (macii_state == idle) {
+ via[ACR] &= ~SR_OUT;
+ x = via[SR];
via[B] = (via[B] & ~ST_MASK) | ST_IDLE;
- break;
-
- default:
- break;
+ }
}
local_irq_restore(flags);
diff --git a/drivers/md/Makefile b/drivers/md/Makefile
index 31840f9..6d3e234 100644
--- a/drivers/md/Makefile
+++ b/drivers/md/Makefile
@@ -43,6 +43,9 @@
obj-$(CONFIG_MD_CLUSTER) += md-cluster.o
obj-$(CONFIG_BCACHE) += bcache/
obj-$(CONFIG_BLK_DEV_MD) += md-mod.o
+ifeq ($(CONFIG_BLK_DEV_MD),y)
+obj-y += md-autodetect.o
+endif
obj-$(CONFIG_BLK_DEV_DM) += dm-mod.o
obj-$(CONFIG_BLK_DEV_DM_BUILTIN) += dm-builtin.o
obj-$(CONFIG_DM_UNSTRIPED) += dm-unstripe.o
diff --git a/drivers/md/dm-writecache.c b/drivers/md/dm-writecache.c
index cfea054..86dbe0c 100644
--- a/drivers/md/dm-writecache.c
+++ b/drivers/md/dm-writecache.c
@@ -538,7 +538,7 @@
static void writecache_commit_flushed(struct dm_writecache *wc, bool wait_for_ios)
{
if (WC_MODE_PMEM(wc))
- wmb();
+ pmem_wmb();
else
ssd_commit_flushed(wc, wait_for_ios);
}
diff --git a/drivers/md/md-autodetect.c b/drivers/md/md-autodetect.c
new file mode 100644
index 0000000..6bbec89
--- /dev/null
+++ b/drivers/md/md-autodetect.c
@@ -0,0 +1,291 @@
+// SPDX-License-Identifier: GPL-2.0
+#include <linux/kernel.h>
+#include <linux/blkdev.h>
+#include <linux/init.h>
+#include <linux/mount.h>
+#include <linux/major.h>
+#include <linux/delay.h>
+#include <linux/init_syscalls.h>
+#include <linux/raid/detect.h>
+#include <linux/raid/md_u.h>
+#include <linux/raid/md_p.h>
+#include "md.h"
+
+/*
+ * When md (and any require personalities) are compiled into the kernel
+ * (not a module), arrays can be assembles are boot time using with AUTODETECT
+ * where specially marked partitions are registered with md_autodetect_dev(),
+ * and with MD_BOOT where devices to be collected are given on the boot line
+ * with md=.....
+ * The code for that is here.
+ */
+
+#ifdef CONFIG_MD_AUTODETECT
+static int __initdata raid_noautodetect;
+#else
+static int __initdata raid_noautodetect=1;
+#endif
+static int __initdata raid_autopart;
+
+static struct md_setup_args {
+ int minor;
+ int partitioned;
+ int level;
+ int chunk;
+ char *device_names;
+} md_setup_args[256] __initdata;
+
+static int md_setup_ents __initdata;
+
+/*
+ * Parse the command-line parameters given our kernel, but do not
+ * actually try to invoke the MD device now; that is handled by
+ * md_setup_drive after the low-level disk drivers have initialised.
+ *
+ * 27/11/1999: Fixed to work correctly with the 2.3 kernel (which
+ * assigns the task of parsing integer arguments to the
+ * invoked program now). Added ability to initialise all
+ * the MD devices (by specifying multiple "md=" lines)
+ * instead of just one. -- KTK
+ * 18May2000: Added support for persistent-superblock arrays:
+ * md=n,0,factor,fault,device-list uses RAID0 for device n
+ * md=n,-1,factor,fault,device-list uses LINEAR for device n
+ * md=n,device-list reads a RAID superblock from the devices
+ * elements in device-list are read by name_to_kdev_t so can be
+ * a hex number or something like /dev/hda1 /dev/sdb
+ * 2001-06-03: Dave Cinege <dcinege@psychosis.com>
+ * Shifted name_to_kdev_t() and related operations to md_set_drive()
+ * for later execution. Rewrote section to make devfs compatible.
+ */
+static int __init md_setup(char *str)
+{
+ int minor, level, factor, fault, partitioned = 0;
+ char *pername = "";
+ char *str1;
+ int ent;
+
+ if (*str == 'd') {
+ partitioned = 1;
+ str++;
+ }
+ if (get_option(&str, &minor) != 2) { /* MD Number */
+ printk(KERN_WARNING "md: Too few arguments supplied to md=.\n");
+ return 0;
+ }
+ str1 = str;
+ for (ent=0 ; ent< md_setup_ents ; ent++)
+ if (md_setup_args[ent].minor == minor &&
+ md_setup_args[ent].partitioned == partitioned) {
+ printk(KERN_WARNING "md: md=%s%d, Specified more than once. "
+ "Replacing previous definition.\n", partitioned?"d":"", minor);
+ break;
+ }
+ if (ent >= ARRAY_SIZE(md_setup_args)) {
+ printk(KERN_WARNING "md: md=%s%d - too many md initialisations\n", partitioned?"d":"", minor);
+ return 0;
+ }
+ if (ent >= md_setup_ents)
+ md_setup_ents++;
+ switch (get_option(&str, &level)) { /* RAID level */
+ case 2: /* could be 0 or -1.. */
+ if (level == 0 || level == LEVEL_LINEAR) {
+ if (get_option(&str, &factor) != 2 || /* Chunk Size */
+ get_option(&str, &fault) != 2) {
+ printk(KERN_WARNING "md: Too few arguments supplied to md=.\n");
+ return 0;
+ }
+ md_setup_args[ent].level = level;
+ md_setup_args[ent].chunk = 1 << (factor+12);
+ if (level == LEVEL_LINEAR)
+ pername = "linear";
+ else
+ pername = "raid0";
+ break;
+ }
+ /* FALL THROUGH */
+ case 1: /* the first device is numeric */
+ str = str1;
+ /* FALL THROUGH */
+ case 0:
+ md_setup_args[ent].level = LEVEL_NONE;
+ pername="super-block";
+ }
+
+ printk(KERN_INFO "md: Will configure md%d (%s) from %s, below.\n",
+ minor, pername, str);
+ md_setup_args[ent].device_names = str;
+ md_setup_args[ent].partitioned = partitioned;
+ md_setup_args[ent].minor = minor;
+
+ return 1;
+}
+
+static void __init md_setup_drive(struct md_setup_args *args)
+{
+ char *devname = args->device_names;
+ dev_t devices[MD_SB_DISKS + 1], mdev;
+ struct mdu_array_info_s ainfo = { };
+ struct block_device *bdev;
+ struct mddev *mddev;
+ int err = 0, i;
+ char name[16];
+
+ if (args->partitioned) {
+ mdev = MKDEV(mdp_major, args->minor << MdpMinorShift);
+ sprintf(name, "md_d%d", args->minor);
+ } else {
+ mdev = MKDEV(MD_MAJOR, args->minor);
+ sprintf(name, "md%d", args->minor);
+ }
+
+ for (i = 0; i < MD_SB_DISKS && devname != NULL; i++) {
+ struct kstat stat;
+ char *p;
+ char comp_name[64];
+ dev_t dev;
+
+ p = strchr(devname, ',');
+ if (p)
+ *p++ = 0;
+
+ dev = name_to_dev_t(devname);
+ if (strncmp(devname, "/dev/", 5) == 0)
+ devname += 5;
+ snprintf(comp_name, 63, "/dev/%s", devname);
+ if (init_stat(comp_name, &stat, 0) == 0 && S_ISBLK(stat.mode))
+ dev = new_decode_dev(stat.rdev);
+ if (!dev) {
+ pr_warn("md: Unknown device name: %s\n", devname);
+ break;
+ }
+
+ devices[i] = dev;
+ devname = p;
+ }
+ devices[i] = 0;
+
+ if (!i)
+ return;
+
+ pr_info("md: Loading %s: %s\n", name, args->device_names);
+
+ bdev = blkdev_get_by_dev(mdev, FMODE_READ, NULL);
+ if (IS_ERR(bdev)) {
+ pr_err("md: open failed - cannot start array %s\n", name);
+ return;
+ }
+
+ err = -EIO;
+ if (WARN(bdev->bd_disk->fops != &md_fops,
+ "Opening block device %x resulted in non-md device\n",
+ mdev))
+ goto out_blkdev_put;
+
+ mddev = bdev->bd_disk->private_data;
+
+ err = mddev_lock(mddev);
+ if (err) {
+ pr_err("md: failed to lock array %s\n", name);
+ goto out_blkdev_put;
+ }
+
+ if (!list_empty(&mddev->disks) || mddev->raid_disks) {
+ pr_warn("md: Ignoring %s, already autodetected. (Use raid=noautodetect)\n",
+ name);
+ goto out_unlock;
+ }
+
+ if (args->level != LEVEL_NONE) {
+ /* non-persistent */
+ ainfo.level = args->level;
+ ainfo.md_minor = args->minor;
+ ainfo.not_persistent = 1;
+ ainfo.state = (1 << MD_SB_CLEAN);
+ ainfo.chunk_size = args->chunk;
+ while (devices[ainfo.raid_disks])
+ ainfo.raid_disks++;
+ }
+
+ err = md_set_array_info(mddev, &ainfo);
+
+ for (i = 0; i <= MD_SB_DISKS && devices[i]; i++) {
+ struct mdu_disk_info_s dinfo = {
+ .major = MAJOR(devices[i]),
+ .minor = MINOR(devices[i]),
+ };
+
+ if (args->level != LEVEL_NONE) {
+ dinfo.number = i;
+ dinfo.raid_disk = i;
+ dinfo.state =
+ (1 << MD_DISK_ACTIVE) | (1 << MD_DISK_SYNC);
+ }
+
+ md_add_new_disk(mddev, &dinfo);
+ }
+
+ if (!err)
+ err = do_md_run(mddev);
+ if (err)
+ pr_warn("md: starting %s failed\n", name);
+out_unlock:
+ mddev_unlock(mddev);
+out_blkdev_put:
+ blkdev_put(bdev, FMODE_READ);
+}
+
+static int __init raid_setup(char *str)
+{
+ int len, pos;
+
+ len = strlen(str) + 1;
+ pos = 0;
+
+ while (pos < len) {
+ char *comma = strchr(str+pos, ',');
+ int wlen;
+ if (comma)
+ wlen = (comma-str)-pos;
+ else wlen = (len-1)-pos;
+
+ if (!strncmp(str, "noautodetect", wlen))
+ raid_noautodetect = 1;
+ if (!strncmp(str, "autodetect", wlen))
+ raid_noautodetect = 0;
+ if (strncmp(str, "partitionable", wlen)==0)
+ raid_autopart = 1;
+ if (strncmp(str, "part", wlen)==0)
+ raid_autopart = 1;
+ pos += wlen+1;
+ }
+ return 1;
+}
+
+__setup("raid=", raid_setup);
+__setup("md=", md_setup);
+
+static void __init autodetect_raid(void)
+{
+ /*
+ * Since we don't want to detect and use half a raid array, we need to
+ * wait for the known devices to complete their probing
+ */
+ printk(KERN_INFO "md: Waiting for all devices to be available before autodetect\n");
+ printk(KERN_INFO "md: If you don't use raid, use raid=noautodetect\n");
+
+ wait_for_device_probe();
+ md_autostart_arrays(raid_autopart);
+}
+
+void __init md_run_setup(void)
+{
+ int ent;
+
+ if (raid_noautodetect)
+ printk(KERN_INFO "md: Skipping autodetection of RAID arrays. (raid=autodetect will force)\n");
+ else
+ autodetect_raid();
+
+ for (ent = 0; ent < md_setup_ents; ent++)
+ md_setup_drive(&md_setup_args[ent]);
+}
diff --git a/drivers/md/md.c b/drivers/md/md.c
index 153bc766..15bbdc1 100644
--- a/drivers/md/md.c
+++ b/drivers/md/md.c
@@ -68,10 +68,6 @@
#include "md-bitmap.h"
#include "md-cluster.h"
-#ifndef MODULE
-static void autostart_arrays(int part);
-#endif
-
/* pers_list is a list of registered personalities protected
* by pers_lock.
* pers_lock does extra service to protect accesses to
@@ -332,8 +328,6 @@
{ }
};
-static const struct block_device_operations md_fops;
-
static int start_readonly;
/*
@@ -4426,7 +4420,6 @@
static int do_md_stop(struct mddev *mddev, int ro, struct block_device *bdev);
static int md_set_readonly(struct mddev *mddev, struct block_device *bdev);
-static int do_md_run(struct mddev *mddev);
static int restart_array(struct mddev *mddev);
static ssize_t
@@ -6089,7 +6082,7 @@
}
EXPORT_SYMBOL_GPL(md_run);
-static int do_md_run(struct mddev *mddev)
+int do_md_run(struct mddev *mddev)
{
int err;
@@ -6724,7 +6717,7 @@
return 0;
}
-static int add_new_disk(struct mddev *mddev, mdu_disk_info_t *info)
+int md_add_new_disk(struct mddev *mddev, struct mdu_disk_info_s *info)
{
char b[BDEVNAME_SIZE], b2[BDEVNAME_SIZE];
struct md_rdev *rdev;
@@ -6770,7 +6763,7 @@
}
/*
- * add_new_disk can be used once the array is assembled
+ * md_add_new_disk can be used once the array is assembled
* to add "hot spares". They must already have a superblock
* written
*/
@@ -6883,7 +6876,7 @@
return err;
}
- /* otherwise, add_new_disk is only allowed
+ /* otherwise, md_add_new_disk is only allowed
* for major_version==0 superblocks
*/
if (mddev->major_version != 0) {
@@ -7128,7 +7121,7 @@
}
/*
- * set_array_info is used two different ways
+ * md_set_array_info is used two different ways
* The original usage is when creating a new array.
* In this usage, raid_disks is > 0 and it together with
* level, size, not_persistent,layout,chunksize determine the
@@ -7140,9 +7133,8 @@
* The minor and patch _version numbers are also kept incase the
* super_block handler wishes to interpret them.
*/
-static int set_array_info(struct mddev *mddev, mdu_array_info_t *info)
+int md_set_array_info(struct mddev *mddev, struct mdu_array_info_s *info)
{
-
if (info->raid_disks == 0) {
/* just setting version number for superblock loading */
if (info->major_version < 0 ||
@@ -7492,7 +7484,6 @@
case GET_DISK_INFO:
case HOT_ADD_DISK:
case HOT_REMOVE_DISK:
- case RAID_AUTORUN:
case RAID_VERSION:
case RESTART_ARRAY_RW:
case RUN_ARRAY:
@@ -7538,13 +7529,6 @@
case RAID_VERSION:
err = get_version(argp);
goto out;
-
-#ifndef MODULE
- case RAID_AUTORUN:
- err = 0;
- autostart_arrays(arg);
- goto out;
-#endif
default:;
}
@@ -7643,7 +7627,7 @@
err = -EBUSY;
goto unlock;
}
- err = set_array_info(mddev, &info);
+ err = md_set_array_info(mddev, &info);
if (err) {
pr_warn("md: couldn't set array info. %d\n", err);
goto unlock;
@@ -7697,7 +7681,7 @@
/* Need to clear read-only for this */
break;
else
- err = add_new_disk(mddev, &info);
+ err = md_add_new_disk(mddev, &info);
goto unlock;
}
break;
@@ -7765,7 +7749,7 @@
if (copy_from_user(&info, argp, sizeof(info)))
err = -EFAULT;
else
- err = add_new_disk(mddev, &info);
+ err = md_add_new_disk(mddev, &info);
goto unlock;
}
@@ -7888,7 +7872,7 @@
return ret;
}
-static const struct block_device_operations md_fops =
+const struct block_device_operations md_fops =
{
.owner = THIS_MODULE,
.submit_bio = md_submit_bio,
@@ -9791,7 +9775,7 @@
}
}
-static void autostart_arrays(int part)
+void md_autostart_arrays(int part)
{
struct md_rdev *rdev;
struct detected_devices_node *node_detected_dev;
diff --git a/drivers/md/md.h b/drivers/md/md.h
index f79b5b4..d9c4e6b 100644
--- a/drivers/md/md.h
+++ b/drivers/md/md.h
@@ -803,4 +803,16 @@
!bio->bi_disk->queue->limits.max_write_zeroes_sectors)
mddev->queue->limits.max_write_zeroes_sectors = 0;
}
+
+struct mdu_array_info_s;
+struct mdu_disk_info_s;
+
+extern int mdp_major;
+void md_autostart_arrays(int part);
+int md_set_array_info(struct mddev *mddev, struct mdu_array_info_s *info);
+int md_add_new_disk(struct mddev *mddev, struct mdu_disk_info_s *info);
+int do_md_run(struct mddev *mddev);
+
+extern const struct block_device_operations md_fops;
+
#endif /* _MD_MD_H */
diff --git a/drivers/misc/ocxl/Kconfig b/drivers/misc/ocxl/Kconfig
index 2d2266c..6551007 100644
--- a/drivers/misc/ocxl/Kconfig
+++ b/drivers/misc/ocxl/Kconfig
@@ -23,7 +23,7 @@
The ocxl driver enables userspace programs to access these
accelerators through devices in /dev/ocxl/.
- For more information, see http://opencapi.org.
+ For more information, see https://opencapi.org.
This is not to be confused with the support for IBM CAPI
accelerators (CONFIG_CXL), which are PCI-based instead of a
diff --git a/drivers/misc/ocxl/config.c b/drivers/misc/ocxl/config.c
index e3b99a3..4d490b9 100644
--- a/drivers/misc/ocxl/config.c
+++ b/drivers/misc/ocxl/config.c
@@ -71,6 +71,20 @@
return 0;
}
+/**
+ * get_function_0() - Find a related PCI device (function 0)
+ * @device: PCI device to match
+ *
+ * Returns a pointer to the related device, or null if not found
+ */
+static struct pci_dev *get_function_0(struct pci_dev *dev)
+{
+ unsigned int devfn = PCI_DEVFN(PCI_SLOT(dev->devfn), 0);
+
+ return pci_get_domain_bus_and_slot(pci_domain_nr(dev->bus),
+ dev->bus->number, devfn);
+}
+
static void read_pasid(struct pci_dev *dev, struct ocxl_fn_config *fn)
{
u16 val;
@@ -159,14 +173,15 @@
static int read_dvsec_vendor(struct pci_dev *dev)
{
int pos;
- u32 cfg, tlx, dlx;
+ u32 cfg, tlx, dlx, reset_reload;
/*
- * vendor specific DVSEC is optional
+ * vendor specific DVSEC, for IBM images only. Some older
+ * images may not have it
*
- * It's currently only used on function 0 to specify the
- * version of some logic blocks. Some older images may not
- * even have it so we ignore any errors
+ * It's only used on function 0 to specify the version of some
+ * logic blocks and to give access to special registers to
+ * enable host-based flashing.
*/
if (PCI_FUNC(dev->devfn) != 0)
return 0;
@@ -178,11 +193,67 @@
pci_read_config_dword(dev, pos + OCXL_DVSEC_VENDOR_CFG_VERS, &cfg);
pci_read_config_dword(dev, pos + OCXL_DVSEC_VENDOR_TLX_VERS, &tlx);
pci_read_config_dword(dev, pos + OCXL_DVSEC_VENDOR_DLX_VERS, &dlx);
+ pci_read_config_dword(dev, pos + OCXL_DVSEC_VENDOR_RESET_RELOAD,
+ &reset_reload);
dev_dbg(&dev->dev, "Vendor specific DVSEC:\n");
dev_dbg(&dev->dev, " CFG version = 0x%x\n", cfg);
dev_dbg(&dev->dev, " TLX version = 0x%x\n", tlx);
dev_dbg(&dev->dev, " DLX version = 0x%x\n", dlx);
+ dev_dbg(&dev->dev, " ResetReload = 0x%x\n", reset_reload);
+ return 0;
+}
+
+static int get_dvsec_vendor0(struct pci_dev *dev, struct pci_dev **dev0,
+ int *out_pos)
+{
+ int pos;
+
+ if (PCI_FUNC(dev->devfn) != 0) {
+ dev = get_function_0(dev);
+ if (!dev)
+ return -1;
+ }
+ pos = find_dvsec(dev, OCXL_DVSEC_VENDOR_ID);
+ if (!pos)
+ return -1;
+ *dev0 = dev;
+ *out_pos = pos;
+ return 0;
+}
+
+int ocxl_config_get_reset_reload(struct pci_dev *dev, int *val)
+{
+ struct pci_dev *dev0;
+ u32 reset_reload;
+ int pos;
+
+ if (get_dvsec_vendor0(dev, &dev0, &pos))
+ return -1;
+
+ pci_read_config_dword(dev0, pos + OCXL_DVSEC_VENDOR_RESET_RELOAD,
+ &reset_reload);
+ *val = !!(reset_reload & BIT(0));
+ return 0;
+}
+
+int ocxl_config_set_reset_reload(struct pci_dev *dev, int val)
+{
+ struct pci_dev *dev0;
+ u32 reset_reload;
+ int pos;
+
+ if (get_dvsec_vendor0(dev, &dev0, &pos))
+ return -1;
+
+ pci_read_config_dword(dev0, pos + OCXL_DVSEC_VENDOR_RESET_RELOAD,
+ &reset_reload);
+ if (val)
+ reset_reload |= BIT(0);
+ else
+ reset_reload &= ~BIT(0);
+ pci_write_config_dword(dev0, pos + OCXL_DVSEC_VENDOR_RESET_RELOAD,
+ reset_reload);
return 0;
}
@@ -273,7 +344,7 @@
}
/**
- * read_template_version - Read the template version from the AFU
+ * read_template_version() - Read the template version from the AFU
* @dev: the device for the AFU
* @fn: the AFU offsets
* @len: outputs the template length
@@ -282,7 +353,7 @@
* Returns 0 on success, negative on failure
*/
static int read_template_version(struct pci_dev *dev, struct ocxl_fn_config *fn,
- u16 *len, u16 *version)
+ u16 *len, u16 *version)
{
u32 val32;
u8 major, minor;
@@ -476,7 +547,7 @@
}
/**
- * read_afu_lpc_memory_info - Populate AFU metadata regarding LPC memory
+ * read_afu_lpc_memory_info() - Populate AFU metadata regarding LPC memory
* @dev: the device for the AFU
* @fn: the AFU offsets
* @afu: the AFU struct to populate the LPC metadata into
@@ -484,8 +555,8 @@
* Returns 0 on success, negative on failure
*/
static int read_afu_lpc_memory_info(struct pci_dev *dev,
- struct ocxl_fn_config *fn,
- struct ocxl_afu_config *afu)
+ struct ocxl_fn_config *fn,
+ struct ocxl_afu_config *afu)
{
int rc;
u32 val32;
diff --git a/drivers/misc/ocxl/ocxl_internal.h b/drivers/misc/ocxl/ocxl_internal.h
index 345bf843..0bad0a1 100644
--- a/drivers/misc/ocxl/ocxl_internal.h
+++ b/drivers/misc/ocxl/ocxl_internal.h
@@ -113,6 +113,12 @@
int ocxl_config_get_pasid_info(struct pci_dev *dev, int *count);
/*
+ * Control whether the FPGA is reloaded on a link reset
+ */
+int ocxl_config_get_reset_reload(struct pci_dev *dev, int *val);
+int ocxl_config_set_reset_reload(struct pci_dev *dev, int val);
+
+/*
* Check if an AFU index is valid for the given function.
*
* AFU indexes can be sparse, so a driver should check all indexes up
@@ -122,11 +128,12 @@
struct ocxl_fn_config *fn, int afu_idx);
/**
- * Update values within a Process Element
+ * ocxl_link_update_pe() - Update values within a Process Element
+ * @link_handle: the link handle associated with the process element
+ * @pasid: the PASID for the AFU context
+ * @tid: the new thread id for the process element
*
- * link_handle: the link handle associated with the process element
- * pasid: the PASID for the AFU context
- * tid: the new thread id for the process element
+ * Returns 0 on success
*/
int ocxl_link_update_pe(void *link_handle, int pasid, __u16 tid);
diff --git a/drivers/misc/ocxl/sysfs.c b/drivers/misc/ocxl/sysfs.c
index 58f1ba2..25c78df 100644
--- a/drivers/misc/ocxl/sysfs.c
+++ b/drivers/misc/ocxl/sysfs.c
@@ -51,11 +51,46 @@
afu->pasid_count, afu->pasid_max);
}
+static ssize_t reload_on_reset_show(struct device *device,
+ struct device_attribute *attr,
+ char *buf)
+{
+ struct ocxl_afu *afu = to_afu(device);
+ struct ocxl_fn *fn = afu->fn;
+ struct pci_dev *pci_dev = to_pci_dev(fn->dev.parent);
+ int val;
+
+ if (ocxl_config_get_reset_reload(pci_dev, &val))
+ return scnprintf(buf, PAGE_SIZE, "unavailable\n");
+
+ return scnprintf(buf, PAGE_SIZE, "%d\n", val);
+}
+
+static ssize_t reload_on_reset_store(struct device *device,
+ struct device_attribute *attr,
+ const char *buf, size_t count)
+{
+ struct ocxl_afu *afu = to_afu(device);
+ struct ocxl_fn *fn = afu->fn;
+ struct pci_dev *pci_dev = to_pci_dev(fn->dev.parent);
+ int rc, val;
+
+ rc = kstrtoint(buf, 0, &val);
+ if (rc || (val != 0 && val != 1))
+ return -EINVAL;
+
+ if (ocxl_config_set_reset_reload(pci_dev, val))
+ return -ENODEV;
+
+ return count;
+}
+
static struct device_attribute afu_attrs[] = {
__ATTR_RO(global_mmio_size),
__ATTR_RO(pp_mmio_size),
__ATTR_RO(afu_version),
__ATTR_RO(contexts),
+ __ATTR_RW(reload_on_reset),
};
static ssize_t global_mmio_read(struct file *filp, struct kobject *kobj,
diff --git a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_thermal.c b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_thermal.c
index 3de8a5e..e3510e9 100644
--- a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_thermal.c
+++ b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_thermal.c
@@ -92,6 +92,14 @@
ch_thermal->tzdev = NULL;
return ret;
}
+
+ ret = thermal_zone_device_enable(ch_thermal->tzdev);
+ if (ret) {
+ dev_err(adap->pdev_dev, "Failed to enable thermal zone\n");
+ thermal_zone_device_unregister(adap->ch_thermal.tzdev);
+ return ret;
+ }
+
return 0;
}
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/alloc.c b/drivers/net/ethernet/mellanox/mlx5/core/alloc.c
index 42198e6..8db4b5f 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/alloc.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/alloc.c
@@ -299,11 +299,18 @@
}
EXPORT_SYMBOL_GPL(mlx5_fill_page_array);
-void mlx5_fill_page_frag_array(struct mlx5_frag_buf *buf, __be64 *pas)
+void mlx5_fill_page_frag_array_perm(struct mlx5_frag_buf *buf, __be64 *pas, u8 perm)
{
int i;
+ WARN_ON(perm & 0xfc);
for (i = 0; i < buf->npages; i++)
- pas[i] = cpu_to_be64(buf->frags[i].map);
+ pas[i] = cpu_to_be64(buf->frags[i].map | perm);
+}
+EXPORT_SYMBOL_GPL(mlx5_fill_page_frag_array_perm);
+
+void mlx5_fill_page_frag_array(struct mlx5_frag_buf *buf, __be64 *pas)
+{
+ mlx5_fill_page_frag_array_perm(buf, pas, 0);
}
EXPORT_SYMBOL_GPL(mlx5_fill_page_frag_array);
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/fs_core.c b/drivers/net/ethernet/mellanox/mlx5/core/fs_core.c
index 7e70a817..9ccec5f 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/fs_core.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/fs_core.c
@@ -1577,6 +1577,7 @@
static bool counter_is_valid(u32 action)
{
return (action & (MLX5_FLOW_CONTEXT_ACTION_DROP |
+ MLX5_FLOW_CONTEXT_ACTION_ALLOW |
MLX5_FLOW_CONTEXT_ACTION_FWD_DEST));
}
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/ipoib/ipoib.c b/drivers/net/ethernet/mellanox/mlx5/core/ipoib/ipoib.c
index 5763965..97b5fcb 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/ipoib/ipoib.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/ipoib/ipoib.c
@@ -226,13 +226,20 @@
int mlx5i_create_underlay_qp(struct mlx5e_priv *priv)
{
+ unsigned char *dev_addr = priv->netdev->dev_addr;
u32 out[MLX5_ST_SZ_DW(create_qp_out)] = {};
u32 in[MLX5_ST_SZ_DW(create_qp_in)] = {};
struct mlx5i_priv *ipriv = priv->ppriv;
void *addr_path;
+ int qpn = 0;
int ret = 0;
void *qpc;
+ if (MLX5_CAP_GEN(priv->mdev, mkey_by_name)) {
+ qpn = (dev_addr[1] << 16) + (dev_addr[2] << 8) + dev_addr[3];
+ MLX5_SET(create_qp_in, in, input_qpn, qpn);
+ }
+
qpc = MLX5_ADDR_OF(create_qp_in, in, qpc);
MLX5_SET(qpc, qpc, st, MLX5_QP_ST_UD);
MLX5_SET(qpc, qpc, pm_state, MLX5_QP_PM_MIGRATED);
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/main.c b/drivers/net/ethernet/mellanox/mlx5/core/main.c
index e32d46c33..ce43e3f 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/main.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/main.c
@@ -557,6 +557,9 @@
if (MLX5_CAP_GEN_MAX(dev, release_all_pages))
MLX5_SET(cmd_hca_cap, set_hca_cap, release_all_pages, 1);
+ if (MLX5_CAP_GEN_MAX(dev, mkey_by_name))
+ MLX5_SET(cmd_hca_cap, set_hca_cap, mkey_by_name, 1);
+
return set_caps(dev, set_ctx, MLX5_SET_HCA_CAP_OP_MOD_GENERAL_DEVICE);
}
diff --git a/drivers/net/ethernet/mellanox/mlxsw/core_thermal.c b/drivers/net/ethernet/mellanox/mlxsw/core_thermal.c
index 05f8d5a..8fa286c 100644
--- a/drivers/net/ethernet/mellanox/mlxsw/core_thermal.c
+++ b/drivers/net/ethernet/mellanox/mlxsw/core_thermal.c
@@ -98,7 +98,6 @@
struct mlxsw_thermal *parent;
struct thermal_zone_device *tzdev;
struct mlxsw_thermal_trip trips[MLXSW_THERMAL_NUM_TRIPS];
- enum thermal_device_mode mode;
int module; /* Module or gearbox number */
};
@@ -110,7 +109,6 @@
struct thermal_cooling_device *cdevs[MLXSW_MFCR_PWMS_MAX];
u8 cooling_levels[MLXSW_THERMAL_MAX_STATE + 1];
struct mlxsw_thermal_trip trips[MLXSW_THERMAL_NUM_TRIPS];
- enum thermal_device_mode mode;
struct mlxsw_thermal_module *tz_module_arr;
u8 tz_module_num;
struct mlxsw_thermal_module *tz_gearbox_arr;
@@ -277,36 +275,6 @@
return 0;
}
-static int mlxsw_thermal_get_mode(struct thermal_zone_device *tzdev,
- enum thermal_device_mode *mode)
-{
- struct mlxsw_thermal *thermal = tzdev->devdata;
-
- *mode = thermal->mode;
-
- return 0;
-}
-
-static int mlxsw_thermal_set_mode(struct thermal_zone_device *tzdev,
- enum thermal_device_mode mode)
-{
- struct mlxsw_thermal *thermal = tzdev->devdata;
-
- mutex_lock(&tzdev->lock);
-
- if (mode == THERMAL_DEVICE_ENABLED)
- tzdev->polling_delay = thermal->polling_delay;
- else
- tzdev->polling_delay = 0;
-
- mutex_unlock(&tzdev->lock);
-
- thermal->mode = mode;
- thermal_zone_device_update(tzdev, THERMAL_EVENT_UNSPECIFIED);
-
- return 0;
-}
-
static int mlxsw_thermal_get_temp(struct thermal_zone_device *tzdev,
int *p_temp)
{
@@ -406,8 +374,6 @@
static struct thermal_zone_device_ops mlxsw_thermal_ops = {
.bind = mlxsw_thermal_bind,
.unbind = mlxsw_thermal_unbind,
- .get_mode = mlxsw_thermal_get_mode,
- .set_mode = mlxsw_thermal_set_mode,
.get_temp = mlxsw_thermal_get_temp,
.get_trip_type = mlxsw_thermal_get_trip_type,
.get_trip_temp = mlxsw_thermal_get_trip_temp,
@@ -465,37 +431,6 @@
return err;
}
-static int mlxsw_thermal_module_mode_get(struct thermal_zone_device *tzdev,
- enum thermal_device_mode *mode)
-{
- struct mlxsw_thermal_module *tz = tzdev->devdata;
-
- *mode = tz->mode;
-
- return 0;
-}
-
-static int mlxsw_thermal_module_mode_set(struct thermal_zone_device *tzdev,
- enum thermal_device_mode mode)
-{
- struct mlxsw_thermal_module *tz = tzdev->devdata;
- struct mlxsw_thermal *thermal = tz->parent;
-
- mutex_lock(&tzdev->lock);
-
- if (mode == THERMAL_DEVICE_ENABLED)
- tzdev->polling_delay = thermal->polling_delay;
- else
- tzdev->polling_delay = 0;
-
- mutex_unlock(&tzdev->lock);
-
- tz->mode = mode;
- thermal_zone_device_update(tzdev, THERMAL_EVENT_UNSPECIFIED);
-
- return 0;
-}
-
static int mlxsw_thermal_module_temp_get(struct thermal_zone_device *tzdev,
int *p_temp)
{
@@ -611,8 +546,6 @@
static struct thermal_zone_device_ops mlxsw_thermal_module_ops = {
.bind = mlxsw_thermal_module_bind,
.unbind = mlxsw_thermal_module_unbind,
- .get_mode = mlxsw_thermal_module_mode_get,
- .set_mode = mlxsw_thermal_module_mode_set,
.get_temp = mlxsw_thermal_module_temp_get,
.get_trip_type = mlxsw_thermal_module_trip_type_get,
.get_trip_temp = mlxsw_thermal_module_trip_temp_get,
@@ -650,8 +583,6 @@
static struct thermal_zone_device_ops mlxsw_thermal_gearbox_ops = {
.bind = mlxsw_thermal_module_bind,
.unbind = mlxsw_thermal_module_unbind,
- .get_mode = mlxsw_thermal_module_mode_get,
- .set_mode = mlxsw_thermal_module_mode_set,
.get_temp = mlxsw_thermal_gearbox_temp_get,
.get_trip_type = mlxsw_thermal_module_trip_type_get,
.get_trip_temp = mlxsw_thermal_module_trip_temp_get,
@@ -780,8 +711,11 @@
return err;
}
- module_tz->mode = THERMAL_DEVICE_ENABLED;
- return 0;
+ err = thermal_zone_device_enable(module_tz->tzdev);
+ if (err)
+ thermal_zone_device_unregister(module_tz->tzdev);
+
+ return err;
}
static void mlxsw_thermal_module_tz_fini(struct thermal_zone_device *tzdev)
@@ -884,6 +818,7 @@
mlxsw_thermal_gearbox_tz_init(struct mlxsw_thermal_module *gearbox_tz)
{
char tz_name[MLXSW_THERMAL_ZONE_MAX_NAME];
+ int ret;
snprintf(tz_name, sizeof(tz_name), "mlxsw-gearbox%d",
gearbox_tz->module + 1);
@@ -896,8 +831,11 @@
if (IS_ERR(gearbox_tz->tzdev))
return PTR_ERR(gearbox_tz->tzdev);
- gearbox_tz->mode = THERMAL_DEVICE_ENABLED;
- return 0;
+ ret = thermal_zone_device_enable(gearbox_tz->tzdev);
+ if (ret)
+ thermal_zone_device_unregister(gearbox_tz->tzdev);
+
+ return ret;
}
static void
@@ -1065,10 +1003,15 @@
if (err)
goto err_unreg_modules_tzdev;
- thermal->mode = THERMAL_DEVICE_ENABLED;
+ err = thermal_zone_device_enable(thermal->tzdev);
+ if (err)
+ goto err_unreg_gearboxes;
+
*p_thermal = thermal;
return 0;
+err_unreg_gearboxes:
+ mlxsw_thermal_gearboxes_fini(thermal);
err_unreg_modules_tzdev:
mlxsw_thermal_modules_fini(thermal);
err_unreg_tzdev:
diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/tt.c b/drivers/net/wireless/intel/iwlwifi/mvm/tt.c
index 418e59b..0c95663 100644
--- a/drivers/net/wireless/intel/iwlwifi/mvm/tt.c
+++ b/drivers/net/wireless/intel/iwlwifi/mvm/tt.c
@@ -733,7 +733,7 @@
static void iwl_mvm_thermal_zone_register(struct iwl_mvm *mvm)
{
- int i;
+ int i, ret;
char name[16];
static atomic_t counter = ATOMIC_INIT(0);
@@ -759,6 +759,13 @@
return;
}
+ ret = thermal_zone_device_enable(mvm->tz_device.tzone);
+ if (ret) {
+ IWL_DEBUG_TEMP(mvm, "Failed to enable thermal zone\n");
+ thermal_zone_device_unregister(mvm->tz_device.tzone);
+ return;
+ }
+
/* 0 is a valid temperature,
* so initialize the array with S16_MIN which invalid temperature
*/
diff --git a/drivers/nvdimm/of_pmem.c b/drivers/nvdimm/of_pmem.c
index 6826a27..10dbdcd 100644
--- a/drivers/nvdimm/of_pmem.c
+++ b/drivers/nvdimm/of_pmem.c
@@ -90,6 +90,7 @@
static const struct of_device_id of_pmem_region_match[] = {
{ .compatible = "pmem-region" },
+ { .compatible = "pmem-region-v2" },
{ },
};
diff --git a/drivers/nvdimm/region_devs.c b/drivers/nvdimm/region_devs.c
index 4502f9c..c3237c2 100644
--- a/drivers/nvdimm/region_devs.c
+++ b/drivers/nvdimm/region_devs.c
@@ -1206,13 +1206,13 @@
idx = this_cpu_add_return(flush_idx, hash_32(current->pid + idx, 8));
/*
- * The first wmb() is needed to 'sfence' all previous writes
- * such that they are architecturally visible for the platform
- * buffer flush. Note that we've already arranged for pmem
+ * The pmem_wmb() is needed to 'sfence' all
+ * previous writes such that they are architecturally visible for
+ * the platform buffer flush. Note that we've already arranged for pmem
* writes to avoid the cache via memcpy_flushcache(). The final
* wmb() ensures ordering for the NVDIMM flush write.
*/
- wmb();
+ pmem_wmb();
for (i = 0; i < nd_region->ndr_mappings; i++)
if (ndrd_get_flush_wpq(ndrd, i, 0))
writeq(1, ndrd_get_flush_wpq(ndrd, i, idx));
diff --git a/drivers/platform/x86/acerhdf.c b/drivers/platform/x86/acerhdf.c
index a7a0b2e..44b6bfb 100644
--- a/drivers/platform/x86/acerhdf.c
+++ b/drivers/platform/x86/acerhdf.c
@@ -397,39 +397,24 @@
{
acerhdf_change_fanstate(ACERHDF_FAN_AUTO);
kernelmode = 0;
- if (thz_dev)
- thz_dev->polling_delay = 0;
+
pr_notice("kernel mode fan control OFF\n");
}
static inline void acerhdf_enable_kernelmode(void)
{
kernelmode = 1;
- thz_dev->polling_delay = interval*1000;
- thermal_zone_device_update(thz_dev, THERMAL_EVENT_UNSPECIFIED);
pr_notice("kernel mode fan control ON\n");
}
-static int acerhdf_get_mode(struct thermal_zone_device *thermal,
- enum thermal_device_mode *mode)
-{
- if (verbose)
- pr_notice("kernel mode fan control %d\n", kernelmode);
-
- *mode = (kernelmode) ? THERMAL_DEVICE_ENABLED
- : THERMAL_DEVICE_DISABLED;
-
- return 0;
-}
-
/*
* set operation mode;
* enabled: the thermal layer of the kernel takes care about
* the temperature and the fan.
* disabled: the BIOS takes control of the fan.
*/
-static int acerhdf_set_mode(struct thermal_zone_device *thermal,
- enum thermal_device_mode mode)
+static int acerhdf_change_mode(struct thermal_zone_device *thermal,
+ enum thermal_device_mode mode)
{
if (mode == THERMAL_DEVICE_DISABLED && kernelmode)
acerhdf_revert_to_bios_mode();
@@ -488,8 +473,7 @@
.bind = acerhdf_bind,
.unbind = acerhdf_unbind,
.get_temp = acerhdf_get_ec_temp,
- .get_mode = acerhdf_get_mode,
- .set_mode = acerhdf_set_mode,
+ .change_mode = acerhdf_change_mode,
.get_trip_type = acerhdf_get_trip_type,
.get_trip_hyst = acerhdf_get_trip_hyst,
.get_trip_temp = acerhdf_get_trip_temp,
@@ -733,6 +717,8 @@
static int __init acerhdf_register_thermal(void)
{
+ int ret;
+
cl_dev = thermal_cooling_device_register("acerhdf-fan", NULL,
&acerhdf_cooling_ops);
@@ -746,6 +732,13 @@
if (IS_ERR(thz_dev))
return -EINVAL;
+ if (kernelmode)
+ ret = thermal_zone_device_enable(thz_dev);
+ else
+ ret = thermal_zone_device_disable(thz_dev);
+ if (ret)
+ return ret;
+
if (strcmp(thz_dev->governor->name,
acerhdf_zone_params.governor_name)) {
pr_err("Didn't get thermal governor %s, perhaps not compiled into thermal subsystem.\n",
diff --git a/drivers/platform/x86/intel_mid_thermal.c b/drivers/platform/x86/intel_mid_thermal.c
index f402e2e..f12f4e7 100644
--- a/drivers/platform/x86/intel_mid_thermal.c
+++ b/drivers/platform/x86/intel_mid_thermal.c
@@ -493,6 +493,12 @@
ret = PTR_ERR(pinfo->tzd[i]);
goto err;
}
+ ret = thermal_zone_device_enable(pinfo->tzd[i]);
+ if (ret) {
+ kfree(td_info);
+ thermal_zone_device_unregister(pinfo->tzd[i]);
+ goto err;
+ }
}
pinfo->pdev = pdev;
diff --git a/drivers/power/supply/power_supply_core.c b/drivers/power/supply/power_supply_core.c
index 02b37fe..90e5673 100644
--- a/drivers/power/supply/power_supply_core.c
+++ b/drivers/power/supply/power_supply_core.c
@@ -939,7 +939,7 @@
static int psy_register_thermal(struct power_supply *psy)
{
- int i;
+ int i, ret;
if (psy->desc->no_thermal)
return 0;
@@ -949,7 +949,12 @@
if (psy->desc->properties[i] == POWER_SUPPLY_PROP_TEMP) {
psy->tzd = thermal_zone_device_register(psy->desc->name,
0, 0, psy, &psy_tzd_ops, NULL, 0, 0);
- return PTR_ERR_OR_ZERO(psy->tzd);
+ if (IS_ERR(psy->tzd))
+ return PTR_ERR(psy->tzd);
+ ret = thermal_zone_device_enable(psy->tzd);
+ if (ret)
+ thermal_zone_device_unregister(psy->tzd);
+ return ret;
}
}
return 0;
diff --git a/drivers/s390/scsi/zfcp_ccw.c b/drivers/s390/scsi/zfcp_ccw.c
index 49eda14..d9fd0a4 100644
--- a/drivers/s390/scsi/zfcp_ccw.c
+++ b/drivers/s390/scsi/zfcp_ccw.c
@@ -124,13 +124,12 @@
return;
write_lock_irq(&adapter->port_list_lock);
- list_for_each_entry_safe(port, p, &adapter->port_list, list) {
+ list_for_each_entry(port, &adapter->port_list, list) {
write_lock(&port->unit_list_lock);
- list_for_each_entry_safe(unit, u, &port->unit_list, list)
- list_move(&unit->list, &unit_remove_lh);
+ list_splice_init(&port->unit_list, &unit_remove_lh);
write_unlock(&port->unit_list_lock);
- list_move(&port->list, &port_remove_lh);
}
+ list_splice_init(&adapter->port_list, &port_remove_lh);
write_unlock_irq(&adapter->port_list_lock);
zfcp_ccw_adapter_put(adapter); /* put from zfcp_ccw_adapter_by_cdev */
diff --git a/drivers/s390/scsi/zfcp_erp.c b/drivers/s390/scsi/zfcp_erp.c
index 79f6e8f..59e662d 100644
--- a/drivers/s390/scsi/zfcp_erp.c
+++ b/drivers/s390/scsi/zfcp_erp.c
@@ -68,7 +68,7 @@
{
struct zfcp_adapter *adapter = act->adapter;
- list_move(&act->list, &act->adapter->erp_ready_head);
+ list_move(&act->list, &adapter->erp_ready_head);
zfcp_dbf_rec_run("erardy1", act);
wake_up(&adapter->erp_ready_wq);
zfcp_dbf_rec_run("erardy2", act);
diff --git a/drivers/s390/scsi/zfcp_fc.c b/drivers/s390/scsi/zfcp_fc.c
index b018b61b..d24cafe 100644
--- a/drivers/s390/scsi/zfcp_fc.c
+++ b/drivers/s390/scsi/zfcp_fc.c
@@ -48,7 +48,7 @@
{
if (!port_scan_backoff)
return 0;
- return get_random_int() % port_scan_backoff;
+ return prandom_u32_max(port_scan_backoff);
}
static void zfcp_fc_port_scan_time(struct zfcp_adapter *adapter)
diff --git a/drivers/s390/scsi/zfcp_qdio.c b/drivers/s390/scsi/zfcp_qdio.c
index 3a7f337..e78d65b 100644
--- a/drivers/s390/scsi/zfcp_qdio.c
+++ b/drivers/s390/scsi/zfcp_qdio.c
@@ -246,7 +246,7 @@
}
/**
- * zfcp_qdio_send - set PCI flag in first SBALE and send req to QDIO
+ * zfcp_qdio_send - send req to QDIO
* @qdio: pointer to struct zfcp_qdio
* @q_req: pointer to struct zfcp_qdio_req
* Returns: 0 on success, error otherwise
@@ -260,17 +260,20 @@
zfcp_qdio_account(qdio);
spin_unlock(&qdio->stat_lock);
+ atomic_sub(sbal_number, &qdio->req_q_free);
+
retval = do_QDIO(qdio->adapter->ccw_device, QDIO_FLAG_SYNC_OUTPUT, 0,
q_req->sbal_first, sbal_number);
if (unlikely(retval)) {
+ /* Failed to submit the IO, roll back our modifications. */
+ atomic_add(sbal_number, &qdio->req_q_free);
zfcp_qdio_zero_sbals(qdio->req_q, q_req->sbal_first,
sbal_number);
return retval;
}
/* account for transferred buffers */
- atomic_sub(sbal_number, &qdio->req_q_free);
qdio->req_q_idx += sbal_number;
qdio->req_q_idx %= QDIO_MAX_BUFFERS_PER_Q;
diff --git a/drivers/scsi/Kconfig b/drivers/scsi/Kconfig
index e9ff4cd..701b61e 100644
--- a/drivers/scsi/Kconfig
+++ b/drivers/scsi/Kconfig
@@ -1154,6 +1154,7 @@
config SCSI_LPFC
tristate "Emulex LightPulse Fibre Channel Support"
depends on PCI && SCSI
+ depends on CPU_FREQ
depends on SCSI_FC_ATTRS
depends on NVME_TARGET_FC || NVME_TARGET_FC=n
depends on NVME_FC || NVME_FC=n
@@ -1469,14 +1470,19 @@
module will be called sun_esp.
config ZFCP
- tristate "FCP host bus adapter driver for IBM eServer zSeries"
+ tristate "FCP host bus adapter driver for IBM mainframes"
depends on S390 && QDIO && SCSI
depends on SCSI_FC_ATTRS
help
- If you want to access SCSI devices attached to your IBM eServer
- zSeries by means of Fibre Channel interfaces say Y.
- For details please refer to the documentation provided by IBM at
- <http://oss.software.ibm.com/developerworks/opensource/linux390>
+ If you want to access SCSI devices attached to your IBM mainframe by
+ means of Fibre Channel Protocol host bus adapters say Y.
+
+ Supported HBAs include different models of the FICON Express and FCP
+ Express I/O cards.
+
+ For a more complete list, and for more details about setup and
+ operation refer to the IBM publication "Device Drivers, Features, and
+ Commands", SC33-8411.
This driver is also available as a module. This module will be
called zfcp. If you want to compile it as a module, say M here
diff --git a/drivers/scsi/aacraid/aachba.c b/drivers/scsi/aacraid/aachba.c
index 2b868f8..769af4c 100644
--- a/drivers/scsi/aacraid/aachba.c
+++ b/drivers/scsi/aacraid/aachba.c
@@ -350,7 +350,8 @@
/**
* aac_get_config_status - check the adapter configuration
- * @common: adapter to query
+ * @dev: aac driver data
+ * @commit_flag: force sending CT_COMMIT_CONFIG
*
* Query config status, and commit the configuration if needed.
*/
@@ -442,7 +443,7 @@
/**
* aac_get_containers - list containers
- * @common: adapter to probe
+ * @dev: aac driver data
*
* Make a list of all containers on this controller
*/
@@ -561,7 +562,7 @@
scsicmd->scsi_done(scsicmd);
}
-/**
+/*
* aac_get_container_name - get container name, none blocking.
*/
static int aac_get_container_name(struct scsi_cmnd * scsicmd)
@@ -786,8 +787,7 @@
/**
* aac_probe_container - query a logical volume
- * @dev: device to query
- * @cid: container identifier
+ * @scsicmd: the scsi command block
*
* Queries the controller about the given volume. The volume information
* is updated in the struct fsa_dev_info structure rather than returned.
@@ -1098,7 +1098,7 @@
scsicmd->scsi_done(scsicmd);
}
-/**
+/*
* aac_get_container_serial - get container serial, none blocking.
*/
static int aac_get_container_serial(struct scsi_cmnd * scsicmd)
@@ -1952,8 +1952,6 @@
/**
* aac_set_safw_attr_all_targets- update current hba map with data from FW
* @dev: aac_dev structure
- * @phys_luns: FW information from report phys luns
- * @rescan: Indicates scan type
*
* Update our hba map with the information gathered from the FW
*/
@@ -3391,15 +3389,12 @@
}
/**
- *
* aac_srb_callback
* @context: the context set in the fib - here it is scsi cmd
* @fibptr: pointer to the fib
*
* Handles the completion of a scsi command to a non dasd device
- *
*/
-
static void aac_srb_callback(void *context, struct fib * fibptr)
{
struct aac_srb_reply *srbreply;
@@ -3684,13 +3679,11 @@
}
/**
- *
* aac_hba_callback
* @context: the context set in the fib - here it is scsi cmd
* @fibptr: pointer to the fib
*
* Handles the completion of a native HBA scsi command
- *
*/
void aac_hba_callback(void *context, struct fib *fibptr)
{
@@ -3749,14 +3742,12 @@
}
/**
- *
* aac_send_srb_fib
* @scsicmd: the scsi command block
*
* This routine will form a FIB and fill in the aac_srb from the
* scsicmd passed in.
*/
-
static int aac_send_srb_fib(struct scsi_cmnd* scsicmd)
{
struct fib* cmd_fibcontext;
@@ -3792,7 +3783,6 @@
}
/**
- *
* aac_send_hba_fib
* @scsicmd: the scsi command block
*
diff --git a/drivers/scsi/aacraid/commctrl.c b/drivers/scsi/aacraid/commctrl.c
index 34e65de..59e82a8 100644
--- a/drivers/scsi/aacraid/commctrl.c
+++ b/drivers/scsi/aacraid/commctrl.c
@@ -32,6 +32,8 @@
#include "aacraid.h"
+# define AAC_DEBUG_PREAMBLE KERN_INFO
+# define AAC_DEBUG_POSTAMBLE
/**
* ioctl_send_fib - send a FIB from userspace
* @dev: adapter is being processed
@@ -40,9 +42,6 @@
* This routine sends a fib to the adapter on behalf of a user level
* program.
*/
-# define AAC_DEBUG_PREAMBLE KERN_INFO
-# define AAC_DEBUG_POSTAMBLE
-
static int ioctl_send_fib(struct aac_dev * dev, void __user *arg)
{
struct hw_fib * kfib;
@@ -158,11 +157,12 @@
/**
* open_getadapter_fib - Get the next fib
+ * @dev: adapter is being processed
+ * @arg: arguments to the open call
*
* This routine will get the next Fib, if available, from the AdapterFibContext
* passed in from the user.
*/
-
static int open_getadapter_fib(struct aac_dev * dev, void __user *arg)
{
struct aac_fib_context * fibctx;
@@ -234,7 +234,6 @@
* This routine will get the next Fib, if available, from the AdapterFibContext
* passed in from the user.
*/
-
static int next_getadapter_fib(struct aac_dev * dev, void __user *arg)
{
struct fib_ioctl f;
@@ -455,11 +454,10 @@
/**
- *
* aac_send_raw_scb
- *
+ * @dev: adapter is being processed
+ * @arg: arguments to the send call
*/
-
static int aac_send_raw_srb(struct aac_dev* dev, void __user * arg)
{
struct fib* srbfib;
diff --git a/drivers/scsi/aacraid/commsup.c b/drivers/scsi/aacraid/commsup.c
index 8ee4e1a..adbdc3b 100644
--- a/drivers/scsi/aacraid/commsup.c
+++ b/drivers/scsi/aacraid/commsup.c
@@ -214,6 +214,7 @@
/**
* aac_fib_alloc_tag-allocate a fib using tags
* @dev: Adapter to allocate the fib for
+ * @scmd: SCSI command
*
* Allocate a fib from the adapter fib pool using tags
* from the blk layer.
@@ -405,8 +406,8 @@
* aac_queue_get - get the next free QE
* @dev: Adapter
* @index: Returned index
- * @priority: Priority of fib
- * @fib: Fib to associate with the queue entry
+ * @qid: Queue number
+ * @hw_fib: Fib to associate with the queue entry
* @wait: Wait if queue full
* @fibptr: Driver fib object to go with fib
* @nonotify: Don't notify the adapter
@@ -934,7 +935,7 @@
/**
* aac_fib_complete - fib completion handler
- * @fib: FIB to complete
+ * @fibptr: FIB to complete
*
* Will do all necessary work to complete a FIB.
*/
@@ -1049,6 +1050,7 @@
}
}
+#define AIF_SNIFF_TIMEOUT (500*HZ)
/**
* aac_handle_aif - Handle a message from the firmware
* @dev: Which adapter this fib is from
@@ -1057,8 +1059,6 @@
* This routine handles a driver notify fib from the adapter and
* dispatches it to the appropriate routine for handling.
*/
-
-#define AIF_SNIFF_TIMEOUT (500*HZ)
static void aac_handle_aif(struct aac_dev * dev, struct fib * fibptr)
{
struct hw_fib * hw_fib = fibptr->hw_fib_va;
@@ -2416,7 +2416,7 @@
/**
* aac_command_thread - command processing thread
- * @dev: Adapter to monitor
+ * @data: Adapter to monitor
*
* Waits on the commandready event in it's queue. When the event gets set
* it will pull FIBs off it's queue. It will continue to pull FIBs off
diff --git a/drivers/scsi/aacraid/dpcsup.c b/drivers/scsi/aacraid/dpcsup.c
index a557aa6..fbe334c 100644
--- a/drivers/scsi/aacraid/dpcsup.c
+++ b/drivers/scsi/aacraid/dpcsup.c
@@ -99,10 +99,11 @@
}
if (hwfib->header.XferState & cpu_to_le32(NoResponseExpected | Async))
{
- if (hwfib->header.XferState & cpu_to_le32(NoResponseExpected))
+ if (hwfib->header.XferState & cpu_to_le32(NoResponseExpected)) {
FIB_COUNTER_INCREMENT(aac_config.NoResponseRecved);
- else
+ } else {
FIB_COUNTER_INCREMENT(aac_config.AsyncRecved);
+ }
/*
* NOTE: we cannot touch the fib after this
* call, because it may have been deallocated.
@@ -229,7 +230,6 @@
struct fib *fibctx;
struct aac_dev *dev;
struct aac_aifcmd *cmd;
- int status;
fibctx = (struct fib *)context;
BUG_ON(fibptr == NULL);
@@ -249,7 +249,7 @@
cmd = (struct aac_aifcmd *) fib_data(fibctx);
cmd->command = cpu_to_le32(AifReqEvent);
- status = aac_fib_send(AifRequest,
+ aac_fib_send(AifRequest,
fibctx,
sizeof(struct hw_fib)-sizeof(struct aac_fibhdr),
FsaNormal,
@@ -258,7 +258,7 @@
}
-/**
+/*
* aac_intr_normal - Handle command replies
* @dev: Device
* @index: completion reference
@@ -403,12 +403,13 @@
if (hwfib->header.XferState &
cpu_to_le32(NoResponseExpected | Async)) {
if (hwfib->header.XferState & cpu_to_le32(
- NoResponseExpected))
+ NoResponseExpected)) {
FIB_COUNTER_INCREMENT(
aac_config.NoResponseRecved);
- else
+ } else {
FIB_COUNTER_INCREMENT(
aac_config.AsyncRecved);
+ }
start_callback = 1;
} else {
unsigned long flagv;
diff --git a/drivers/scsi/aacraid/linit.c b/drivers/scsi/aacraid/linit.c
index a308e86a..f043e37 100644
--- a/drivers/scsi/aacraid/linit.c
+++ b/drivers/scsi/aacraid/linit.c
@@ -230,8 +230,8 @@
/**
* aac_queuecommand - queue a SCSI command
+ * @shost: Scsi host to queue command on
* @cmd: SCSI command to queue
- * @done: Function to call on command completion
*
* Queues a command for execution by the associated Host Adapter.
*
@@ -363,9 +363,10 @@
param->cylinders = cap_to_cyls(capacity, param->heads * param->sectors);
if (num < 4 && end_sec == param->sectors) {
- if (param->cylinders != saved_cylinders)
+ if (param->cylinders != saved_cylinders) {
dprintk((KERN_DEBUG "Adopting geometry: heads=%d, sectors=%d from partition table %d.\n",
param->heads, param->sectors, num));
+ }
} else if (end_head > 0 || end_sec > 0) {
dprintk((KERN_DEBUG "Strange geometry: heads=%d, sectors=%d in partition table %d.\n",
end_head + 1, end_sec, num));
@@ -1159,7 +1160,6 @@
/**
* aac_cfg_ioctl - AAC configuration request
- * @inode: inode of device
* @file: file handle
* @cmd: ioctl command code
* @arg: argument
diff --git a/drivers/scsi/aacraid/nark.c b/drivers/scsi/aacraid/nark.c
index b5d6b24..4745a99 100644
--- a/drivers/scsi/aacraid/nark.c
+++ b/drivers/scsi/aacraid/nark.c
@@ -24,6 +24,7 @@
/**
* aac_nark_ioremap
+ * @dev: device to ioremap
* @size: mapping resize request
*
*/
diff --git a/drivers/scsi/aacraid/rkt.c b/drivers/scsi/aacraid/rkt.c
index 5f2cede..8ebc67e 100644
--- a/drivers/scsi/aacraid/rkt.c
+++ b/drivers/scsi/aacraid/rkt.c
@@ -57,6 +57,7 @@
/**
* aac_rkt_ioremap
+ * @dev: device to ioremap
* @size: mapping resize request
*
*/
@@ -77,8 +78,8 @@
* aac_rkt_init - initialize an i960 based AAC card
* @dev: device to configure
*
- * Allocate and set up resources for the i960 based AAC variants. The
- * device_interface in the commregion will be allocated and linked
+ * Allocate and set up resources for the i960 based AAC variants. The
+ * device_interface in the commregion will be allocated and linked
* to the comm region.
*/
diff --git a/drivers/scsi/aacraid/rx.c b/drivers/scsi/aacraid/rx.c
index 3dea348..cdccf9a 100644
--- a/drivers/scsi/aacraid/rx.c
+++ b/drivers/scsi/aacraid/rx.c
@@ -144,7 +144,16 @@
* @dev: Adapter
* @command: Command to execute
* @p1: first parameter
- * @ret: adapter status
+ * @p2: second parameter
+ * @p3: third parameter
+ * @p4: forth parameter
+ * @p5: fifth parameter
+ * @p6: sixth parameter
+ * @status: adapter status
+ * @r1: first return value
+ * @r2: second return value
+ * @r3: third return value
+ * @r4: forth return value
*
* This routine will send a synchronous command to the adapter and wait
* for its completion.
@@ -443,6 +452,7 @@
/**
* aac_rx_ioremap
+ * @dev: adapter
* @size: mapping resize request
*
*/
diff --git a/drivers/scsi/aacraid/sa.c b/drivers/scsi/aacraid/sa.c
index aa5d763..c9a1dad 100644
--- a/drivers/scsi/aacraid/sa.c
+++ b/drivers/scsi/aacraid/sa.c
@@ -135,13 +135,21 @@
* @dev: Adapter
* @command: Command to execute
* @p1: first parameter
+ * @p2: second parameter
+ * @p3: third parameter
+ * @p4: forth parameter
+ * @p5: fifth parameter
+ * @p6: sixth parameter
* @ret: adapter status
+ * @r1: first return value
+ * @r2: second return value
+ * @r3: third return value
+ * @r4: forth return value
*
- * This routine will send a synchronous command to the adapter and wait
+ * This routine will send a synchronous command to the adapter and wait
* for its completion.
*/
-
-static int sa_sync_cmd(struct aac_dev *dev, u32 command,
+static int sa_sync_cmd(struct aac_dev *dev, u32 command,
u32 p1, u32 p2, u32 p3, u32 p4, u32 p5, u32 p6,
u32 *ret, u32 *r1, u32 *r2, u32 *r3, u32 *r4)
{
@@ -283,6 +291,7 @@
/**
* aac_sa_ioremap
+ * @dev: device to ioremap
* @size: mapping resize request
*
*/
@@ -300,8 +309,8 @@
* aac_sa_init - initialize an ARM based AAC card
* @dev: device to configure
*
- * Allocate and set up resources for the ARM based AAC variants. The
- * device_interface in the commregion will be allocated and linked
+ * Allocate and set up resources for the ARM based AAC variants. The
+ * device_interface in the commregion will be allocated and linked
* to the comm region.
*/
diff --git a/drivers/scsi/aacraid/src.c b/drivers/scsi/aacraid/src.c
index 787ec9b..11ef582 100644
--- a/drivers/scsi/aacraid/src.c
+++ b/drivers/scsi/aacraid/src.c
@@ -191,7 +191,16 @@
* @dev: Adapter
* @command: Command to execute
* @p1: first parameter
- * @ret: adapter status
+ * @p2: second parameter
+ * @p3: third parameter
+ * @p4: forth parameter
+ * @p5: fifth parameter
+ * @p6: sixth parameter
+ * @status: adapter status
+ * @r1: first return value
+ * @r2: second return valu
+ * @r3: third return value
+ * @r4: forth return value
*
* This routine will send a synchronous command to the adapter and wait
* for its completion.
@@ -602,6 +611,7 @@
/**
* aac_src_ioremap
+ * @dev: device ioremap
* @size: mapping resize request
*
*/
@@ -632,6 +642,7 @@
/**
* aac_srcv_ioremap
+ * @dev: device ioremap
* @size: mapping resize request
*
*/
diff --git a/drivers/scsi/aha152x.c b/drivers/scsi/aha152x.c
index 90f97df..d8e19af 100644
--- a/drivers/scsi/aha152x.c
+++ b/drivers/scsi/aha152x.c
@@ -2030,8 +2030,7 @@
fifodata, GETPORT(FIFOSTAT));
SETPORT(DMACNTRL0, ENDMA|_8BIT);
while(fifodata>0) {
- int data;
- data=GETPORT(DATAPORT);
+ GETPORT(DATAPORT);
fifodata--;
DATA_LEN++;
}
diff --git a/drivers/scsi/aic7xxx/aic79xx_core.c b/drivers/scsi/aic7xxx/aic79xx_core.c
index e4a09b9..c912d29 100644
--- a/drivers/scsi/aic7xxx/aic79xx_core.c
+++ b/drivers/scsi/aic7xxx/aic79xx_core.c
@@ -1735,10 +1735,8 @@
sg_list = (struct ahd_dma64_seg*)scb->sg_list;
for (i = 0; i < scb->sg_count; i++) {
uint64_t addr;
- uint32_t len;
addr = ahd_le64toh(sg_list[i].addr);
- len = ahd_le32toh(sg_list[i].len);
printk("sg[%d] - Addr 0x%x%x : Length %d%s\n",
i,
(uint32_t)((addr >> 32) & 0xFFFFFFFF),
@@ -1906,9 +1904,6 @@
{
struct ahd_devinfo devinfo;
struct scb *scb;
- struct ahd_initiator_tinfo *targ_info;
- struct ahd_tmode_tstate *tstate;
- struct ahd_transinfo *tinfo;
u_int scbid;
/*
@@ -1936,12 +1931,6 @@
SCB_GET_LUN(scb),
SCB_GET_CHANNEL(ahd, scb),
ROLE_INITIATOR);
- targ_info = ahd_fetch_transinfo(ahd,
- devinfo.channel,
- devinfo.our_scsiid,
- devinfo.target,
- &tstate);
- tinfo = &targ_info->curr;
ahd_set_width(ahd, &devinfo, MSG_EXT_WDTR_BUS_8_BIT,
AHD_TRANS_ACTIVE, /*paused*/TRUE);
ahd_set_syncrate(ahd, &devinfo, /*period*/0,
@@ -2669,7 +2658,6 @@
struct scb *scb;
u_int scbid;
u_int lqistat1;
- u_int lqistat2;
u_int msg_out;
u_int curphase;
u_int lastphase;
@@ -2680,7 +2668,7 @@
scb = NULL;
ahd_set_modes(ahd, AHD_MODE_SCSI, AHD_MODE_SCSI);
lqistat1 = ahd_inb(ahd, LQISTAT1) & ~(LQIPHASE_LQ|LQIPHASE_NLQ);
- lqistat2 = ahd_inb(ahd, LQISTAT2);
+ ahd_inb(ahd, LQISTAT2);
if ((lqistat1 & (LQICRCI_NLQ|LQICRCI_LQ)) == 0
&& (ahd->bugs & AHD_NLQICRC_DELAYED_BUG) != 0) {
u_int lqistate;
@@ -4218,13 +4206,11 @@
pending_scb_count = 0;
LIST_FOREACH(pending_scb, &ahd->pending_scbs, pending_links) {
struct ahd_devinfo devinfo;
- struct ahd_initiator_tinfo *tinfo;
struct ahd_tmode_tstate *tstate;
ahd_scb_devinfo(ahd, &devinfo, pending_scb);
- tinfo = ahd_fetch_transinfo(ahd, devinfo.channel,
- devinfo.our_scsiid,
- devinfo.target, &tstate);
+ ahd_fetch_transinfo(ahd, devinfo.channel, devinfo.our_scsiid,
+ devinfo.target, &tstate);
if ((tstate->auto_negotiate & devinfo.target_mask) == 0
&& (pending_scb->flags & SCB_AUTO_NEGOTIATE) != 0) {
pending_scb->flags &= ~SCB_AUTO_NEGOTIATE;
diff --git a/drivers/scsi/aic7xxx/aic79xx_osm.c b/drivers/scsi/aic7xxx/aic79xx_osm.c
index dc4fe33..d019e3f 100644
--- a/drivers/scsi/aic7xxx/aic79xx_osm.c
+++ b/drivers/scsi/aic7xxx/aic79xx_osm.c
@@ -700,9 +700,6 @@
static int
ahd_linux_slave_configure(struct scsi_device *sdev)
{
- struct ahd_softc *ahd;
-
- ahd = *((struct ahd_softc **)sdev->host->hostdata);
if (bootverbose)
sdev_printk(KERN_INFO, sdev, "Slave Configure\n");
@@ -778,16 +775,13 @@
struct scb *reset_scb;
u_int cdb_byte;
int retval = SUCCESS;
- int paused;
- int wait;
struct ahd_initiator_tinfo *tinfo;
struct ahd_tmode_tstate *tstate;
unsigned long flags;
DECLARE_COMPLETION_ONSTACK(done);
reset_scb = NULL;
- paused = FALSE;
- wait = FALSE;
+
ahd = *(struct ahd_softc **)cmd->device->host->hostdata;
scmd_printk(KERN_INFO, cmd,
@@ -1793,10 +1787,12 @@
*/
cmd->sense_buffer[0] = 0;
if (ahd_get_transaction_status(scb) == CAM_REQ_INPROG) {
+#ifdef AHD_REPORT_UNDERFLOWS
uint32_t amount_xferred;
amount_xferred =
ahd_get_transfer_length(scb) - ahd_get_residual(scb);
+#endif
if ((scb->flags & SCB_TRANSMISSION_ERROR) != 0) {
#ifdef AHD_DEBUG
if ((ahd_debug & AHD_SHOW_MISC) != 0) {
@@ -2147,7 +2143,7 @@
u_int last_phase;
u_int saved_scsiid;
u_int cdb_byte;
- int retval;
+ int retval = SUCCESS;
int was_paused;
int paused;
int wait;
@@ -2185,8 +2181,7 @@
* so we must not still own the command.
*/
scmd_printk(KERN_INFO, cmd, "Is not an active device\n");
- retval = SUCCESS;
- goto no_cmd;
+ goto done;
}
/*
@@ -2199,7 +2194,7 @@
if (pending_scb == NULL) {
scmd_printk(KERN_INFO, cmd, "Command not found\n");
- goto no_cmd;
+ goto done;
}
if ((pending_scb->flags & SCB_RECOVERY_SCB) != 0) {
@@ -2207,7 +2202,7 @@
* We can't queue two recovery actions using the same SCB
*/
retval = FAILED;
- goto done;
+ goto done;
}
/*
@@ -2222,7 +2217,7 @@
if ((pending_scb->flags & SCB_ACTIVE) == 0) {
scmd_printk(KERN_INFO, cmd, "Command already completed\n");
- goto no_cmd;
+ goto done;
}
printk("%s: At time of recovery, card was %spaused\n",
@@ -2239,7 +2234,6 @@
printk("%s:%d:%d:%d: Cmd aborted from QINFIFO\n",
ahd_name(ahd), cmd->device->channel,
cmd->device->id, (u8)cmd->device->lun);
- retval = SUCCESS;
goto done;
}
@@ -2336,17 +2330,10 @@
} else {
scmd_printk(KERN_INFO, cmd, "Unable to deliver message\n");
retval = FAILED;
- goto done;
}
-no_cmd:
- /*
- * Our assumption is that if we don't have the command, no
- * recovery action was required, so we return success. Again,
- * the semantics of the mid-layer recovery engine are not
- * well defined, so this may change in time.
- */
- retval = SUCCESS;
+
+ ahd_restore_modes(ahd, saved_modes);
done:
if (paused)
ahd_unpause(ahd);
diff --git a/drivers/scsi/aic7xxx/aic7xxx_osm.c b/drivers/scsi/aic7xxx/aic7xxx_osm.c
index 2edfa05..e7ccb8b 100644
--- a/drivers/scsi/aic7xxx/aic7xxx_osm.c
+++ b/drivers/scsi/aic7xxx/aic7xxx_osm.c
@@ -564,8 +564,6 @@
struct scsi_target **ahc_targp = ahc_linux_target_in_softc(starget);
unsigned short scsirate;
struct ahc_devinfo devinfo;
- struct ahc_initiator_tinfo *tinfo;
- struct ahc_tmode_tstate *tstate;
char channel = starget->channel + 'A';
unsigned int our_id = ahc->our_id;
unsigned int target_offset;
@@ -612,9 +610,6 @@
spi_max_offset(starget) = 0;
spi_min_period(starget) =
ahc_find_period(ahc, scsirate, maxsync);
-
- tinfo = ahc_fetch_transinfo(ahc, channel, ahc->our_id,
- starget->id, &tstate);
}
ahc_compile_devinfo(&devinfo, our_id, starget->id,
CAM_LUN_WILDCARD, channel,
@@ -671,10 +666,6 @@
static int
ahc_linux_slave_configure(struct scsi_device *sdev)
{
- struct ahc_softc *ahc;
-
- ahc = *((struct ahc_softc **)sdev->host->hostdata);
-
if (bootverbose)
sdev_printk(KERN_INFO, sdev, "Slave Configure\n");
@@ -1601,7 +1592,6 @@
case AC_TRANSFER_NEG:
{
struct scsi_target *starget;
- struct ahc_linux_target *targ;
struct ahc_initiator_tinfo *tinfo;
struct ahc_tmode_tstate *tstate;
int target_offset;
@@ -1635,7 +1625,6 @@
starget = ahc->platform_data->starget[target_offset];
if (starget == NULL)
break;
- targ = scsi_transport_target_data(starget);
target_ppr_options =
(spi_dt(starget) ? MSG_EXT_PPR_DT_REQ : 0)
@@ -1722,10 +1711,12 @@
*/
cmd->sense_buffer[0] = 0;
if (ahc_get_transaction_status(scb) == CAM_REQ_INPROG) {
+#ifdef AHC_REPORT_UNDERFLOWS
uint32_t amount_xferred;
amount_xferred =
ahc_get_transfer_length(scb) - ahc_get_residual(scb);
+#endif
if ((scb->flags & SCB_TRANSMISSION_ERROR) != 0) {
#ifdef AHC_DEBUG
if ((ahc_debug & AHC_SHOW_MISC) != 0) {
diff --git a/drivers/scsi/aic94xx/aic94xx_dev.c b/drivers/scsi/aic94xx/aic94xx_dev.c
index 604a533..73506a4 100644
--- a/drivers/scsi/aic94xx/aic94xx_dev.c
+++ b/drivers/scsi/aic94xx/aic94xx_dev.c
@@ -236,7 +236,7 @@
/**
* asd_init_sata_pm_port_ddb -- SATA Port Multiplier Port
- * dev: pointer to domain device
+ * @dev: pointer to domain device
*
* For SATA Port Multiplier Ports we need to allocate one SATA Port
* Multiplier Port DDB and depending on whether the target on it
@@ -281,7 +281,7 @@
/**
* asd_init_sata_pm_ddb -- SATA Port Multiplier
- * dev: pointer to domain device
+ * @dev: pointer to domain device
*
* For STP and direct-attached SATA Port Multipliers we need
* one target port DDB entry and one SATA PM table DDB entry.
diff --git a/drivers/scsi/aic94xx/aic94xx_hwi.c b/drivers/scsi/aic94xx/aic94xx_hwi.c
index c5a46c5..9256ab7 100644
--- a/drivers/scsi/aic94xx/aic94xx_hwi.c
+++ b/drivers/scsi/aic94xx/aic94xx_hwi.c
@@ -575,7 +575,7 @@
/**
* asd_init_ctxmem -- initialize context memory
- * asd_ha: pointer to host adapter structure
+ * @asd_ha: pointer to host adapter structure
*
* This function sets the maximum number of SCBs and
* DDBs which can be used by the sequencer. This is normally
@@ -1146,7 +1146,6 @@
/**
* asd_start_timers -- (add and) start timers of SCBs
* @list: pointer to struct list_head of the scbs
- * @to: timeout in jiffies
*
* If an SCB in the @list has no timer function, assign the default
* one, then start the timer of the SCB. This function is
diff --git a/drivers/scsi/aic94xx/aic94xx_init.c b/drivers/scsi/aic94xx/aic94xx_init.c
index bef47f3..a195bfe 100644
--- a/drivers/scsi/aic94xx/aic94xx_init.c
+++ b/drivers/scsi/aic94xx/aic94xx_init.c
@@ -530,7 +530,7 @@
return 0;
}
-/**
+/*
* asd_free_edbs -- free empty data buffers
* asd_ha: pointer to host adapter structure
*/
diff --git a/drivers/scsi/aic94xx/aic94xx_scb.c b/drivers/scsi/aic94xx/aic94xx_scb.c
index 4a80ec0..c264b4b 100644
--- a/drivers/scsi/aic94xx/aic94xx_scb.c
+++ b/drivers/scsi/aic94xx/aic94xx_scb.c
@@ -123,8 +123,8 @@
/**
* asd_get_attached_sas_addr -- extract/generate attached SAS address
- * phy: pointer to asd_phy
- * sas_addr: pointer to buffer where the SAS address is to be written
+ * @phy: pointer to asd_phy
+ * @sas_addr: pointer to buffer where the SAS address is to be written
*
* This function extracts the SAS address from an IDENTIFY frame
* received. If OOB is SATA, then a SAS address is generated from the
@@ -847,7 +847,7 @@
/**
* asd_ascb_timedout -- called when a pending SCB's timer has expired
- * @data: unsigned long, a pointer to the ascb in question
+ * @t: Timer context used to fetch the SCB
*
* This is the default timeout function which does the most necessary.
* Upper layers can implement their own timeout function, say to free
diff --git a/drivers/scsi/aic94xx/aic94xx_seq.c b/drivers/scsi/aic94xx/aic94xx_seq.c
index 11853ec..c0f685c 100644
--- a/drivers/scsi/aic94xx/aic94xx_seq.c
+++ b/drivers/scsi/aic94xx/aic94xx_seq.c
@@ -582,6 +582,7 @@
/**
* asd_init_lseq_mip -- initialize LSEQ Mode independent pages 0-3
* @asd_ha: pointer to host adapter structure
+ * @lseq: link sequencer
*/
static void asd_init_lseq_mip(struct asd_ha_struct *asd_ha, u8 lseq)
{
@@ -669,6 +670,7 @@
/**
* asd_init_lseq_mdp -- initialize LSEQ mode dependent pages.
* @asd_ha: pointer to host adapter structure
+ * @lseq: link sequencer
*/
static void asd_init_lseq_mdp(struct asd_ha_struct *asd_ha, int lseq)
{
@@ -953,6 +955,7 @@
/**
* asd_init_lseq_cio -- initialize LmSEQ CIO registers
* @asd_ha: pointer to host adapter structure
+ * @lseq: link sequencer
*/
static void asd_init_lseq_cio(struct asd_ha_struct *asd_ha, int lseq)
{
@@ -1345,7 +1348,8 @@
/**
* asd_update_port_links -- update port_map_by_links and phy_is_up
- * @sas_phy: pointer to the phy which has been added to a port
+ * @asd_ha: pointer to host adapter structure
+ * @phy: pointer to the phy which has been added to a port
*
* 1) When a link reset has completed and we got BYTES DMAED with a
* valid frame we call this function for that phy, to indicate that
diff --git a/drivers/scsi/aic94xx/aic94xx_tmf.c b/drivers/scsi/aic94xx/aic94xx_tmf.c
index f814026..1fcee65 100644
--- a/drivers/scsi/aic94xx/aic94xx_tmf.c
+++ b/drivers/scsi/aic94xx/aic94xx_tmf.c
@@ -673,7 +673,7 @@
/**
* asd_query_task -- send a QUERY TASK TMF to an I_T_L_Q nexus
- * task: pointer to sas_task struct of interest
+ * @task: pointer to sas_task struct of interest
*
* Returns: TMF_RESP_FUNC_COMPLETE if the task is not in the task set,
* or TMF_RESP_FUNC_SUCC if the task is in the task set.
diff --git a/drivers/scsi/arcmsr/arcmsr_hba.c b/drivers/scsi/arcmsr/arcmsr_hba.c
index 30914c8..fa562a0 100644
--- a/drivers/scsi/arcmsr/arcmsr_hba.c
+++ b/drivers/scsi/arcmsr/arcmsr_hba.c
@@ -283,11 +283,10 @@
}
case ACB_ADAPTER_TYPE_D: {
void __iomem *mem_base0;
- unsigned long addr, range, flags;
+ unsigned long addr, range;
addr = (unsigned long)pci_resource_start(pdev, 0);
range = pci_resource_len(pdev, 0);
- flags = pci_resource_flags(pdev, 0);
mem_base0 = ioremap(addr, range);
if (!mem_base0) {
pr_notice("arcmsr%d: memory mapping region fail\n",
@@ -1067,12 +1066,11 @@
static int arcmsr_suspend(struct pci_dev *pdev, pm_message_t state)
{
- uint32_t intmask_org;
struct Scsi_Host *host = pci_get_drvdata(pdev);
struct AdapterControlBlock *acb =
(struct AdapterControlBlock *)host->hostdata;
- intmask_org = arcmsr_disable_outbound_ints(acb);
+ arcmsr_disable_outbound_ints(acb);
arcmsr_free_irq(pdev, acb);
del_timer_sync(&acb->eternal_timer);
if (set_date_time)
@@ -1407,7 +1405,7 @@
struct ARCMSR_CDB *pARCMSR_CDB;
bool error;
struct CommandControlBlock *pCCB;
- unsigned long ccb_cdb_phy, cdb_phy_hipart;
+ unsigned long ccb_cdb_phy;
switch (acb->adapter_type) {
@@ -1489,8 +1487,6 @@
((toggle ^ 0x4000) + 1);
doneq_index = pmu->doneq_index;
spin_unlock_irqrestore(&acb->doneq_lock, flags);
- cdb_phy_hipart = pmu->done_qbuffer[doneq_index &
- 0xFFF].addressHigh;
addressLow = pmu->done_qbuffer[doneq_index &
0xFFF].addressLow;
ccb_cdb_phy = (addressLow & 0xFFFFFFF0);
@@ -2445,7 +2441,7 @@
struct MessageUnit_D *pmu;
struct ARCMSR_CDB *arcmsr_cdb;
struct CommandControlBlock *ccb;
- unsigned long flags, ccb_cdb_phy, cdb_phy_hipart;
+ unsigned long flags, ccb_cdb_phy;
spin_lock_irqsave(&acb->doneq_lock, flags);
pmu = acb->pmuD;
@@ -2459,8 +2455,6 @@
pmu->doneq_index = index_stripped ? (index_stripped | toggle) :
((toggle ^ 0x4000) + 1);
doneq_index = pmu->doneq_index;
- cdb_phy_hipart = pmu->done_qbuffer[doneq_index &
- 0xFFF].addressHigh;
addressLow = pmu->done_qbuffer[doneq_index &
0xFFF].addressLow;
ccb_cdb_phy = (addressLow & 0xFFFFFFF0);
@@ -3495,7 +3489,7 @@
bool error;
uint32_t poll_ccb_done = 0, poll_count = 0, flag_ccb;
int rtn, doneq_index, index_stripped, outbound_write_pointer, toggle;
- unsigned long flags, ccb_cdb_phy, cdb_phy_hipart;
+ unsigned long flags, ccb_cdb_phy;
struct ARCMSR_CDB *arcmsr_cdb;
struct CommandControlBlock *pCCB;
struct MessageUnit_D *pmu = acb->pmuD;
@@ -3527,8 +3521,6 @@
((toggle ^ 0x4000) + 1);
doneq_index = pmu->doneq_index;
spin_unlock_irqrestore(&acb->doneq_lock, flags);
- cdb_phy_hipart = pmu->done_qbuffer[doneq_index &
- 0xFFF].addressHigh;
flag_ccb = pmu->done_qbuffer[doneq_index & 0xFFF].addressLow;
ccb_cdb_phy = (flag_ccb & 0xFFFFFFF0);
if (acb->cdb_phyadd_hipart)
diff --git a/drivers/scsi/arm/cumana_2.c b/drivers/scsi/arm/cumana_2.c
index 65691c2..29294f0 100644
--- a/drivers/scsi/arm/cumana_2.c
+++ b/drivers/scsi/arm/cumana_2.c
@@ -450,7 +450,7 @@
if (info->info.scsi.dma != NO_DMA)
free_dma(info->info.scsi.dma);
- free_irq(ec->irq, host);
+ free_irq(ec->irq, info);
out_release:
fas216_release(host);
diff --git a/drivers/scsi/arm/eesox.c b/drivers/scsi/arm/eesox.c
index 6e204a2..591ae2a 100644
--- a/drivers/scsi/arm/eesox.c
+++ b/drivers/scsi/arm/eesox.c
@@ -571,7 +571,7 @@
if (info->info.scsi.dma != NO_DMA)
free_dma(info->info.scsi.dma);
- free_irq(ec->irq, host);
+ free_irq(ec->irq, info);
out_remove:
fas216_remove(host);
diff --git a/drivers/scsi/arm/powertec.c b/drivers/scsi/arm/powertec.c
index 772a13e..d99ef01 100644
--- a/drivers/scsi/arm/powertec.c
+++ b/drivers/scsi/arm/powertec.c
@@ -378,7 +378,7 @@
if (info->info.scsi.dma != NO_DMA)
free_dma(info->info.scsi.dma);
- free_irq(ec->irq, host);
+ free_irq(ec->irq, info);
out_release:
fas216_release(host);
diff --git a/drivers/scsi/be2iscsi/be_iscsi.c b/drivers/scsi/be2iscsi/be_iscsi.c
index 2058d50..93da634 100644
--- a/drivers/scsi/be2iscsi/be_iscsi.c
+++ b/drivers/scsi/be2iscsi/be_iscsi.c
@@ -27,6 +27,7 @@
/**
* beiscsi_session_create - creates a new iscsi session
+ * @ep: pointer to iscsi ep
* @cmds_max: max commands supported
* @qdepth: max queue depth supported
* @initial_cmdsn: initial iscsi CMDSN
@@ -164,6 +165,7 @@
* @cls_session: pointer to iscsi cls session
* @cls_conn: pointer to iscsi cls conn
* @transport_fd: EP handle(64 bit)
+ * @is_leading: indicate if this is the session leading connection (MCS)
*
* This function binds the TCP Conn with iSCSI Connection and Session.
*/
@@ -992,7 +994,7 @@
/**
* beiscsi_free_ep - free endpoint
- * @ep: pointer to iscsi endpoint structure
+ * @beiscsi_ep: pointer to device endpoint struct
*/
static void beiscsi_free_ep(struct beiscsi_endpoint *beiscsi_ep)
{
@@ -1027,9 +1029,10 @@
/**
* beiscsi_open_conn - Ask FW to open a TCP connection
- * @ep: endpoint to be used
+ * @ep: pointer to device endpoint struct
* @src_addr: The source IP address
* @dst_addr: The Destination IP address
+ * @non_blocking: blocking or non-blocking call
*
* Asks the FW to open a TCP connection
*/
@@ -1123,7 +1126,7 @@
/**
* beiscsi_ep_connect - Ask chip to create TCP Conn
- * @scsi_host: Pointer to scsi_host structure
+ * @shost: Pointer to scsi_host structure
* @dst_addr: The IP address of Target
* @non_blocking: blocking or non-blocking call
*
@@ -1228,7 +1231,7 @@
/**
* beiscsi_conn_close - Invalidate and upload connection
- * @ep: The iscsi endpoint
+ * @beiscsi_ep: pointer to device endpoint struct
*
* Returns 0 on success, -1 on failure.
*/
diff --git a/drivers/scsi/be2iscsi/be_main.c b/drivers/scsi/be2iscsi/be_main.c
index 9b81cfbb..8dc2e08 100644
--- a/drivers/scsi/be2iscsi/be_main.c
+++ b/drivers/scsi/be2iscsi/be_main.c
@@ -977,7 +977,7 @@
* alloc_wrb_handle - To allocate a wrb handle
* @phba: The hba pointer
* @cid: The cid to use for allocation
- * @pwrb_context: ptr to ptr to wrb context
+ * @pcontext: ptr to ptr to wrb context
*
* This happens under session_lock until submission to chip
*/
@@ -1394,7 +1394,7 @@
spin_unlock_bh(&session->back_lock);
}
-/**
+/*
* ASYNC PDUs include
* a. Unsolicited NOP-In (target initiated NOP-In)
* b. ASYNC Messages
diff --git a/drivers/scsi/be2iscsi/be_mgmt.c b/drivers/scsi/be2iscsi/be_mgmt.c
index a2d69b28..96d6e38 100644
--- a/drivers/scsi/be2iscsi/be_mgmt.c
+++ b/drivers/scsi/be2iscsi/be_mgmt.c
@@ -97,6 +97,7 @@
/**
* mgmt_open_connection()- Establish a TCP CXN
+ * @phba: driver priv structure
* @dst_addr: Destination Address
* @beiscsi_ep: ptr to device endpoint struct
* @nonemb_cmd: ptr to memory allocated for command
@@ -209,7 +210,7 @@
return tag;
}
-/*
+/**
* beiscsi_exec_nemb_cmd()- execute non-embedded MBX cmd
* @phba: driver priv structure
* @nonemb_cmd: DMA address of the MBX command to be issued
diff --git a/drivers/scsi/bfa/bfa_core.c b/drivers/scsi/bfa/bfa_core.c
index fb4c469..6846ca8 100644
--- a/drivers/scsi/bfa/bfa_core.c
+++ b/drivers/scsi/bfa/bfa_core.c
@@ -1237,7 +1237,7 @@
complete(&bfad->disable_comp);
}
-/**
+/*
* configure queue registers from firmware response
*/
static void
diff --git a/drivers/scsi/bfa/bfa_fcpim.c b/drivers/scsi/bfa/bfa_fcpim.c
index 766f2b5..29f9956 100644
--- a/drivers/scsi/bfa/bfa_fcpim.c
+++ b/drivers/scsi/bfa/bfa_fcpim.c
@@ -2335,9 +2335,7 @@
wwn_t rpwwn, struct scsi_lun lun)
{
struct bfa_lun_mask_s *lunm_list;
- struct bfa_rport_s *rp = NULL;
struct bfa_fcs_lport_s *port = NULL;
- struct bfa_fcs_rport_s *rp_fcs;
int i;
/* in min cfg lunm_list could be NULL but no commands should run. */
@@ -2353,12 +2351,8 @@
port = bfa_fcs_lookup_port(
&((struct bfad_s *)bfa->bfad)->bfa_fcs,
vf_id, *pwwn);
- if (port) {
+ if (port)
*pwwn = port->port_cfg.pwwn;
- rp_fcs = bfa_fcs_lport_get_rport_by_pwwn(port, rpwwn);
- if (rp_fcs)
- rp = rp_fcs->bfa_rport;
- }
}
lunm_list = bfa_get_lun_mask_list(bfa);
@@ -3818,7 +3812,7 @@
}
-/**
+/*
* To send config req, first try to use throttle value from flash
* If 0, then use driver parameter
* We need to use min(flash_val, drv_val) because
diff --git a/drivers/scsi/bfa/bfa_fcs_rport.c b/drivers/scsi/bfa/bfa_fcs_rport.c
index fc294e1..143c35b 100644
--- a/drivers/scsi/bfa/bfa_fcs_rport.c
+++ b/drivers/scsi/bfa/bfa_fcs_rport.c
@@ -2240,15 +2240,12 @@
struct bfa_fcxp_s *fcxp;
struct fchs_s fchs;
struct bfa_fcs_lport_s *port = rport->port;
- struct fc_adisc_s *adisc;
bfa_trc(port->fcs, rx_fchs->s_id);
bfa_trc(port->fcs, rx_fchs->d_id);
rport->stats.adisc_rcvd++;
- adisc = (struct fc_adisc_s *) (rx_fchs + 1);
-
/*
* Accept if the itnim for this rport is online.
* Else reject the ADISC.
diff --git a/drivers/scsi/bfa/bfa_ioc.c b/drivers/scsi/bfa/bfa_ioc.c
index 93471d7..dd5821d 100644
--- a/drivers/scsi/bfa/bfa_ioc.c
+++ b/drivers/scsi/bfa/bfa_ioc.c
@@ -701,7 +701,7 @@
bfa_iocpf_sm_fwcheck_entry(struct bfa_iocpf_s *iocpf)
{
struct bfi_ioc_image_hdr_s fwhdr;
- u32 r32, fwstate, pgnum, pgoff, loff = 0;
+ u32 r32, fwstate, pgnum, loff = 0;
int i;
/*
@@ -731,7 +731,6 @@
* Clear fwver hdr
*/
pgnum = PSS_SMEM_PGNUM(iocpf->ioc->ioc_regs.smem_pg0, loff);
- pgoff = PSS_SMEM_PGOFF(loff);
writel(pgnum, iocpf->ioc->ioc_regs.host_page_num_fn);
for (i = 0; i < sizeof(struct bfi_ioc_image_hdr_s) / sizeof(u32); i++) {
@@ -1440,13 +1439,12 @@
void
bfa_ioc_fwver_get(struct bfa_ioc_s *ioc, struct bfi_ioc_image_hdr_s *fwhdr)
{
- u32 pgnum, pgoff;
+ u32 pgnum;
u32 loff = 0;
int i;
u32 *fwsig = (u32 *) fwhdr;
pgnum = PSS_SMEM_PGNUM(ioc->ioc_regs.smem_pg0, loff);
- pgoff = PSS_SMEM_PGOFF(loff);
writel(pgnum, ioc->ioc_regs.host_page_num_fn);
for (i = 0; i < (sizeof(struct bfi_ioc_image_hdr_s) / sizeof(u32));
@@ -1662,7 +1660,7 @@
bfa_ioc_fwsig_invalidate(struct bfa_ioc_s *ioc)
{
- u32 pgnum, pgoff;
+ u32 pgnum;
u32 loff = 0;
enum bfi_ioc_state ioc_fwstate;
@@ -1671,7 +1669,6 @@
return BFA_STATUS_ADAPTER_ENABLED;
pgnum = PSS_SMEM_PGNUM(ioc->ioc_regs.smem_pg0, loff);
- pgoff = PSS_SMEM_PGOFF(loff);
writel(pgnum, ioc->ioc_regs.host_page_num_fn);
bfa_mem_write(ioc->ioc_regs.smem_page_start, loff, BFA_IOC_FW_INV_SIGN);
@@ -1863,7 +1860,7 @@
u32 boot_env)
{
u32 *fwimg;
- u32 pgnum, pgoff;
+ u32 pgnum;
u32 loff = 0;
u32 chunkno = 0;
u32 i;
@@ -1892,8 +1889,6 @@
pgnum = PSS_SMEM_PGNUM(ioc->ioc_regs.smem_pg0, loff);
- pgoff = PSS_SMEM_PGOFF(loff);
-
writel(pgnum, ioc->ioc_regs.host_page_num_fn);
for (i = 0; i < fwimg_size; i++) {
@@ -4763,11 +4758,9 @@
struct bfa_ioc_s *ioc = diag->ioc;
struct bfa_diag_memtest_result *res = diag->result;
u32 loff = BFI_BOOT_MEMTEST_RES_ADDR;
- u32 pgnum, pgoff, i;
+ u32 pgnum, i;
pgnum = PSS_SMEM_PGNUM(ioc->ioc_regs.smem_pg0, loff);
- pgoff = PSS_SMEM_PGOFF(loff);
-
writel(pgnum, ioc->ioc_regs.host_page_num_fn);
for (i = 0; i < (sizeof(struct bfa_diag_memtest_result) /
@@ -5026,7 +5019,7 @@
/*
* Diag hmbox handler
*/
-void
+static void
bfa_diag_intr(void *diagarg, struct bfi_mbmsg_s *msg)
{
struct bfa_diag_s *diag = diagarg;
@@ -6649,8 +6642,8 @@
BFA_FLASH_READ_STATUS = 0x05, /* read status */
};
-/**
- * @brief hardware error definition
+/*
+ * Hardware error definition
*/
enum bfa_flash_err {
BFA_FLASH_NOT_PRESENT = -1, /*!< flash not present */
@@ -6664,8 +6657,8 @@
BFA_FLASH_ERR_LEN = -9, /*!< invalid length */
};
-/**
- * @brief flash command register data structure
+/*
+ * Flash command register data structure
*/
union bfa_flash_cmd_reg_u {
struct {
@@ -6688,8 +6681,8 @@
u32 i;
};
-/**
- * @brief flash device status register data structure
+/*
+ * Flash device status register data structure
*/
union bfa_flash_dev_status_reg_u {
struct {
@@ -6714,8 +6707,8 @@
u32 i;
};
-/**
- * @brief flash address register data structure
+/*
+ * Flash address register data structure
*/
union bfa_flash_addr_reg_u {
struct {
@@ -6730,7 +6723,7 @@
u32 i;
};
-/**
+/*
* dg flash_raw_private Flash raw private functions
*/
static void
@@ -6771,7 +6764,7 @@
return 0;
}
-/**
+/*
* @brief
* Flush FLI data fifo.
*
@@ -6784,7 +6777,6 @@
bfa_flash_fifo_flush(void __iomem *pci_bar)
{
u32 i;
- u32 t;
union bfa_flash_dev_status_reg_u dev_status;
dev_status.i = readl(pci_bar + FLI_DEV_STATUS_REG);
@@ -6794,7 +6786,7 @@
/* fifo counter in terms of words */
for (i = 0; i < dev_status.r.fifo_cnt; i++)
- t = readl(pci_bar + FLI_RDDATA_REG);
+ readl(pci_bar + FLI_RDDATA_REG);
/*
* Check the device status. It may take some time.
@@ -6811,7 +6803,7 @@
return 0;
}
-/**
+/*
* @brief
* Read flash status.
*
@@ -6856,7 +6848,7 @@
return ret_status;
}
-/**
+/*
* @brief
* Start flash read operation.
*
@@ -6902,7 +6894,7 @@
return 0;
}
-/**
+/*
* @brief
* Check flash read operation.
*
@@ -6918,7 +6910,8 @@
return 0;
}
-/**
+
+/*
* @brief
* End flash read operation.
*
@@ -6944,7 +6937,7 @@
bfa_flash_fifo_flush(pci_bar);
}
-/**
+/*
* @brief
* Perform flash raw read.
*
@@ -6970,7 +6963,7 @@
}
-bfa_status_t
+static bfa_status_t
bfa_flash_sem_get(void __iomem *bar)
{
u32 n = FLASH_BLOCKING_OP_MAX;
@@ -6983,7 +6976,7 @@
return BFA_STATUS_OK;
}
-void
+static void
bfa_flash_sem_put(void __iomem *bar)
{
writel(0, (bar + FLASH_SEM_LOCK_REG));
diff --git a/drivers/scsi/bfa/bfa_ioc_ct.c b/drivers/scsi/bfa/bfa_ioc_ct.c
index 6fd3383..fb74829 100644
--- a/drivers/scsi/bfa/bfa_ioc_ct.c
+++ b/drivers/scsi/bfa/bfa_ioc_ct.c
@@ -496,7 +496,7 @@
return BFA_FALSE;
}
-/**
+/*
* Called from bfa_ioc_attach() to map asic specific calls.
*/
static void
@@ -517,7 +517,7 @@
hwif->ioc_get_alt_fwstate = bfa_ioc_ct_get_alt_ioc_fwstate;
}
-/**
+/*
* Called from bfa_ioc_attach() to map asic specific calls.
*/
void
@@ -532,7 +532,7 @@
ioc->ioc_hwif = &hwif_ct;
}
-/**
+/*
* Called from bfa_ioc_attach() to map asic specific calls.
*/
void
diff --git a/drivers/scsi/bfa/bfa_port.c b/drivers/scsi/bfa/bfa_port.c
index 4511ec8..cfe2c9c 100644
--- a/drivers/scsi/bfa/bfa_port.c
+++ b/drivers/scsi/bfa/bfa_port.c
@@ -756,7 +756,7 @@
* @return void
*/
-void
+static void
bfa_cee_isr(void *cbarg, struct bfi_mbmsg_s *m)
{
union bfi_cee_i2h_msg_u *msg;
@@ -792,7 +792,7 @@
* @return void
*/
-void
+static void
bfa_cee_notify(void *arg, enum bfa_ioc_event_e event)
{
struct bfa_cee_s *cee = (struct bfa_cee_s *) arg;
diff --git a/drivers/scsi/bfa/bfa_svc.c b/drivers/scsi/bfa/bfa_svc.c
index 0b7d2e8..1e266c1 100644
--- a/drivers/scsi/bfa/bfa_svc.c
+++ b/drivers/scsi/bfa/bfa_svc.c
@@ -2718,7 +2718,7 @@
case BFA_FCPORT_SM_DPORTDISABLE:
case BFA_FCPORT_SM_ENABLE:
case BFA_FCPORT_SM_START:
- /**
+ /*
* Ignore event for a port that is ddport
*/
break;
@@ -3839,7 +3839,7 @@
return fcport->topology;
}
-/**
+/*
* Get config topology.
*/
enum bfa_port_topology
diff --git a/drivers/scsi/bfa/bfad_bsg.c b/drivers/scsi/bfa/bfad_bsg.c
index 412dbe1..fc51542 100644
--- a/drivers/scsi/bfa/bfad_bsg.c
+++ b/drivers/scsi/bfa/bfad_bsg.c
@@ -15,7 +15,7 @@
BFA_TRC_FILE(LDRV, BSG);
-int
+static int
bfad_iocmd_ioc_enable(struct bfad_s *bfad, void *cmd)
{
struct bfa_bsg_gen_s *iocmd = (struct bfa_bsg_gen_s *)cmd;
@@ -38,7 +38,7 @@
return 0;
}
-int
+static int
bfad_iocmd_ioc_disable(struct bfad_s *bfad, void *cmd)
{
struct bfa_bsg_gen_s *iocmd = (struct bfa_bsg_gen_s *)cmd;
@@ -146,7 +146,7 @@
return 0;
}
-int
+static int
bfad_iocmd_ioc_get_fwstats(struct bfad_s *bfad, void *cmd,
unsigned int payload_len)
{
@@ -176,7 +176,7 @@
return 0;
}
-int
+static int
bfad_iocmd_ioc_reset_stats(struct bfad_s *bfad, void *cmd, unsigned int v_cmd)
{
struct bfa_bsg_gen_s *iocmd = (struct bfa_bsg_gen_s *)cmd;
@@ -194,7 +194,7 @@
return 0;
}
-int
+static int
bfad_iocmd_ioc_set_name(struct bfad_s *bfad, void *cmd, unsigned int v_cmd)
{
struct bfa_bsg_ioc_name_s *iocmd = (struct bfa_bsg_ioc_name_s *) cmd;
@@ -208,7 +208,7 @@
return 0;
}
-int
+static int
bfad_iocmd_iocfc_get_attr(struct bfad_s *bfad, void *cmd)
{
struct bfa_bsg_iocfc_attr_s *iocmd = (struct bfa_bsg_iocfc_attr_s *)cmd;
@@ -219,7 +219,7 @@
return 0;
}
-int
+static int
bfad_iocmd_ioc_fw_sig_inv(struct bfad_s *bfad, void *cmd)
{
struct bfa_bsg_gen_s *iocmd = (struct bfa_bsg_gen_s *)cmd;
@@ -231,7 +231,7 @@
return 0;
}
-int
+static int
bfad_iocmd_iocfc_set_intr(struct bfad_s *bfad, void *cmd)
{
struct bfa_bsg_iocfc_intr_s *iocmd = (struct bfa_bsg_iocfc_intr_s *)cmd;
@@ -244,7 +244,7 @@
return 0;
}
-int
+static int
bfad_iocmd_port_enable(struct bfad_s *bfad, void *cmd)
{
struct bfa_bsg_gen_s *iocmd = (struct bfa_bsg_gen_s *)cmd;
@@ -265,7 +265,7 @@
return 0;
}
-int
+static int
bfad_iocmd_port_disable(struct bfad_s *bfad, void *cmd)
{
struct bfa_bsg_gen_s *iocmd = (struct bfa_bsg_gen_s *)cmd;
@@ -315,7 +315,7 @@
return 0;
}
-int
+static int
bfad_iocmd_port_get_stats(struct bfad_s *bfad, void *cmd,
unsigned int payload_len)
{
@@ -349,7 +349,7 @@
return 0;
}
-int
+static int
bfad_iocmd_port_reset_stats(struct bfad_s *bfad, void *cmd)
{
struct bfa_bsg_gen_s *iocmd = (struct bfa_bsg_gen_s *)cmd;
@@ -370,7 +370,7 @@
return 0;
}
-int
+static int
bfad_iocmd_set_port_cfg(struct bfad_s *bfad, void *iocmd, unsigned int v_cmd)
{
struct bfa_bsg_port_cfg_s *cmd = (struct bfa_bsg_port_cfg_s *)iocmd;
@@ -390,7 +390,7 @@
return 0;
}
-int
+static int
bfad_iocmd_port_cfg_maxfrsize(struct bfad_s *bfad, void *cmd)
{
struct bfa_bsg_port_cfg_maxfrsize_s *iocmd =
@@ -404,7 +404,7 @@
return 0;
}
-int
+static int
bfad_iocmd_port_cfg_bbcr(struct bfad_s *bfad, unsigned int cmd, void *pcmd)
{
struct bfa_bsg_bbcr_enable_s *iocmd =
@@ -427,7 +427,7 @@
return 0;
}
-int
+static int
bfad_iocmd_port_get_bbcr_attr(struct bfad_s *bfad, void *pcmd)
{
struct bfa_bsg_bbcr_attr_s *iocmd = (struct bfa_bsg_bbcr_attr_s *) pcmd;
@@ -465,7 +465,7 @@
return 0;
}
-int
+static int
bfad_iocmd_lport_get_stats(struct bfad_s *bfad, void *cmd)
{
struct bfa_fcs_lport_s *fcs_port;
@@ -489,7 +489,7 @@
return 0;
}
-int
+static int
bfad_iocmd_lport_reset_stats(struct bfad_s *bfad, void *cmd)
{
struct bfa_fcs_lport_s *fcs_port;
@@ -523,7 +523,7 @@
return 0;
}
-int
+static int
bfad_iocmd_lport_get_iostats(struct bfad_s *bfad, void *cmd)
{
struct bfa_fcs_lport_s *fcs_port;
@@ -548,7 +548,7 @@
return 0;
}
-int
+static int
bfad_iocmd_lport_get_rports(struct bfad_s *bfad, void *cmd,
unsigned int payload_len)
{
@@ -590,7 +590,7 @@
return 0;
}
-int
+static int
bfad_iocmd_rport_get_attr(struct bfad_s *bfad, void *cmd)
{
struct bfa_bsg_rport_attr_s *iocmd = (struct bfa_bsg_rport_attr_s *)cmd;
@@ -676,7 +676,7 @@
return 0;
}
-int
+static int
bfad_iocmd_rport_get_stats(struct bfad_s *bfad, void *cmd)
{
struct bfa_bsg_rport_stats_s *iocmd =
@@ -717,7 +717,7 @@
return 0;
}
-int
+static int
bfad_iocmd_rport_clr_stats(struct bfad_s *bfad, void *cmd)
{
struct bfa_bsg_rport_reset_stats_s *iocmd =
@@ -753,7 +753,7 @@
return 0;
}
-int
+static int
bfad_iocmd_rport_set_speed(struct bfad_s *bfad, void *cmd)
{
struct bfa_bsg_rport_set_speed_s *iocmd =
@@ -789,7 +789,7 @@
return 0;
}
-int
+static int
bfad_iocmd_vport_get_attr(struct bfad_s *bfad, void *cmd)
{
struct bfa_fcs_vport_s *fcs_vport;
@@ -812,7 +812,7 @@
return 0;
}
-int
+static int
bfad_iocmd_vport_get_stats(struct bfad_s *bfad, void *cmd)
{
struct bfa_fcs_vport_s *fcs_vport;
@@ -840,7 +840,7 @@
return 0;
}
-int
+static int
bfad_iocmd_vport_clr_stats(struct bfad_s *bfad, void *cmd)
{
struct bfa_fcs_vport_s *fcs_vport;
@@ -907,7 +907,7 @@
return 0;
}
-int
+static int
bfad_iocmd_qos_set_bw(struct bfad_s *bfad, void *pcmd)
{
struct bfa_bsg_qos_bw_s *iocmd = (struct bfa_bsg_qos_bw_s *)pcmd;
@@ -920,7 +920,7 @@
return 0;
}
-int
+static int
bfad_iocmd_ratelim(struct bfad_s *bfad, unsigned int cmd, void *pcmd)
{
struct bfa_bsg_gen_s *iocmd = (struct bfa_bsg_gen_s *)pcmd;
@@ -949,7 +949,7 @@
return 0;
}
-int
+static int
bfad_iocmd_ratelim_speed(struct bfad_s *bfad, unsigned int cmd, void *pcmd)
{
struct bfa_bsg_trl_speed_s *iocmd = (struct bfa_bsg_trl_speed_s *)pcmd;
@@ -978,7 +978,7 @@
return 0;
}
-int
+static int
bfad_iocmd_cfg_fcpim(struct bfad_s *bfad, void *cmd)
{
struct bfa_bsg_fcpim_s *iocmd = (struct bfa_bsg_fcpim_s *)cmd;
@@ -991,7 +991,7 @@
return 0;
}
-int
+static int
bfad_iocmd_fcpim_get_modstats(struct bfad_s *bfad, void *cmd)
{
struct bfa_bsg_fcpim_modstats_s *iocmd =
@@ -1013,7 +1013,7 @@
return 0;
}
-int
+static int
bfad_iocmd_fcpim_clr_modstats(struct bfad_s *bfad, void *cmd)
{
struct bfa_bsg_fcpim_modstatsclr_s *iocmd =
@@ -1035,7 +1035,7 @@
return 0;
}
-int
+static int
bfad_iocmd_fcpim_get_del_itn_stats(struct bfad_s *bfad, void *cmd)
{
struct bfa_bsg_fcpim_del_itn_stats_s *iocmd =
@@ -1160,7 +1160,7 @@
return 0;
}
-int
+static int
bfad_iocmd_fcport_enable(struct bfad_s *bfad, void *cmd)
{
struct bfa_bsg_gen_s *iocmd = (struct bfa_bsg_gen_s *)cmd;
@@ -1173,7 +1173,7 @@
return 0;
}
-int
+static int
bfad_iocmd_fcport_disable(struct bfad_s *bfad, void *cmd)
{
struct bfa_bsg_gen_s *iocmd = (struct bfa_bsg_gen_s *)cmd;
@@ -1186,7 +1186,7 @@
return 0;
}
-int
+static int
bfad_iocmd_ioc_get_pcifn_cfg(struct bfad_s *bfad, void *cmd)
{
struct bfa_bsg_pcifn_cfg_s *iocmd = (struct bfa_bsg_pcifn_cfg_s *)cmd;
@@ -1208,7 +1208,7 @@
return 0;
}
-int
+static int
bfad_iocmd_pcifn_create(struct bfad_s *bfad, void *cmd)
{
struct bfa_bsg_pcifn_s *iocmd = (struct bfa_bsg_pcifn_s *)cmd;
@@ -1231,7 +1231,7 @@
return 0;
}
-int
+static int
bfad_iocmd_pcifn_delete(struct bfad_s *bfad, void *cmd)
{
struct bfa_bsg_pcifn_s *iocmd = (struct bfa_bsg_pcifn_s *)cmd;
@@ -1253,7 +1253,7 @@
return 0;
}
-int
+static int
bfad_iocmd_pcifn_bw(struct bfad_s *bfad, void *cmd)
{
struct bfa_bsg_pcifn_s *iocmd = (struct bfa_bsg_pcifn_s *)cmd;
@@ -1277,7 +1277,7 @@
return 0;
}
-int
+static int
bfad_iocmd_adapter_cfg_mode(struct bfad_s *bfad, void *cmd)
{
struct bfa_bsg_adapter_cfg_mode_s *iocmd =
@@ -1300,7 +1300,7 @@
return 0;
}
-int
+static int
bfad_iocmd_port_cfg_mode(struct bfad_s *bfad, void *cmd)
{
struct bfa_bsg_port_cfg_mode_s *iocmd =
@@ -1324,7 +1324,7 @@
return 0;
}
-int
+static int
bfad_iocmd_ablk_optrom(struct bfad_s *bfad, unsigned int cmd, void *pcmd)
{
struct bfa_bsg_gen_s *iocmd = (struct bfa_bsg_gen_s *)pcmd;
@@ -1350,7 +1350,7 @@
return 0;
}
-int
+static int
bfad_iocmd_faa_query(struct bfad_s *bfad, void *cmd)
{
struct bfa_bsg_faa_attr_s *iocmd = (struct bfa_bsg_faa_attr_s *)cmd;
@@ -1373,7 +1373,7 @@
return 0;
}
-int
+static int
bfad_iocmd_cee_attr(struct bfad_s *bfad, void *cmd, unsigned int payload_len)
{
struct bfa_bsg_cee_attr_s *iocmd =
@@ -1409,7 +1409,7 @@
return 0;
}
-int
+static int
bfad_iocmd_cee_get_stats(struct bfad_s *bfad, void *cmd,
unsigned int payload_len)
{
@@ -1446,7 +1446,7 @@
return 0;
}
-int
+static int
bfad_iocmd_cee_reset_stats(struct bfad_s *bfad, void *cmd)
{
struct bfa_bsg_gen_s *iocmd = (struct bfa_bsg_gen_s *)cmd;
@@ -1460,7 +1460,7 @@
return 0;
}
-int
+static int
bfad_iocmd_sfp_media(struct bfad_s *bfad, void *cmd)
{
struct bfa_bsg_sfp_media_s *iocmd = (struct bfa_bsg_sfp_media_s *)cmd;
@@ -1482,7 +1482,7 @@
return 0;
}
-int
+static int
bfad_iocmd_sfp_speed(struct bfad_s *bfad, void *cmd)
{
struct bfa_bsg_sfp_speed_s *iocmd = (struct bfa_bsg_sfp_speed_s *)cmd;
@@ -1503,7 +1503,7 @@
return 0;
}
-int
+static int
bfad_iocmd_flash_get_attr(struct bfad_s *bfad, void *cmd)
{
struct bfa_bsg_flash_attr_s *iocmd =
@@ -1524,7 +1524,7 @@
return 0;
}
-int
+static int
bfad_iocmd_flash_erase_part(struct bfad_s *bfad, void *cmd)
{
struct bfa_bsg_flash_s *iocmd = (struct bfa_bsg_flash_s *)cmd;
@@ -1544,7 +1544,7 @@
return 0;
}
-int
+static int
bfad_iocmd_flash_update_part(struct bfad_s *bfad, void *cmd,
unsigned int payload_len)
{
@@ -1576,7 +1576,7 @@
return 0;
}
-int
+static int
bfad_iocmd_flash_read_part(struct bfad_s *bfad, void *cmd,
unsigned int payload_len)
{
@@ -1608,7 +1608,7 @@
return 0;
}
-int
+static int
bfad_iocmd_diag_temp(struct bfad_s *bfad, void *cmd)
{
struct bfa_bsg_diag_get_temp_s *iocmd =
@@ -1630,7 +1630,7 @@
return 0;
}
-int
+static int
bfad_iocmd_diag_memtest(struct bfad_s *bfad, void *cmd)
{
struct bfa_bsg_diag_memtest_s *iocmd =
@@ -1653,7 +1653,7 @@
return 0;
}
-int
+static int
bfad_iocmd_diag_loopback(struct bfad_s *bfad, void *cmd)
{
struct bfa_bsg_diag_loopback_s *iocmd =
@@ -1676,7 +1676,7 @@
return 0;
}
-int
+static int
bfad_iocmd_diag_fwping(struct bfad_s *bfad, void *cmd)
{
struct bfa_bsg_diag_fwping_s *iocmd =
@@ -1700,7 +1700,7 @@
return 0;
}
-int
+static int
bfad_iocmd_diag_queuetest(struct bfad_s *bfad, void *cmd)
{
struct bfa_bsg_diag_qtest_s *iocmd = (struct bfa_bsg_diag_qtest_s *)cmd;
@@ -1721,7 +1721,7 @@
return 0;
}
-int
+static int
bfad_iocmd_diag_sfp(struct bfad_s *bfad, void *cmd)
{
struct bfa_bsg_sfp_show_s *iocmd =
@@ -1744,7 +1744,7 @@
return 0;
}
-int
+static int
bfad_iocmd_diag_led(struct bfad_s *bfad, void *cmd)
{
struct bfa_bsg_diag_led_s *iocmd = (struct bfa_bsg_diag_led_s *)cmd;
@@ -1757,7 +1757,7 @@
return 0;
}
-int
+static int
bfad_iocmd_diag_beacon_lport(struct bfad_s *bfad, void *cmd)
{
struct bfa_bsg_diag_beacon_s *iocmd =
@@ -1772,7 +1772,7 @@
return 0;
}
-int
+static int
bfad_iocmd_diag_lb_stat(struct bfad_s *bfad, void *cmd)
{
struct bfa_bsg_diag_lb_stat_s *iocmd =
@@ -1787,7 +1787,7 @@
return 0;
}
-int
+static int
bfad_iocmd_diag_dport_enable(struct bfad_s *bfad, void *pcmd)
{
struct bfa_bsg_dport_enable_s *iocmd =
@@ -1809,7 +1809,7 @@
return 0;
}
-int
+static int
bfad_iocmd_diag_dport_disable(struct bfad_s *bfad, void *pcmd)
{
struct bfa_bsg_gen_s *iocmd = (struct bfa_bsg_gen_s *)pcmd;
@@ -1829,7 +1829,7 @@
return 0;
}
-int
+static int
bfad_iocmd_diag_dport_start(struct bfad_s *bfad, void *pcmd)
{
struct bfa_bsg_dport_enable_s *iocmd =
@@ -1854,7 +1854,7 @@
return 0;
}
-int
+static int
bfad_iocmd_diag_dport_show(struct bfad_s *bfad, void *pcmd)
{
struct bfa_bsg_diag_dport_show_s *iocmd =
@@ -1869,7 +1869,7 @@
}
-int
+static int
bfad_iocmd_phy_get_attr(struct bfad_s *bfad, void *cmd)
{
struct bfa_bsg_phy_attr_s *iocmd =
@@ -1890,7 +1890,7 @@
return 0;
}
-int
+static int
bfad_iocmd_phy_get_stats(struct bfad_s *bfad, void *cmd)
{
struct bfa_bsg_phy_stats_s *iocmd =
@@ -1911,7 +1911,7 @@
return 0;
}
-int
+static int
bfad_iocmd_phy_read(struct bfad_s *bfad, void *cmd, unsigned int payload_len)
{
struct bfa_bsg_phy_s *iocmd = (struct bfa_bsg_phy_s *)cmd;
@@ -1943,7 +1943,7 @@
return 0;
}
-int
+static int
bfad_iocmd_vhba_query(struct bfad_s *bfad, void *cmd)
{
struct bfa_bsg_vhba_attr_s *iocmd =
@@ -1962,7 +1962,7 @@
return 0;
}
-int
+static int
bfad_iocmd_phy_update(struct bfad_s *bfad, void *cmd, unsigned int payload_len)
{
struct bfa_bsg_phy_s *iocmd = (struct bfa_bsg_phy_s *)cmd;
@@ -1992,7 +1992,7 @@
return 0;
}
-int
+static int
bfad_iocmd_porglog_get(struct bfad_s *bfad, void *cmd)
{
struct bfa_bsg_debug_s *iocmd = (struct bfa_bsg_debug_s *)cmd;
@@ -2012,7 +2012,7 @@
}
#define BFA_DEBUG_FW_CORE_CHUNK_SZ 0x4000U /* 16K chunks for FW dump */
-int
+static int
bfad_iocmd_debug_fw_core(struct bfad_s *bfad, void *cmd,
unsigned int payload_len)
{
@@ -2046,7 +2046,7 @@
return 0;
}
-int
+static int
bfad_iocmd_debug_ctl(struct bfad_s *bfad, void *cmd, unsigned int v_cmd)
{
struct bfa_bsg_gen_s *iocmd = (struct bfa_bsg_gen_s *)cmd;
@@ -2067,7 +2067,7 @@
return 0;
}
-int
+static int
bfad_iocmd_porglog_ctl(struct bfad_s *bfad, void *cmd)
{
struct bfa_bsg_portlogctl_s *iocmd = (struct bfa_bsg_portlogctl_s *)cmd;
@@ -2081,7 +2081,7 @@
return 0;
}
-int
+static int
bfad_iocmd_fcpim_cfg_profile(struct bfad_s *bfad, void *cmd, unsigned int v_cmd)
{
struct bfa_bsg_fcpim_profile_s *iocmd =
@@ -2125,7 +2125,7 @@
return 0;
}
-int
+static int
bfad_iocmd_fcport_get_stats(struct bfad_s *bfad, void *cmd)
{
struct bfa_bsg_fcport_stats_s *iocmd =
@@ -2150,7 +2150,7 @@
return 0;
}
-int
+static int
bfad_iocmd_fcport_reset_stats(struct bfad_s *bfad, void *cmd)
{
struct bfa_bsg_gen_s *iocmd = (struct bfa_bsg_gen_s *)cmd;
@@ -2174,7 +2174,7 @@
return 0;
}
-int
+static int
bfad_iocmd_boot_cfg(struct bfad_s *bfad, void *cmd)
{
struct bfa_bsg_boot_s *iocmd = (struct bfa_bsg_boot_s *)cmd;
@@ -2196,7 +2196,7 @@
return 0;
}
-int
+static int
bfad_iocmd_boot_query(struct bfad_s *bfad, void *cmd)
{
struct bfa_bsg_boot_s *iocmd = (struct bfa_bsg_boot_s *)cmd;
@@ -2218,7 +2218,7 @@
return 0;
}
-int
+static int
bfad_iocmd_preboot_query(struct bfad_s *bfad, void *cmd)
{
struct bfa_bsg_preboot_s *iocmd = (struct bfa_bsg_preboot_s *)cmd;
@@ -2237,7 +2237,7 @@
return 0;
}
-int
+static int
bfad_iocmd_ethboot_cfg(struct bfad_s *bfad, void *cmd)
{
struct bfa_bsg_ethboot_s *iocmd = (struct bfa_bsg_ethboot_s *)cmd;
@@ -2260,7 +2260,7 @@
return 0;
}
-int
+static int
bfad_iocmd_ethboot_query(struct bfad_s *bfad, void *cmd)
{
struct bfa_bsg_ethboot_s *iocmd = (struct bfa_bsg_ethboot_s *)cmd;
@@ -2283,7 +2283,7 @@
return 0;
}
-int
+static int
bfad_iocmd_cfg_trunk(struct bfad_s *bfad, void *cmd, unsigned int v_cmd)
{
struct bfa_bsg_gen_s *iocmd = (struct bfa_bsg_gen_s *)cmd;
@@ -2323,7 +2323,7 @@
return 0;
}
-int
+static int
bfad_iocmd_trunk_get_attr(struct bfad_s *bfad, void *cmd)
{
struct bfa_bsg_trunk_attr_s *iocmd = (struct bfa_bsg_trunk_attr_s *)cmd;
@@ -2346,7 +2346,7 @@
return 0;
}
-int
+static int
bfad_iocmd_qos(struct bfad_s *bfad, void *cmd, unsigned int v_cmd)
{
struct bfa_bsg_gen_s *iocmd = (struct bfa_bsg_gen_s *)cmd;
@@ -2374,7 +2374,7 @@
return 0;
}
-int
+static int
bfad_iocmd_qos_get_attr(struct bfad_s *bfad, void *cmd)
{
struct bfa_bsg_qos_attr_s *iocmd = (struct bfa_bsg_qos_attr_s *)cmd;
@@ -2400,7 +2400,7 @@
return 0;
}
-int
+static int
bfad_iocmd_qos_get_vc_attr(struct bfad_s *bfad, void *cmd)
{
struct bfa_bsg_qos_vc_attr_s *iocmd =
@@ -2432,7 +2432,7 @@
return 0;
}
-int
+static int
bfad_iocmd_qos_get_stats(struct bfad_s *bfad, void *cmd)
{
struct bfa_bsg_fcport_stats_s *iocmd =
@@ -2464,7 +2464,7 @@
return 0;
}
-int
+static int
bfad_iocmd_qos_reset_stats(struct bfad_s *bfad, void *cmd)
{
struct bfa_bsg_gen_s *iocmd = (struct bfa_bsg_gen_s *)cmd;
@@ -2495,7 +2495,7 @@
return 0;
}
-int
+static int
bfad_iocmd_vf_get_stats(struct bfad_s *bfad, void *cmd)
{
struct bfa_bsg_vf_stats_s *iocmd =
@@ -2518,7 +2518,7 @@
return 0;
}
-int
+static int
bfad_iocmd_vf_clr_stats(struct bfad_s *bfad, void *cmd)
{
struct bfa_bsg_vf_reset_stats_s *iocmd =
@@ -2555,7 +2555,7 @@
bfad_reset_sdev_bflags(vport->drv_port.im_port, lunmask_cfg);
}
-int
+static int
bfad_iocmd_lunmask(struct bfad_s *bfad, void *pcmd, unsigned int v_cmd)
{
struct bfa_bsg_gen_s *iocmd = (struct bfa_bsg_gen_s *)pcmd;
@@ -2578,7 +2578,7 @@
return 0;
}
-int
+static int
bfad_iocmd_fcpim_lunmask_query(struct bfad_s *bfad, void *cmd)
{
struct bfa_bsg_fcpim_lunmask_query_s *iocmd =
@@ -2592,7 +2592,7 @@
return 0;
}
-int
+static int
bfad_iocmd_fcpim_cfg_lunmask(struct bfad_s *bfad, void *cmd, unsigned int v_cmd)
{
struct bfa_bsg_fcpim_lunmask_s *iocmd =
@@ -2611,7 +2611,7 @@
return 0;
}
-int
+static int
bfad_iocmd_fcpim_throttle_query(struct bfad_s *bfad, void *cmd)
{
struct bfa_bsg_fcpim_throttle_s *iocmd =
@@ -2626,7 +2626,7 @@
return 0;
}
-int
+static int
bfad_iocmd_fcpim_throttle_set(struct bfad_s *bfad, void *cmd)
{
struct bfa_bsg_fcpim_throttle_s *iocmd =
@@ -2641,7 +2641,7 @@
return 0;
}
-int
+static int
bfad_iocmd_tfru_read(struct bfad_s *bfad, void *cmd)
{
struct bfa_bsg_tfru_s *iocmd =
@@ -2663,7 +2663,7 @@
return 0;
}
-int
+static int
bfad_iocmd_tfru_write(struct bfad_s *bfad, void *cmd)
{
struct bfa_bsg_tfru_s *iocmd =
@@ -2685,7 +2685,7 @@
return 0;
}
-int
+static int
bfad_iocmd_fruvpd_read(struct bfad_s *bfad, void *cmd)
{
struct bfa_bsg_fruvpd_s *iocmd =
@@ -2707,7 +2707,7 @@
return 0;
}
-int
+static int
bfad_iocmd_fruvpd_update(struct bfad_s *bfad, void *cmd)
{
struct bfa_bsg_fruvpd_s *iocmd =
@@ -2729,7 +2729,7 @@
return 0;
}
-int
+static int
bfad_iocmd_fruvpd_get_max_size(struct bfad_s *bfad, void *cmd)
{
struct bfa_bsg_fruvpd_max_size_s *iocmd =
@@ -3177,7 +3177,7 @@
}
/* FC passthru call backs */
-u64
+static u64
bfad_fcxp_get_req_sgaddr_cb(void *bfad_fcxp, int sgeid)
{
struct bfad_fcxp *drv_fcxp = bfad_fcxp;
@@ -3189,7 +3189,7 @@
return addr;
}
-u32
+static u32
bfad_fcxp_get_req_sglen_cb(void *bfad_fcxp, int sgeid)
{
struct bfad_fcxp *drv_fcxp = bfad_fcxp;
@@ -3199,7 +3199,7 @@
return sge->sg_len;
}
-u64
+static u64
bfad_fcxp_get_rsp_sgaddr_cb(void *bfad_fcxp, int sgeid)
{
struct bfad_fcxp *drv_fcxp = bfad_fcxp;
@@ -3211,7 +3211,7 @@
return addr;
}
-u32
+static u32
bfad_fcxp_get_rsp_sglen_cb(void *bfad_fcxp, int sgeid)
{
struct bfad_fcxp *drv_fcxp = bfad_fcxp;
@@ -3221,7 +3221,7 @@
return sge->sg_len;
}
-void
+static void
bfad_send_fcpt_cb(void *bfad_fcxp, struct bfa_fcxp_s *fcxp, void *cbarg,
bfa_status_t req_status, u32 rsp_len, u32 resid_len,
struct fchs_s *rsp_fchs)
@@ -3236,7 +3236,7 @@
complete(&drv_fcxp->comp);
}
-struct bfad_buf_info *
+static struct bfad_buf_info *
bfad_fcxp_map_sg(struct bfad_s *bfad, void *payload_kbuf,
uint32_t payload_len, uint32_t *num_sgles)
{
@@ -3280,7 +3280,7 @@
return NULL;
}
-void
+static void
bfad_fcxp_free_mem(struct bfad_s *bfad, struct bfad_buf_info *buf_base,
uint32_t num_sgles)
{
@@ -3298,7 +3298,7 @@
}
}
-int
+static int
bfad_fcxp_bsg_send(struct bsg_job *job, struct bfad_fcxp *drv_fcxp,
bfa_bsg_fcpt_t *bsg_fcpt)
{
@@ -3338,7 +3338,7 @@
return BFA_STATUS_OK;
}
-int
+static int
bfad_im_bsg_els_ct_request(struct bsg_job *job)
{
struct bfa_bsg_data *bsg_data;
diff --git a/drivers/scsi/bnx2fc/bnx2fc_fcoe.c b/drivers/scsi/bnx2fc/bnx2fc_fcoe.c
index 0e33324..5cdeeb3 100644
--- a/drivers/scsi/bnx2fc/bnx2fc_fcoe.c
+++ b/drivers/scsi/bnx2fc/bnx2fc_fcoe.c
@@ -1071,9 +1071,8 @@
/**
* bnx2fc_update_src_mac - Update Ethernet MAC filters.
*
- * @fip: FCoE controller.
- * @old: Unicast MAC address to delete if the MAC is non-zero.
- * @new: Unicast MAC address to add.
+ * @lport: The local port
+ * @addr: Location of data to copy
*
* Remove any previously-set unicast MAC filter.
* Add secondary FCoE MAC address filter for our OUI.
@@ -1659,8 +1658,7 @@
/**
* bnx2fc_destroy - Destroy a bnx2fc FCoE interface
*
- * @buffer: The name of the Ethernet interface to be destroyed
- * @kp: The associated kernel parameter
+ * @netdev: The net device that the FCoE interface is on
*
* Called from sysfs.
*
@@ -2101,7 +2099,7 @@
return 0;
}
-/**
+/*
* Deperecated: Use bnx2fc_enabled()
*/
static int bnx2fc_disable(struct net_device *netdev)
@@ -2229,7 +2227,7 @@
return 0;
}
-/**
+/*
* Deprecated: Use bnx2fc_enabled()
*/
static int bnx2fc_enable(struct net_device *netdev)
@@ -2523,7 +2521,7 @@
/**
* bnx2fc_ulp_exit - shuts down adapter instance and frees all resources
*
- * @dev cnic device handle
+ * @dev: cnic device handle
*/
static void bnx2fc_ulp_exit(struct cnic_dev *dev)
{
@@ -2956,7 +2954,7 @@
NULL,
};
-/**
+/*
* scsi_host_template structure used while registering with SCSI-ml
*/
static struct scsi_host_template bnx2fc_shost_template = {
@@ -2989,7 +2987,7 @@
.rport_event_callback = bnx2fc_rport_event_handler,
};
-/**
+/*
* bnx2fc_cnic_cb - global template of bnx2fc - cnic driver interface
* structure carrying callback function pointers
*/
diff --git a/drivers/scsi/bnx2fc/bnx2fc_hwi.c b/drivers/scsi/bnx2fc/bnx2fc_hwi.c
index 1f7c58b..e72d7bb 100644
--- a/drivers/scsi/bnx2fc/bnx2fc_hwi.c
+++ b/drivers/scsi/bnx2fc/bnx2fc_hwi.c
@@ -485,7 +485,7 @@
/**
* bnx2fc_send_session_destroy_req - initiates FCoE Session destroy
*
- * @port: port structure pointer
+ * @hba: adapter structure pointer
* @tgt: bnx2fc_rport structure pointer
*/
int bnx2fc_send_session_destroy_req(struct bnx2fc_hba *hba,
@@ -635,7 +635,6 @@
struct bnx2fc_cmd *io_req = NULL;
struct bnx2fc_interface *interface = tgt->port->priv;
struct bnx2fc_hba *hba = interface->hba;
- int task_idx, index;
int rc = 0;
u64 err_warn_bit_map;
u8 err_warn = 0xff;
@@ -701,15 +700,12 @@
BNX2FC_TGT_DBG(tgt, "buf_offsets - tx = 0x%x, rx = 0x%x\n",
err_entry->data.tx_buf_off, err_entry->data.rx_buf_off);
-
if (xid > hba->max_xid) {
BNX2FC_TGT_DBG(tgt, "xid(0x%x) out of FW range\n",
xid);
goto ret_err_rqe;
}
- task_idx = xid / BNX2FC_TASKS_PER_PAGE;
- index = xid % BNX2FC_TASKS_PER_PAGE;
io_req = (struct bnx2fc_cmd *)hba->cmd_mgr->cmds[xid];
if (!io_req)
@@ -833,8 +829,6 @@
}
BNX2FC_TGT_DBG(tgt, "warn = 0x%x\n", err_warn);
- task_idx = xid / BNX2FC_TASKS_PER_PAGE;
- index = xid % BNX2FC_TASKS_PER_PAGE;
io_req = (struct bnx2fc_cmd *)hba->cmd_mgr->cmds[xid];
if (!io_req)
goto ret_warn_rqe;
@@ -1008,7 +1002,6 @@
unsigned char *rq_data = NULL;
unsigned char rq_data_buff[BNX2FC_RQ_BUF_SZ];
int task_idx, index;
- unsigned char *dummy;
u16 xid;
u8 num_rq;
int i;
@@ -1038,7 +1031,7 @@
if (num_rq > 1) {
/* We do not need extra sense data */
for (i = 1; i < num_rq; i++)
- dummy = bnx2fc_get_next_rqe(tgt, 1);
+ bnx2fc_get_next_rqe(tgt, 1);
}
if (rq_data)
@@ -1341,8 +1334,8 @@
/**
* bnx2fc_indicae_kcqe - process KCQE
*
- * @hba: adapter structure pointer
- * @kcqe: kcqe pointer
+ * @context: adapter structure pointer
+ * @kcq: kcqe pointer
* @num_cqe: Number of completion queue elements
*
* Generic KCQ event handler
@@ -1510,7 +1503,6 @@
u64 phys_addr = (u64)orig_io_req->bd_tbl->bd_tbl_dma;
u32 orig_offset = offset;
int bd_count;
- int orig_task_idx, index;
int i;
memset(task, 0, sizeof(struct fcoe_task_ctx_entry));
@@ -1560,8 +1552,6 @@
offset; /* adjusted offset */
task->txwr_only.sgl_ctx.sgl.mul_sgl.cur_sge_idx = i;
} else {
- orig_task_idx = orig_xid / BNX2FC_TASKS_PER_PAGE;
- index = orig_xid % BNX2FC_TASKS_PER_PAGE;
/* Multiple SGEs were used for this IO */
sgl = &task->rxwr_only.union_ctx.read_info.sgl_ctx.sgl;
@@ -2089,11 +2079,7 @@
pbl = hba->hash_tbl_pbl;
i = 0;
while (*pbl && *(pbl + 1)) {
- u32 lo;
- u32 hi;
- lo = *pbl;
++pbl;
- hi = *pbl;
++pbl;
++i;
}
diff --git a/drivers/scsi/bnx2fc/bnx2fc_tgt.c b/drivers/scsi/bnx2fc/bnx2fc_tgt.c
index 50384b4..a3e2a38 100644
--- a/drivers/scsi/bnx2fc/bnx2fc_tgt.c
+++ b/drivers/scsi/bnx2fc/bnx2fc_tgt.c
@@ -431,7 +431,7 @@
return 0;
}
-/**
+/*
* This event_callback is called after successful completion of libfc
* initiated target login. bnx2fc can proceed with initiating the session
* establishment.
@@ -656,9 +656,8 @@
spin_unlock_bh(&hba->hba_lock);
}
-/**
- *bnx2fc_alloc_session_resc - Allocate qp resources for the session
- *
+/*
+ * bnx2fc_alloc_session_resc - Allocate qp resources for the session
*/
static int bnx2fc_alloc_session_resc(struct bnx2fc_hba *hba,
struct bnx2fc_rport *tgt)
diff --git a/drivers/scsi/bnx2i/bnx2i_hwi.c b/drivers/scsi/bnx2i/bnx2i_hwi.c
index e53ebc5..bad396e 100644
--- a/drivers/scsi/bnx2i/bnx2i_hwi.c
+++ b/drivers/scsi/bnx2i/bnx2i_hwi.c
@@ -181,7 +181,7 @@
/**
* bnx2i_get_rq_buf - copy RQ buffer contents to driver buffer
- * @conn: iscsi connection on which RQ event occurred
+ * @bnx2i_conn: iscsi connection on which RQ event occurred
* @ptr: driver buffer to which RQ buffer contents is to
* be copied
* @len: length of valid data inside RQ buf
@@ -223,7 +223,7 @@
/**
* bnx2i_put_rq_buf - Replenish RQ buffer, if required ring on chip doorbell
- * @conn: iscsi connection on which event to post
+ * @bnx2i_conn: iscsi connection on which event to post
* @count: number of RQ buffer being posted to chip
*
* No need to ring hardware doorbell for 57710 family of devices
@@ -258,7 +258,7 @@
/**
* bnx2i_ring_sq_dbell - Ring SQ doorbell to wake-up the processing engine
- * @conn: iscsi connection to which new SQ entries belong
+ * @bnx2i_conn: iscsi connection to which new SQ entries belong
* @count: number of SQ WQEs to post
*
* SQ DB is updated in host memory and TX Doorbell is rung for 57710 family
@@ -283,7 +283,7 @@
/**
* bnx2i_ring_dbell_update_sq_params - update SQ driver parameters
- * @conn: iscsi connection to which new SQ entries belong
+ * @bnx2i_conn: iscsi connection to which new SQ entries belong
* @count: number of SQ WQEs to post
*
* this routine will update SQ driver parameters and ring the doorbell
@@ -320,9 +320,9 @@
/**
* bnx2i_send_iscsi_login - post iSCSI login request MP WQE to hardware
- * @conn: iscsi connection
- * @cmd: driver command structure which is requesting
- * a WQE to sent to chip for further processing
+ * @bnx2i_conn: iscsi connection
+ * @task: transport layer's command structure pointer which is requesting
+ * a WQE to sent to chip for further processing
*
* prepare and post an iSCSI Login request WQE to CNIC firmware
*/
@@ -373,7 +373,7 @@
/**
* bnx2i_send_iscsi_tmf - post iSCSI task management request MP WQE to hardware
- * @conn: iscsi connection
+ * @bnx2i_conn: iscsi connection
* @mtask: driver command structure which is requesting
* a WQE to sent to chip for further processing
*
@@ -447,7 +447,7 @@
/**
* bnx2i_send_iscsi_text - post iSCSI text WQE to hardware
- * @conn: iscsi connection
+ * @bnx2i_conn: iscsi connection
* @mtask: driver command structure which is requesting
* a WQE to sent to chip for further processing
*
@@ -495,7 +495,7 @@
/**
* bnx2i_send_iscsi_scsicmd - post iSCSI scsicmd request WQE to hardware
- * @conn: iscsi connection
+ * @bnx2i_conn: iscsi connection
* @cmd: driver command structure which is requesting
* a WQE to sent to chip for further processing
*
@@ -517,9 +517,9 @@
/**
* bnx2i_send_iscsi_nopout - post iSCSI NOPOUT request WQE to hardware
- * @conn: iscsi connection
- * @cmd: driver command structure which is requesting
- * a WQE to sent to chip for further processing
+ * @bnx2i_conn: iscsi connection
+ * @task: transport layer's command structure pointer which is
+ * requesting a WQE to sent to chip for further processing
* @datap: payload buffer pointer
* @data_len: payload data length
* @unsol: indicated whether nopout pdu is unsolicited pdu or
@@ -579,9 +579,9 @@
/**
* bnx2i_send_iscsi_logout - post iSCSI logout request WQE to hardware
- * @conn: iscsi connection
- * @cmd: driver command structure which is requesting
- * a WQE to sent to chip for further processing
+ * @bnx2i_conn: iscsi connection
+ * @task: transport layer's command structure pointer which is
+ * requesting a WQE to sent to chip for further processing
*
* prepare and post logout request WQE to CNIC firmware
*/
@@ -678,7 +678,8 @@
/**
* bnx2i_ep_ofld_timer - post iSCSI logout request WQE to hardware
- * @data: endpoint (transport handle) structure pointer
+ * @t: timer context used to fetch the endpoint (transport
+ * handle) structure pointer
*
* routine to handle connection offload/destroy request timeout
*/
@@ -1662,7 +1663,7 @@
/**
* bnx2i_unsol_pdu_adjust_rq - makes adjustments to RQ after unsol pdu is recvd
- * @conn: iscsi connection
+ * @bnx2i_conn: iscsi connection
*
* Firmware advances RQ producer index for every unsolicited PDU even if
* payload data length is '0'. This function makes corresponding
@@ -1885,7 +1886,9 @@
/**
* bnx2i_queue_scsi_cmd_resp - queue cmd completion to the percpu thread
+ * @session: iscsi session
* @bnx2i_conn: bnx2i connection
+ * @cqe: pointer to newly DMA'ed CQE entry for processing
*
* this function is called by generic KCQ handler to queue all pending cmd
* completion CQEs
@@ -2466,8 +2469,9 @@
/**
* bnx2i_indicate_kcqe - process iscsi conn update completion KCQE
- * @hba: adapter structure pointer
- * @update_kcqe: kcqe pointer
+ * @context: adapter structure pointer
+ * @kcqe: kcqe pointer
+ * @num_cqe: number of kcqes to process
*
* Generic KCQ event handler/dispatcher
*/
@@ -2614,8 +2618,7 @@
/**
* bnx2i_cm_remote_close - process received TCP FIN
- * @hba: adapter structure pointer
- * @update_kcqe: kcqe pointer
+ * @cm_sk: cnic sock structure pointer
*
* function callback exported via bnx2i - cnic driver interface to indicate
* async TCP events such as FIN
@@ -2631,8 +2634,7 @@
/**
* bnx2i_cm_remote_abort - process TCP RST and start conn cleanup
- * @hba: adapter structure pointer
- * @update_kcqe: kcqe pointer
+ * @cm_sk: cnic sock structure pointer
*
* function callback exported via bnx2i - cnic driver interface to
* indicate async TCP events (RST) sent by the peer.
@@ -2669,10 +2671,9 @@
}
-/**
+/*
* bnx2i_cnic_cb - global template of bnx2i - cnic driver interface structure
* carrying callback function pointers
- *
*/
struct cnic_ulp_ops bnx2i_cnic_cb = {
.cnic_init = bnx2i_ulp_init,
diff --git a/drivers/scsi/bnx2i/bnx2i_init.c b/drivers/scsi/bnx2i/bnx2i_init.c
index 4ebcda8..6018cdd 100644
--- a/drivers/scsi/bnx2i/bnx2i_init.c
+++ b/drivers/scsi/bnx2i/bnx2i_init.c
@@ -73,7 +73,7 @@
/**
* bnx2i_identify_device - identifies NetXtreme II device type
* @hba: Adapter structure pointer
- * @cnic: Corresponding cnic device
+ * @dev: Corresponding cnic device
*
* This function identifies the NX2 device type and sets appropriate
* queue mailbox register access method, 5709 requires driver to
diff --git a/drivers/scsi/bnx2i/bnx2i_iscsi.c b/drivers/scsi/bnx2i/bnx2i_iscsi.c
index 0b28d44..fdd4467 100644
--- a/drivers/scsi/bnx2i/bnx2i_iscsi.c
+++ b/drivers/scsi/bnx2i/bnx2i_iscsi.c
@@ -228,7 +228,7 @@
/**
* bnx2i_bind_conn_to_iscsi_cid - bind conn structure to 'iscsi_cid'
* @hba: pointer to adapter instance
- * @conn: pointer to iscsi connection
+ * @bnx2i_conn: pointer to iscsi connection
* @iscsi_cid: iscsi context ID, range 0 - (MAX_CONN - 1)
*
* update iscsi cid table entry with connection pointer. This enables
@@ -463,7 +463,6 @@
* bnx2i_destroy_cmd_pool - destroys iscsi command pool and release BD table
* @hba: adapter instance pointer
* @session: iscsi session pointer
- * @cmd: iscsi command structure
*/
static void bnx2i_destroy_cmd_pool(struct bnx2i_hba *hba,
struct iscsi_session *session)
@@ -582,8 +581,7 @@
/**
* bnx2i_drop_session - notifies iscsid of connection error.
- * @hba: adapter instance pointer
- * @session: iscsi session pointer
+ * @cls_session: iscsi cls session pointer
*
* This notifies iscsid that there is a error, so it can initiate
* recovery.
@@ -1277,7 +1275,8 @@
/**
* bnx2i_session_create - create a new iscsi session
- * @cmds_max: max commands supported
+ * @ep: pointer to iscsi endpoint
+ * @cmds_max: user specified maximum commands
* @qdepth: scsi queue depth to support
* @initial_cmdsn: initial iscsi CMDSN to be used for this session
*
@@ -1971,7 +1970,7 @@
/**
* bnx2i_ep_tcp_conn_active - check EP state transition
- * @ep: endpoint pointer
+ * @bnx2i_ep: endpoint pointer
*
* check if underlying TCP connection is active
*/
@@ -2014,9 +2013,9 @@
}
-/*
+/**
* bnx2i_hw_ep_disconnect - executes TCP connection teardown process in the hw
- * @ep: TCP connection (bnx2i endpoint) handle
+ * @bnx2i_ep: TCP connection (bnx2i endpoint) handle
*
* executes TCP connection teardown process
*/
@@ -2171,8 +2170,8 @@
/**
* bnx2i_nl_set_path - ISCSI_UEVENT_PATH_UPDATE user message handler
- * @buf: pointer to buffer containing iscsi path message
- *
+ * @shost: scsi host pointer
+ * @params: pointer to buffer containing iscsi path message
*/
static int bnx2i_nl_set_path(struct Scsi_Host *shost, struct iscsi_path *params)
{
diff --git a/drivers/scsi/bnx2i/bnx2i_sysfs.c b/drivers/scsi/bnx2i/bnx2i_sysfs.c
index 6d56fd6..3dc7900 100644
--- a/drivers/scsi/bnx2i/bnx2i_sysfs.c
+++ b/drivers/scsi/bnx2i/bnx2i_sysfs.c
@@ -30,6 +30,7 @@
/**
* bnx2i_show_sq_info - return(s currently configured send queue (SQ) size
* @dev: device pointer
+ * @attr: device attribute (unused)
* @buf: buffer to return current SQ size parameter
*
* Returns current SQ size parameter, this paramater determines the number
@@ -47,6 +48,7 @@
/**
* bnx2i_set_sq_info - update send queue (SQ) size parameter
* @dev: device pointer
+ * @attr: device attribute (unused)
* @buf: buffer to return current SQ size parameter
* @count: parameter buffer size
*
@@ -87,6 +89,7 @@
/**
* bnx2i_show_ccell_info - returns command cell (HQ) size
* @dev: device pointer
+ * @attr: device attribute (unused)
* @buf: buffer to return current SQ size parameter
*
* returns per-connection TCP history queue size parameter
@@ -103,6 +106,7 @@
/**
* bnx2i_get_link_state - set command cell (HQ) size
* @dev: device pointer
+ * @attr: device attribute (unused)
* @buf: buffer to return current SQ size parameter
* @count: parameter buffer size
*
diff --git a/drivers/scsi/csiostor/csio_hw.c b/drivers/scsi/csiostor/csio_hw.c
index 950f9cd..98d4d39 100644
--- a/drivers/scsi/csiostor/csio_hw.c
+++ b/drivers/scsi/csiostor/csio_hw.c
@@ -306,7 +306,7 @@
uint8_t *vpd, csum;
const struct t4_vpd_hdr *v;
/* To get around compilation warning from strstrip */
- char *s;
+ char __always_unused *s;
if (csio_is_valid_vpd(hw))
return 0;
diff --git a/drivers/scsi/csiostor/csio_hw_t5.c b/drivers/scsi/csiostor/csio_hw_t5.c
index f24def6..1df8891 100644
--- a/drivers/scsi/csiostor/csio_hw_t5.c
+++ b/drivers/scsi/csiostor/csio_hw_t5.c
@@ -148,12 +148,11 @@
{
int i;
uint32_t mc_bist_cmd_reg, mc_bist_cmd_addr_reg, mc_bist_cmd_len_reg;
- uint32_t mc_bist_status_rdata_reg, mc_bist_data_pattern_reg;
+ uint32_t mc_bist_data_pattern_reg;
mc_bist_cmd_reg = MC_REG(MC_P_BIST_CMD_A, idx);
mc_bist_cmd_addr_reg = MC_REG(MC_P_BIST_CMD_ADDR_A, idx);
mc_bist_cmd_len_reg = MC_REG(MC_P_BIST_CMD_LEN_A, idx);
- mc_bist_status_rdata_reg = MC_REG(MC_P_BIST_STATUS_RDATA_A, idx);
mc_bist_data_pattern_reg = MC_REG(MC_P_BIST_DATA_PATTERN_A, idx);
if (csio_rd_reg32(hw, mc_bist_cmd_reg) & START_BIST_F)
@@ -196,7 +195,7 @@
{
int i;
uint32_t edc_bist_cmd_reg, edc_bist_cmd_addr_reg, edc_bist_cmd_len_reg;
- uint32_t edc_bist_cmd_data_pattern, edc_bist_status_rdata_reg;
+ uint32_t edc_bist_cmd_data_pattern;
/*
* These macro are missing in t4_regs.h file.
@@ -208,7 +207,6 @@
edc_bist_cmd_addr_reg = EDC_REG_T5(EDC_H_BIST_CMD_ADDR_A, idx);
edc_bist_cmd_len_reg = EDC_REG_T5(EDC_H_BIST_CMD_LEN_A, idx);
edc_bist_cmd_data_pattern = EDC_REG_T5(EDC_H_BIST_DATA_PATTERN_A, idx);
- edc_bist_status_rdata_reg = EDC_REG_T5(EDC_H_BIST_STATUS_RDATA_A, idx);
#undef EDC_REG_T5
#undef EDC_STRIDE_T5
diff --git a/drivers/scsi/csiostor/csio_init.c b/drivers/scsi/csiostor/csio_init.c
index 8dea7d5..390b07b 100644
--- a/drivers/scsi/csiostor/csio_init.c
+++ b/drivers/scsi/csiostor/csio_init.c
@@ -582,7 +582,7 @@
* @hw: The HW module.
* @dev: The device associated with this invocation.
* @probe: Called from probe context or not?
- * @os_pln: Parent lnode if any.
+ * @pln: Parent lnode if any.
*
* Allocates lnode structure via scsi_host_alloc, initializes
* shost, initializes lnode module and registers with SCSI ML
diff --git a/drivers/scsi/csiostor/csio_lnode.c b/drivers/scsi/csiostor/csio_lnode.c
index 74ff8ad..61cf5420 100644
--- a/drivers/scsi/csiostor/csio_lnode.c
+++ b/drivers/scsi/csiostor/csio_lnode.c
@@ -2068,10 +2068,9 @@
ln->fcfinfo = NULL;
}
-/**
+/*
* csio_lnode_init - Initialize the members of an lnode.
* @ln: lnode
- *
*/
int
csio_lnode_init(struct csio_lnode *ln, struct csio_hw *hw,
diff --git a/drivers/scsi/csiostor/csio_rnode.c b/drivers/scsi/csiostor/csio_rnode.c
index e9c3b04..713e13a 100644
--- a/drivers/scsi/csiostor/csio_rnode.c
+++ b/drivers/scsi/csiostor/csio_rnode.c
@@ -862,7 +862,7 @@
/**
* csio_rnode_fwevt_handler - Event handler for firmware rnode events.
* @rn: rnode
- *
+ * @fwevt: firmware event to handle
*/
void
csio_rnode_fwevt_handler(struct csio_rnode *rn, uint8_t fwevt)
diff --git a/drivers/scsi/cxgbi/cxgb3i/cxgb3i.c b/drivers/scsi/cxgbi/cxgb3i/cxgb3i.c
index ec7d01f..2b48954 100644
--- a/drivers/scsi/cxgbi/cxgb3i/cxgb3i.c
+++ b/drivers/scsi/cxgbi/cxgb3i/cxgb3i.c
@@ -361,7 +361,7 @@
/* len includes the length of any HW ULP additions */
req->len = htonl(len);
/* V_TX_ULP_SUBMODE sets both the mode and submode */
- req->flags = htonl(V_TX_ULP_SUBMODE(cxgbi_skcb_ulp_mode(skb)) |
+ req->flags = htonl(V_TX_ULP_SUBMODE(cxgbi_skcb_tx_ulp_mode(skb)) |
V_TX_SHOVE((skb_peek(&csk->write_queue) ? 0 : 1)));
req->sndseq = htonl(csk->snd_nxt);
req->param = htonl(V_TX_PORT(l2t->smt_idx));
@@ -375,10 +375,8 @@
}
}
-/**
+/*
* push_tx_frames -- start transmit
- * @c3cn: the offloaded connection
- * @req_completion: request wr_ack or not
*
* Prepends TX_DATA_WR or CPL_CLOSE_CON_REQ headers to buffers waiting in a
* connection's send queue and sends them on to T3. Must be called with the
@@ -442,7 +440,7 @@
req_completion = 1;
csk->wr_una_cred = 0;
}
- len += cxgbi_ulp_extra_len(cxgbi_skcb_ulp_mode(skb));
+ len += cxgbi_ulp_extra_len(cxgbi_skcb_tx_ulp_mode(skb));
make_tx_data_wr(csk, skb, len, req_completion);
csk->snd_nxt += len;
cxgbi_skcb_clear_flag(skb, SKCBF_TX_NEED_HDR);
@@ -886,11 +884,6 @@
return -ENOMEM;
}
-/**
- * release_offload_resources - release offload resource
- * @c3cn: the offloaded iscsi tcp connection.
- * Release resources held by an offload connection (TID, L2T entry, etc.)
- */
static void l2t_put(struct cxgbi_sock *csk)
{
struct t3cdev *t3dev = (struct t3cdev *)csk->cdev->lldev;
@@ -902,6 +895,10 @@
}
}
+/*
+ * release_offload_resources - release offload resource
+ * Release resources held by an offload connection (TID, L2T entry, etc.)
+ */
static void release_offload_resources(struct cxgbi_sock *csk)
{
struct t3cdev *t3dev = (struct t3cdev *)csk->cdev->lldev;
diff --git a/drivers/scsi/cxgbi/cxgb4i/cxgb4i.c b/drivers/scsi/cxgbi/cxgb4i/cxgb4i.c
index 8ce8592..4e82c14 100644
--- a/drivers/scsi/cxgbi/cxgb4i/cxgb4i.c
+++ b/drivers/scsi/cxgbi/cxgb4i/cxgb4i.c
@@ -197,7 +197,10 @@
if (likely(cxgbi_skcb_test_flag(skb, SKCBF_TX_NEED_HDR)))
len += sizeof(struct fw_ofld_tx_data_wr);
- return len <= MAX_IMM_TX_PKT_LEN;
+ if (likely(cxgbi_skcb_test_flag((struct sk_buff *)skb, SKCBF_TX_ISO)))
+ len += sizeof(struct cpl_tx_data_iso);
+
+ return (len <= MAX_IMM_OFLD_TX_DATA_WR_LEN);
}
static void send_act_open_req(struct cxgbi_sock *csk, struct sk_buff *skb,
@@ -641,7 +644,10 @@
flowc->mnemval[8].mnemonic = 0;
flowc->mnemval[8].val = 0;
flowc->mnemval[8].mnemonic = FW_FLOWC_MNEM_TXDATAPLEN_MAX;
- flowc->mnemval[8].val = 16384;
+ if (csk->cdev->skb_iso_txhdr)
+ flowc->mnemval[8].val = cpu_to_be32(CXGBI_MAX_ISO_DATA_IN_SKB);
+ else
+ flowc->mnemval[8].val = cpu_to_be32(16128);
#ifdef CONFIG_CHELSIO_T4_DCB
flowc->mnemval[9].mnemonic = FW_FLOWC_MNEM_DCBPRIO;
if (vlan == CPL_L2T_VLAN_NONE) {
@@ -667,38 +673,86 @@
return flowclen16;
}
-static inline void make_tx_data_wr(struct cxgbi_sock *csk, struct sk_buff *skb,
- int dlen, int len, u32 credits, int compl)
+static void
+cxgb4i_make_tx_iso_cpl(struct sk_buff *skb, struct cpl_tx_data_iso *cpl)
{
+ struct cxgbi_iso_info *info = (struct cxgbi_iso_info *)skb->head;
+ u32 imm_en = !!(info->flags & CXGBI_ISO_INFO_IMM_ENABLE);
+ u32 fslice = !!(info->flags & CXGBI_ISO_INFO_FSLICE);
+ u32 lslice = !!(info->flags & CXGBI_ISO_INFO_LSLICE);
+ u32 pdu_type = (info->op == ISCSI_OP_SCSI_CMD) ? 0 : 1;
+ u32 submode = cxgbi_skcb_tx_ulp_mode(skb) & 0x3;
+
+ cpl->op_to_scsi = cpu_to_be32(CPL_TX_DATA_ISO_OP_V(CPL_TX_DATA_ISO) |
+ CPL_TX_DATA_ISO_FIRST_V(fslice) |
+ CPL_TX_DATA_ISO_LAST_V(lslice) |
+ CPL_TX_DATA_ISO_CPLHDRLEN_V(0) |
+ CPL_TX_DATA_ISO_HDRCRC_V(submode & 1) |
+ CPL_TX_DATA_ISO_PLDCRC_V(((submode >> 1) & 1)) |
+ CPL_TX_DATA_ISO_IMMEDIATE_V(imm_en) |
+ CPL_TX_DATA_ISO_SCSI_V(pdu_type));
+
+ cpl->ahs_len = info->ahs;
+ cpl->mpdu = cpu_to_be16(DIV_ROUND_UP(info->mpdu, 4));
+ cpl->burst_size = cpu_to_be32(info->burst_size);
+ cpl->len = cpu_to_be32(info->len);
+ cpl->reserved2_seglen_offset =
+ cpu_to_be32(CPL_TX_DATA_ISO_SEGLEN_OFFSET_V(info->segment_offset));
+ cpl->datasn_offset = cpu_to_be32(info->datasn_offset);
+ cpl->buffer_offset = cpu_to_be32(info->buffer_offset);
+ cpl->reserved3 = cpu_to_be32(0);
+ log_debug(1 << CXGBI_DBG_ISCSI | 1 << CXGBI_DBG_PDU_TX,
+ "iso: flags 0x%x, op %u, ahs %u, num_pdu %u, mpdu %u, "
+ "burst_size %u, iso_len %u\n",
+ info->flags, info->op, info->ahs, info->num_pdu,
+ info->mpdu, info->burst_size << 2, info->len);
+}
+
+static void
+cxgb4i_make_tx_data_wr(struct cxgbi_sock *csk, struct sk_buff *skb, int dlen,
+ int len, u32 credits, int compl)
+{
+ struct cxgbi_device *cdev = csk->cdev;
+ struct cxgb4_lld_info *lldi = cxgbi_cdev_priv(cdev);
struct fw_ofld_tx_data_wr *req;
- unsigned int submode = cxgbi_skcb_ulp_mode(skb) & 3;
- unsigned int wr_ulp_mode = 0, val;
- bool imm = is_ofld_imm(skb);
+ struct cpl_tx_data_iso *cpl;
+ u32 submode = cxgbi_skcb_tx_ulp_mode(skb) & 0x3;
+ u32 wr_ulp_mode = 0;
+ u32 hdr_size = sizeof(*req);
+ u32 opcode = FW_OFLD_TX_DATA_WR;
+ u32 immlen = 0;
+ u32 force = is_t5(lldi->adapter_type) ? TX_FORCE_V(!submode) :
+ T6_TX_FORCE_F;
- req = __skb_push(skb, sizeof(*req));
-
- if (imm) {
- req->op_to_immdlen = htonl(FW_WR_OP_V(FW_OFLD_TX_DATA_WR) |
- FW_WR_COMPL_F |
- FW_WR_IMMDLEN_V(dlen));
- req->flowid_len16 = htonl(FW_WR_FLOWID_V(csk->tid) |
- FW_WR_LEN16_V(credits));
- } else {
- req->op_to_immdlen =
- cpu_to_be32(FW_WR_OP_V(FW_OFLD_TX_DATA_WR) |
- FW_WR_COMPL_F |
- FW_WR_IMMDLEN_V(0));
- req->flowid_len16 =
- cpu_to_be32(FW_WR_FLOWID_V(csk->tid) |
- FW_WR_LEN16_V(credits));
+ if (cxgbi_skcb_test_flag(skb, SKCBF_TX_ISO)) {
+ hdr_size += sizeof(struct cpl_tx_data_iso);
+ opcode = FW_ISCSI_TX_DATA_WR;
+ immlen += sizeof(struct cpl_tx_data_iso);
+ submode |= 8;
}
+
+ if (is_ofld_imm(skb))
+ immlen += dlen;
+
+ req = (struct fw_ofld_tx_data_wr *)__skb_push(skb, hdr_size);
+ req->op_to_immdlen = cpu_to_be32(FW_WR_OP_V(opcode) |
+ FW_WR_COMPL_V(compl) |
+ FW_WR_IMMDLEN_V(immlen));
+ req->flowid_len16 = cpu_to_be32(FW_WR_FLOWID_V(csk->tid) |
+ FW_WR_LEN16_V(credits));
+ req->plen = cpu_to_be32(len);
+ cpl = (struct cpl_tx_data_iso *)(req + 1);
+
+ if (likely(cxgbi_skcb_test_flag(skb, SKCBF_TX_ISO)))
+ cxgb4i_make_tx_iso_cpl(skb, cpl);
+
if (submode)
wr_ulp_mode = FW_OFLD_TX_DATA_WR_ULPMODE_V(ULP2_MODE_ISCSI) |
- FW_OFLD_TX_DATA_WR_ULPSUBMODE_V(submode);
- val = skb_peek(&csk->write_queue) ? 0 : 1;
- req->tunnel_to_proxy = htonl(wr_ulp_mode |
- FW_OFLD_TX_DATA_WR_SHOVE_V(val));
- req->plen = htonl(len);
+ FW_OFLD_TX_DATA_WR_ULPSUBMODE_V(submode);
+
+ req->tunnel_to_proxy = cpu_to_be32(wr_ulp_mode | force |
+ FW_OFLD_TX_DATA_WR_SHOVE_V(1U));
+
if (!cxgbi_sock_flag(csk, CTPF_TX_DATA_SENT))
cxgbi_sock_set_flag(csk, CTPF_TX_DATA_SENT);
}
@@ -716,30 +770,34 @@
if (unlikely(csk->state < CTP_ESTABLISHED ||
csk->state == CTP_CLOSE_WAIT_1 || csk->state >= CTP_ABORTING)) {
log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK |
- 1 << CXGBI_DBG_PDU_TX,
- "csk 0x%p,%u,0x%lx,%u, in closing state.\n",
- csk, csk->state, csk->flags, csk->tid);
+ 1 << CXGBI_DBG_PDU_TX,
+ "csk 0x%p,%u,0x%lx,%u, in closing state.\n",
+ csk, csk->state, csk->flags, csk->tid);
return 0;
}
- while (csk->wr_cred && (skb = skb_peek(&csk->write_queue)) != NULL) {
- int dlen = skb->len;
- int len = skb->len;
- unsigned int credits_needed;
- int flowclen16 = 0;
+ while (csk->wr_cred && ((skb = skb_peek(&csk->write_queue)) != NULL)) {
+ struct cxgbi_iso_info *iso_cpl;
+ u32 dlen = skb->len;
+ u32 len = skb->len;
+ u32 iso_cpl_len = 0;
+ u32 flowclen16 = 0;
+ u32 credits_needed;
+ u32 num_pdu = 1, hdr_len;
- skb_reset_transport_header(skb);
+ if (cxgbi_skcb_test_flag(skb, SKCBF_TX_ISO))
+ iso_cpl_len = sizeof(struct cpl_tx_data_iso);
+
if (is_ofld_imm(skb))
- credits_needed = DIV_ROUND_UP(dlen, 16);
+ credits_needed = DIV_ROUND_UP(dlen + iso_cpl_len, 16);
else
- credits_needed = DIV_ROUND_UP(
- 8 * calc_tx_flits_ofld(skb),
- 16);
+ credits_needed =
+ DIV_ROUND_UP((8 * calc_tx_flits_ofld(skb)) +
+ iso_cpl_len, 16);
if (likely(cxgbi_skcb_test_flag(skb, SKCBF_TX_NEED_HDR)))
- credits_needed += DIV_ROUND_UP(
- sizeof(struct fw_ofld_tx_data_wr),
- 16);
+ credits_needed +=
+ DIV_ROUND_UP(sizeof(struct fw_ofld_tx_data_wr), 16);
/*
* Assumes the initial credits is large enough to support
@@ -754,14 +812,19 @@
if (csk->wr_cred < credits_needed) {
log_debug(1 << CXGBI_DBG_PDU_TX,
- "csk 0x%p, skb %u/%u, wr %d < %u.\n",
- csk, skb->len, skb->data_len,
- credits_needed, csk->wr_cred);
+ "csk 0x%p, skb %u/%u, wr %d < %u.\n",
+ csk, skb->len, skb->data_len,
+ credits_needed, csk->wr_cred);
+
+ csk->no_tx_credits++;
break;
}
+
+ csk->no_tx_credits = 0;
+
__skb_unlink(skb, &csk->write_queue);
set_wr_txq(skb, CPL_PRIORITY_DATA, csk->port_id);
- skb->csum = credits_needed + flowclen16;
+ skb->csum = (__force __wsum)(credits_needed + flowclen16);
csk->wr_cred -= credits_needed;
csk->wr_una_cred += credits_needed;
cxgbi_sock_enqueue_wr(csk, skb);
@@ -771,25 +834,42 @@
csk, skb->len, skb->data_len, credits_needed,
csk->wr_cred, csk->wr_una_cred);
+ if (!req_completion &&
+ ((csk->wr_una_cred >= (csk->wr_max_cred / 2)) ||
+ after(csk->write_seq, (csk->snd_una + csk->snd_win / 2))))
+ req_completion = 1;
+
if (likely(cxgbi_skcb_test_flag(skb, SKCBF_TX_NEED_HDR))) {
- len += cxgbi_ulp_extra_len(cxgbi_skcb_ulp_mode(skb));
- make_tx_data_wr(csk, skb, dlen, len, credits_needed,
- req_completion);
+ u32 ulp_mode = cxgbi_skcb_tx_ulp_mode(skb);
+
+ if (cxgbi_skcb_test_flag(skb, SKCBF_TX_ISO)) {
+ iso_cpl = (struct cxgbi_iso_info *)skb->head;
+ num_pdu = iso_cpl->num_pdu;
+ hdr_len = cxgbi_skcb_tx_iscsi_hdrlen(skb);
+ len += (cxgbi_ulp_extra_len(ulp_mode) * num_pdu) +
+ (hdr_len * (num_pdu - 1));
+ } else {
+ len += cxgbi_ulp_extra_len(ulp_mode);
+ }
+
+ cxgb4i_make_tx_data_wr(csk, skb, dlen, len,
+ credits_needed, req_completion);
csk->snd_nxt += len;
cxgbi_skcb_clear_flag(skb, SKCBF_TX_NEED_HDR);
} else if (cxgbi_skcb_test_flag(skb, SKCBF_TX_FLAG_COMPL) &&
(csk->wr_una_cred >= (csk->wr_max_cred / 2))) {
struct cpl_close_con_req *req =
(struct cpl_close_con_req *)skb->data;
- req->wr.wr_hi |= htonl(FW_WR_COMPL_F);
+
+ req->wr.wr_hi |= cpu_to_be32(FW_WR_COMPL_F);
}
+
total_size += skb->truesize;
t4_set_arp_err_handler(skb, csk, arp_failure_skb_discard);
log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_PDU_TX,
- "csk 0x%p,%u,0x%lx,%u, skb 0x%p, %u.\n",
- csk, csk->state, csk->flags, csk->tid, skb, len);
-
+ "csk 0x%p,%u,0x%lx,%u, skb 0x%p, %u.\n",
+ csk, csk->state, csk->flags, csk->tid, skb, len);
cxgb4_l2t_send(csk->cdev->ports[csk->port_id], skb, csk->l2t);
}
return total_size;
@@ -2111,10 +2191,30 @@
return 0;
}
+static bool is_memfree(struct adapter *adap)
+{
+ u32 io;
+
+ io = t4_read_reg(adap, MA_TARGET_MEM_ENABLE_A);
+ if (is_t5(adap->params.chip)) {
+ if ((io & EXT_MEM0_ENABLE_F) || (io & EXT_MEM1_ENABLE_F))
+ return false;
+ } else if (io & EXT_MEM_ENABLE_F) {
+ return false;
+ }
+
+ return true;
+}
+
static void *t4_uld_add(const struct cxgb4_lld_info *lldi)
{
struct cxgbi_device *cdev;
struct port_info *pi;
+ struct net_device *ndev;
+ struct adapter *adap;
+ struct tid_info *t;
+ u32 max_cmds = CXGB4I_SCSI_HOST_QDEPTH;
+ u32 max_conn = CXGBI_MAX_CONN;
int i, rc;
cdev = cxgbi_device_register(sizeof(*lldi), lldi->nports);
@@ -2154,14 +2254,40 @@
pr_info("t4 0x%p ddp init failed %d.\n", cdev, rc);
goto err_out;
}
+
+ ndev = cdev->ports[0];
+ adap = netdev2adap(ndev);
+ if (adap) {
+ t = &adap->tids;
+ if (t->ntids <= CXGBI_MAX_CONN)
+ max_conn = t->ntids;
+
+ if (is_memfree(adap)) {
+ cdev->flags |= CXGBI_FLAG_DEV_ISO_OFF;
+ max_cmds = CXGB4I_SCSI_HOST_QDEPTH >> 2;
+
+ pr_info("%s: 0x%p, tid %u, SO adapter.\n",
+ ndev->name, cdev, t->ntids);
+ }
+ } else {
+ pr_info("%s, 0x%p, NO adapter struct.\n", ndev->name, cdev);
+ }
+
+ /* ISO is enabled in T5/T6 firmware version >= 1.13.43.0 */
+ if (!is_t4(lldi->adapter_type) &&
+ (lldi->fw_vers >= 0x10d2b00) &&
+ !(cdev->flags & CXGBI_FLAG_DEV_ISO_OFF))
+ cdev->skb_iso_txhdr = sizeof(struct cpl_tx_data_iso);
+
rc = cxgb4i_ofld_init(cdev);
if (rc) {
pr_info("t4 0x%p ofld init failed.\n", cdev);
goto err_out;
}
- rc = cxgbi_hbas_add(cdev, CXGB4I_MAX_LUN, CXGBI_MAX_CONN,
- &cxgb4i_host_template, cxgb4i_stt);
+ cxgb4i_host_template.can_queue = max_cmds;
+ rc = cxgbi_hbas_add(cdev, CXGB4I_MAX_LUN, max_conn,
+ &cxgb4i_host_template, cxgb4i_stt);
if (rc)
goto err_out;
diff --git a/drivers/scsi/cxgbi/libcxgbi.c b/drivers/scsi/cxgbi/libcxgbi.c
index 4bc794d..71aebaf 100644
--- a/drivers/scsi/cxgbi/libcxgbi.c
+++ b/drivers/scsi/cxgbi/libcxgbi.c
@@ -359,13 +359,15 @@
shost->max_lun = max_lun;
shost->max_id = max_id;
shost->max_channel = 0;
- shost->max_cmd_len = 16;
+ shost->max_cmd_len = SCSI_MAX_VARLEN_CDB_SIZE;
chba = iscsi_host_priv(shost);
chba->cdev = cdev;
chba->ndev = cdev->ports[i];
chba->shost = shost;
+ shost->can_queue = sht->can_queue - ISCSI_MGMT_CMDS_MAX;
+
log_debug(1 << CXGBI_DBG_DEV,
"cdev 0x%p, p#%d %s: chba 0x%p.\n",
cdev, i, cdev->ports[i]->name, chba);
@@ -1136,82 +1138,6 @@
}
EXPORT_SYMBOL_GPL(cxgbi_sock_check_wr_invariants);
-static int cxgbi_sock_send_pdus(struct cxgbi_sock *csk, struct sk_buff *skb)
-{
- struct cxgbi_device *cdev = csk->cdev;
- struct sk_buff *next;
- int err, copied = 0;
-
- spin_lock_bh(&csk->lock);
-
- if (csk->state != CTP_ESTABLISHED) {
- log_debug(1 << CXGBI_DBG_PDU_TX,
- "csk 0x%p,%u,0x%lx,%u, EAGAIN.\n",
- csk, csk->state, csk->flags, csk->tid);
- err = -EAGAIN;
- goto out_err;
- }
-
- if (csk->err) {
- log_debug(1 << CXGBI_DBG_PDU_TX,
- "csk 0x%p,%u,0x%lx,%u, EPIPE %d.\n",
- csk, csk->state, csk->flags, csk->tid, csk->err);
- err = -EPIPE;
- goto out_err;
- }
-
- if (csk->write_seq - csk->snd_una >= csk->snd_win) {
- log_debug(1 << CXGBI_DBG_PDU_TX,
- "csk 0x%p,%u,0x%lx,%u, FULL %u-%u >= %u.\n",
- csk, csk->state, csk->flags, csk->tid, csk->write_seq,
- csk->snd_una, csk->snd_win);
- err = -ENOBUFS;
- goto out_err;
- }
-
- while (skb) {
- int frags = skb_shinfo(skb)->nr_frags +
- (skb->len != skb->data_len);
-
- if (unlikely(skb_headroom(skb) < cdev->skb_tx_rsvd)) {
- pr_err("csk 0x%p, skb head %u < %u.\n",
- csk, skb_headroom(skb), cdev->skb_tx_rsvd);
- err = -EINVAL;
- goto out_err;
- }
-
- if (frags >= SKB_WR_LIST_SIZE) {
- pr_err("csk 0x%p, frags %d, %u,%u >%u.\n",
- csk, skb_shinfo(skb)->nr_frags, skb->len,
- skb->data_len, (uint)(SKB_WR_LIST_SIZE));
- err = -EINVAL;
- goto out_err;
- }
-
- next = skb->next;
- skb->next = NULL;
- cxgbi_skcb_set_flag(skb, SKCBF_TX_NEED_HDR);
- cxgbi_sock_skb_entail(csk, skb);
- copied += skb->len;
- csk->write_seq += skb->len +
- cxgbi_ulp_extra_len(cxgbi_skcb_ulp_mode(skb));
- skb = next;
- }
-
- if (likely(skb_queue_len(&csk->write_queue)))
- cdev->csk_push_tx_frames(csk, 1);
-done:
- spin_unlock_bh(&csk->lock);
- return copied;
-
-out_err:
- if (copied == 0 && err == -EPIPE)
- copied = csk->err ? csk->err : -EPIPE;
- else
- copied = err;
- goto done;
-}
-
static inline void
scmd_get_params(struct scsi_cmnd *sc, struct scatterlist **sgl,
unsigned int *sgcnt, unsigned int *dlen,
@@ -1284,8 +1210,6 @@
* APIs interacting with open-iscsi libraries
*/
-static unsigned char padding[4];
-
int cxgbi_ddp_ppm_setup(void **ppm_pp, struct cxgbi_device *cdev,
struct cxgbi_tag_format *tformat,
unsigned int iscsi_size, unsigned int llimit,
@@ -1833,9 +1757,10 @@
return -EFAULT;
}
-static int sgl_read_to_frags(struct scatterlist *sg, unsigned int sgoffset,
- unsigned int dlen, struct page_frag *frags,
- int frag_max)
+static int
+sgl_read_to_frags(struct scatterlist *sg, unsigned int sgoffset,
+ unsigned int dlen, struct page_frag *frags,
+ int frag_max, u32 *dlimit)
{
unsigned int datalen = dlen;
unsigned int sglen = sg->length - sgoffset;
@@ -1867,6 +1792,7 @@
if (i >= frag_max) {
pr_warn("too many pages %u, dlen %u.\n",
frag_max, dlen);
+ *dlimit = dlen - datalen;
return -EINVAL;
}
@@ -1883,38 +1809,220 @@
return i;
}
-int cxgbi_conn_alloc_pdu(struct iscsi_task *task, u8 opcode)
+static void cxgbi_task_data_sgl_check(struct iscsi_task *task)
{
- struct iscsi_tcp_conn *tcp_conn = task->conn->dd_data;
+ struct scsi_cmnd *sc = task->sc;
+ struct cxgbi_task_data *tdata = iscsi_task_cxgbi_data(task);
+ struct scatterlist *sg, *sgl = NULL;
+ u32 sgcnt = 0;
+ int i;
+
+ tdata->flags = CXGBI_TASK_SGL_CHECKED;
+ if (!sc)
+ return;
+
+ scmd_get_params(sc, &sgl, &sgcnt, &tdata->dlen, 0);
+ if (!sgl || !sgcnt) {
+ tdata->flags |= CXGBI_TASK_SGL_COPY;
+ return;
+ }
+
+ for_each_sg(sgl, sg, sgcnt, i) {
+ if (page_count(sg_page(sg)) < 1) {
+ tdata->flags |= CXGBI_TASK_SGL_COPY;
+ return;
+ }
+ }
+}
+
+static int
+cxgbi_task_data_sgl_read(struct iscsi_task *task, u32 offset, u32 count,
+ u32 *dlimit)
+{
+ struct scsi_cmnd *sc = task->sc;
+ struct cxgbi_task_data *tdata = iscsi_task_cxgbi_data(task);
+ struct scatterlist *sgl = NULL;
+ struct scatterlist *sg;
+ u32 dlen = 0;
+ u32 sgcnt;
+ int err;
+
+ if (!sc)
+ return 0;
+
+ scmd_get_params(sc, &sgl, &sgcnt, &dlen, 0);
+ if (!sgl || !sgcnt)
+ return 0;
+
+ err = sgl_seek_offset(sgl, sgcnt, offset, &tdata->sgoffset, &sg);
+ if (err < 0) {
+ pr_warn("tpdu max, sgl %u, bad offset %u/%u.\n",
+ sgcnt, offset, tdata->dlen);
+ return err;
+ }
+ err = sgl_read_to_frags(sg, tdata->sgoffset, count,
+ tdata->frags, MAX_SKB_FRAGS, dlimit);
+ if (err < 0) {
+ log_debug(1 << CXGBI_DBG_ISCSI,
+ "sgl max limit, sgl %u, offset %u, %u/%u, dlimit %u.\n",
+ sgcnt, offset, count, tdata->dlen, *dlimit);
+ return err;
+ }
+ tdata->offset = offset;
+ tdata->count = count;
+ tdata->nr_frags = err;
+ tdata->total_count = count;
+ tdata->total_offset = offset;
+
+ log_debug(1 << CXGBI_DBG_ISCSI | 1 << CXGBI_DBG_PDU_TX,
+ "%s: offset %u, count %u,\n"
+ "err %u, total_count %u, total_offset %u\n",
+ __func__, offset, count, err, tdata->total_count, tdata->total_offset);
+
+ return 0;
+}
+
+int cxgbi_conn_alloc_pdu(struct iscsi_task *task, u8 op)
+{
+ struct iscsi_conn *conn = task->conn;
+ struct iscsi_session *session = task->conn->session;
+ struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
struct cxgbi_conn *cconn = tcp_conn->dd_data;
struct cxgbi_device *cdev = cconn->chba->cdev;
- struct iscsi_conn *conn = task->conn;
+ struct cxgbi_sock *csk = cconn->cep ? cconn->cep->csk : NULL;
struct iscsi_tcp_task *tcp_task = task->dd_data;
struct cxgbi_task_data *tdata = iscsi_task_cxgbi_data(task);
struct scsi_cmnd *sc = task->sc;
- struct cxgbi_sock *csk = cconn->cep->csk;
- struct net_device *ndev = cdev->ports[csk->port_id];
- int headroom = SKB_TX_ISCSI_PDU_HEADER_MAX;
+ u32 headroom = SKB_TX_ISCSI_PDU_HEADER_MAX;
+ u32 max_txdata_len = conn->max_xmit_dlength;
+ u32 iso_tx_rsvd = 0, local_iso_info = 0;
+ u32 last_tdata_offset, last_tdata_count;
+ int err = 0;
+
+ if (!tcp_task) {
+ pr_err("task 0x%p, tcp_task 0x%p, tdata 0x%p.\n",
+ task, tcp_task, tdata);
+ return -ENOMEM;
+ }
+ if (!csk) {
+ pr_err("task 0x%p, csk gone.\n", task);
+ return -EPIPE;
+ }
+
+ op &= ISCSI_OPCODE_MASK;
tcp_task->dd_data = tdata;
task->hdr = NULL;
- if (SKB_MAX_HEAD(cdev->skb_tx_rsvd) > (512 * MAX_SKB_FRAGS) &&
- (opcode == ISCSI_OP_SCSI_DATA_OUT ||
- (opcode == ISCSI_OP_SCSI_CMD &&
- sc->sc_data_direction == DMA_TO_DEVICE)))
- /* data could goes into skb head */
- headroom += min_t(unsigned int,
- SKB_MAX_HEAD(cdev->skb_tx_rsvd),
- conn->max_xmit_dlength);
+ last_tdata_count = tdata->count;
+ last_tdata_offset = tdata->offset;
- tdata->skb = alloc_skb(cdev->skb_tx_rsvd + headroom, GFP_ATOMIC);
- if (!tdata->skb) {
- ndev->stats.tx_dropped++;
- return -ENOMEM;
+ if ((op == ISCSI_OP_SCSI_DATA_OUT) ||
+ ((op == ISCSI_OP_SCSI_CMD) &&
+ (sc->sc_data_direction == DMA_TO_DEVICE))) {
+ u32 remaining_data_tosend, dlimit = 0;
+ u32 max_pdu_size, max_num_pdu, num_pdu;
+ u32 count;
+
+ /* Preserve conn->max_xmit_dlength because it can get updated to
+ * ISO data size.
+ */
+ if (task->state == ISCSI_TASK_PENDING)
+ tdata->max_xmit_dlength = conn->max_xmit_dlength;
+
+ if (!tdata->offset)
+ cxgbi_task_data_sgl_check(task);
+
+ remaining_data_tosend =
+ tdata->dlen - tdata->offset - tdata->count;
+
+recalculate_sgl:
+ max_txdata_len = tdata->max_xmit_dlength;
+ log_debug(1 << CXGBI_DBG_ISCSI | 1 << CXGBI_DBG_PDU_TX,
+ "tdata->dlen %u, remaining to send %u "
+ "conn->max_xmit_dlength %u, "
+ "tdata->max_xmit_dlength %u\n",
+ tdata->dlen, remaining_data_tosend,
+ conn->max_xmit_dlength, tdata->max_xmit_dlength);
+
+ if (cdev->skb_iso_txhdr && !csk->disable_iso &&
+ (remaining_data_tosend > tdata->max_xmit_dlength) &&
+ !(remaining_data_tosend % 4)) {
+ u32 max_iso_data;
+
+ if ((op == ISCSI_OP_SCSI_CMD) &&
+ session->initial_r2t_en)
+ goto no_iso;
+
+ max_pdu_size = tdata->max_xmit_dlength +
+ ISCSI_PDU_NONPAYLOAD_LEN;
+ max_iso_data = rounddown(CXGBI_MAX_ISO_DATA_IN_SKB,
+ csk->advmss);
+ max_num_pdu = max_iso_data / max_pdu_size;
+
+ num_pdu = (remaining_data_tosend +
+ tdata->max_xmit_dlength - 1) /
+ tdata->max_xmit_dlength;
+
+ if (num_pdu > max_num_pdu)
+ num_pdu = max_num_pdu;
+
+ conn->max_xmit_dlength = tdata->max_xmit_dlength * num_pdu;
+ max_txdata_len = conn->max_xmit_dlength;
+ iso_tx_rsvd = cdev->skb_iso_txhdr;
+ local_iso_info = sizeof(struct cxgbi_iso_info);
+
+ log_debug(1 << CXGBI_DBG_ISCSI | 1 << CXGBI_DBG_PDU_TX,
+ "max_pdu_size %u, max_num_pdu %u, "
+ "max_txdata %u, num_pdu %u\n",
+ max_pdu_size, max_num_pdu,
+ max_txdata_len, num_pdu);
+ }
+no_iso:
+ count = min_t(u32, max_txdata_len, remaining_data_tosend);
+ err = cxgbi_task_data_sgl_read(task,
+ tdata->offset + tdata->count,
+ count, &dlimit);
+ if (unlikely(err < 0)) {
+ log_debug(1 << CXGBI_DBG_ISCSI,
+ "task 0x%p, tcp_task 0x%p, tdata 0x%p, "
+ "sgl err %d, count %u, dlimit %u\n",
+ task, tcp_task, tdata, err, count, dlimit);
+ if (dlimit) {
+ remaining_data_tosend =
+ rounddown(dlimit,
+ tdata->max_xmit_dlength);
+ if (!remaining_data_tosend)
+ remaining_data_tosend = dlimit;
+
+ dlimit = 0;
+
+ conn->max_xmit_dlength = remaining_data_tosend;
+ goto recalculate_sgl;
+ }
+
+ pr_err("task 0x%p, tcp_task 0x%p, tdata 0x%p, "
+ "sgl err %d\n",
+ task, tcp_task, tdata, err);
+ goto ret_err;
+ }
+
+ if ((tdata->flags & CXGBI_TASK_SGL_COPY) ||
+ (tdata->nr_frags > MAX_SKB_FRAGS))
+ headroom += conn->max_xmit_dlength;
}
- skb_reserve(tdata->skb, cdev->skb_tx_rsvd);
+ tdata->skb = alloc_skb(local_iso_info + cdev->skb_tx_rsvd +
+ iso_tx_rsvd + headroom, GFP_ATOMIC);
+ if (!tdata->skb) {
+ tdata->count = last_tdata_count;
+ tdata->offset = last_tdata_offset;
+ err = -ENOMEM;
+ goto ret_err;
+ }
+
+ skb_reserve(tdata->skb, local_iso_info + cdev->skb_tx_rsvd +
+ iso_tx_rsvd);
if (task->sc) {
task->hdr = (struct iscsi_hdr *)tdata->skb->data;
@@ -1923,25 +2031,100 @@
if (!task->hdr) {
__kfree_skb(tdata->skb);
tdata->skb = NULL;
- ndev->stats.tx_dropped++;
return -ENOMEM;
}
}
- task->hdr_max = SKB_TX_ISCSI_PDU_HEADER_MAX; /* BHS + AHS */
+
+ task->hdr_max = SKB_TX_ISCSI_PDU_HEADER_MAX;
+
+ if (iso_tx_rsvd)
+ cxgbi_skcb_set_flag(tdata->skb, SKCBF_TX_ISO);
/* data_out uses scsi_cmd's itt */
- if (opcode != ISCSI_OP_SCSI_DATA_OUT)
+ if (op != ISCSI_OP_SCSI_DATA_OUT)
task_reserve_itt(task, &task->hdr->itt);
log_debug(1 << CXGBI_DBG_ISCSI | 1 << CXGBI_DBG_PDU_TX,
- "task 0x%p, op 0x%x, skb 0x%p,%u+%u/%u, itt 0x%x.\n",
- task, opcode, tdata->skb, cdev->skb_tx_rsvd, headroom,
- conn->max_xmit_dlength, ntohl(task->hdr->itt));
+ "task 0x%p, op 0x%x, skb 0x%p,%u+%u/%u, itt 0x%x.\n",
+ task, op, tdata->skb, cdev->skb_tx_rsvd, headroom,
+ conn->max_xmit_dlength, be32_to_cpu(task->hdr->itt));
return 0;
+
+ret_err:
+ conn->max_xmit_dlength = tdata->max_xmit_dlength;
+ return err;
}
EXPORT_SYMBOL_GPL(cxgbi_conn_alloc_pdu);
+static int
+cxgbi_prep_iso_info(struct iscsi_task *task, struct sk_buff *skb,
+ u32 count)
+{
+ struct cxgbi_iso_info *iso_info = (struct cxgbi_iso_info *)skb->head;
+ struct iscsi_r2t_info *r2t;
+ struct cxgbi_task_data *tdata = iscsi_task_cxgbi_data(task);
+ struct iscsi_conn *conn = task->conn;
+ struct iscsi_session *session = conn->session;
+ struct iscsi_tcp_task *tcp_task = task->dd_data;
+ u32 burst_size = 0, r2t_dlength = 0, dlength;
+ u32 max_pdu_len = tdata->max_xmit_dlength;
+ u32 segment_offset = 0;
+ u32 num_pdu;
+
+ if (unlikely(!cxgbi_skcb_test_flag(skb, SKCBF_TX_ISO)))
+ return 0;
+
+ memset(iso_info, 0, sizeof(struct cxgbi_iso_info));
+
+ if (task->hdr->opcode == ISCSI_OP_SCSI_CMD && session->imm_data_en) {
+ iso_info->flags |= CXGBI_ISO_INFO_IMM_ENABLE;
+ burst_size = count;
+ }
+
+ dlength = ntoh24(task->hdr->dlength);
+ dlength = min(dlength, max_pdu_len);
+ hton24(task->hdr->dlength, dlength);
+
+ num_pdu = (count + max_pdu_len - 1) / max_pdu_len;
+
+ if (iscsi_task_has_unsol_data(task))
+ r2t = &task->unsol_r2t;
+ else
+ r2t = tcp_task->r2t;
+
+ if (r2t) {
+ log_debug(1 << CXGBI_DBG_ISCSI | 1 << CXGBI_DBG_PDU_TX,
+ "count %u, tdata->count %u, num_pdu %u,"
+ "task->hdr_len %u, r2t->data_length %u, r2t->sent %u\n",
+ count, tdata->count, num_pdu, task->hdr_len,
+ r2t->data_length, r2t->sent);
+
+ r2t_dlength = r2t->data_length - r2t->sent;
+ segment_offset = r2t->sent;
+ r2t->datasn += num_pdu - 1;
+ }
+
+ if (!r2t || !r2t->sent)
+ iso_info->flags |= CXGBI_ISO_INFO_FSLICE;
+
+ if (task->hdr->flags & ISCSI_FLAG_CMD_FINAL)
+ iso_info->flags |= CXGBI_ISO_INFO_LSLICE;
+
+ task->hdr->flags &= ~ISCSI_FLAG_CMD_FINAL;
+
+ iso_info->op = task->hdr->opcode;
+ iso_info->ahs = task->hdr->hlength;
+ iso_info->num_pdu = num_pdu;
+ iso_info->mpdu = max_pdu_len;
+ iso_info->burst_size = (burst_size + r2t_dlength) >> 2;
+ iso_info->len = count + task->hdr_len;
+ iso_info->segment_offset = segment_offset;
+
+ cxgbi_skcb_tx_iscsi_hdrlen(skb) = task->hdr_len;
+ return 0;
+}
+
static inline void tx_skb_setmode(struct sk_buff *skb, int hcrc, int dcrc)
{
if (hcrc || dcrc) {
@@ -1951,133 +2134,260 @@
submode |= 1;
if (dcrc)
submode |= 2;
- cxgbi_skcb_ulp_mode(skb) = (ULP2_MODE_ISCSI << 4) | submode;
+ cxgbi_skcb_tx_ulp_mode(skb) = (ULP2_MODE_ISCSI << 4) | submode;
} else
- cxgbi_skcb_ulp_mode(skb) = 0;
+ cxgbi_skcb_tx_ulp_mode(skb) = 0;
}
+static struct page *rsvd_page;
+
int cxgbi_conn_init_pdu(struct iscsi_task *task, unsigned int offset,
unsigned int count)
{
struct iscsi_conn *conn = task->conn;
+ struct iscsi_tcp_task *tcp_task = task->dd_data;
struct cxgbi_task_data *tdata = iscsi_task_cxgbi_data(task);
- struct sk_buff *skb = tdata->skb;
- unsigned int datalen = count;
- int i, padlen = iscsi_padding(count);
+ struct sk_buff *skb;
+ struct scsi_cmnd *sc = task->sc;
+ u32 expected_count, expected_offset;
+ u32 datalen = count, dlimit = 0;
+ u32 i, padlen = iscsi_padding(count);
struct page *pg;
+ int err;
+
+ if (!tcp_task || (tcp_task->dd_data != tdata)) {
+ pr_err("task 0x%p,0x%p, tcp_task 0x%p, tdata 0x%p/0x%p.\n",
+ task, task->sc, tcp_task,
+ tcp_task ? tcp_task->dd_data : NULL, tdata);
+ return -EINVAL;
+ }
+ skb = tdata->skb;
log_debug(1 << CXGBI_DBG_ISCSI | 1 << CXGBI_DBG_PDU_TX,
- "task 0x%p,0x%p, skb 0x%p, 0x%x,0x%x,0x%x, %u+%u.\n",
- task, task->sc, skb, (*skb->data) & ISCSI_OPCODE_MASK,
- ntohl(task->cmdsn), ntohl(task->hdr->itt), offset, count);
+ "task 0x%p,0x%p, skb 0x%p, 0x%x,0x%x,0x%x, %u+%u.\n",
+ task, task->sc, skb, (*skb->data) & ISCSI_OPCODE_MASK,
+ be32_to_cpu(task->cmdsn), be32_to_cpu(task->hdr->itt), offset, count);
skb_put(skb, task->hdr_len);
tx_skb_setmode(skb, conn->hdrdgst_en, datalen ? conn->datadgst_en : 0);
- if (!count)
- return 0;
-
- if (task->sc) {
- struct scsi_data_buffer *sdb = &task->sc->sdb;
- struct scatterlist *sg = NULL;
- int err;
-
- tdata->offset = offset;
+ if (!count) {
tdata->count = count;
- err = sgl_seek_offset(
- sdb->table.sgl, sdb->table.nents,
- tdata->offset, &tdata->sgoffset, &sg);
- if (err < 0) {
- pr_warn("tpdu, sgl %u, bad offset %u/%u.\n",
- sdb->table.nents, tdata->offset, sdb->length);
- return err;
- }
- err = sgl_read_to_frags(sg, tdata->sgoffset, tdata->count,
- tdata->frags, MAX_PDU_FRAGS);
- if (err < 0) {
- pr_warn("tpdu, sgl %u, bad offset %u + %u.\n",
- sdb->table.nents, tdata->offset, tdata->count);
- return err;
- }
- tdata->nr_frags = err;
+ tdata->offset = offset;
+ tdata->nr_frags = 0;
+ tdata->total_offset = 0;
+ tdata->total_count = 0;
+ if (tdata->max_xmit_dlength)
+ conn->max_xmit_dlength = tdata->max_xmit_dlength;
+ cxgbi_skcb_clear_flag(skb, SKCBF_TX_ISO);
+ return 0;
+ }
- if (tdata->nr_frags > MAX_SKB_FRAGS ||
- (padlen && tdata->nr_frags == MAX_SKB_FRAGS)) {
+ log_debug(1 << CXGBI_DBG_ISCSI | 1 << CXGBI_DBG_PDU_TX,
+ "data->total_count %u, tdata->total_offset %u\n",
+ tdata->total_count, tdata->total_offset);
+
+ expected_count = tdata->total_count;
+ expected_offset = tdata->total_offset;
+
+ if ((count != expected_count) ||
+ (offset != expected_offset)) {
+ err = cxgbi_task_data_sgl_read(task, offset, count, &dlimit);
+ if (err < 0) {
+ pr_err("task 0x%p,0x%p, tcp_task 0x%p, tdata 0x%p/0x%p "
+ "dlimit %u, sgl err %d.\n", task, task->sc,
+ tcp_task, tcp_task ? tcp_task->dd_data : NULL,
+ tdata, dlimit, err);
+ return err;
+ }
+ }
+
+ /* Restore original value of conn->max_xmit_dlength because
+ * it can get updated to ISO data size.
+ */
+ conn->max_xmit_dlength = tdata->max_xmit_dlength;
+
+ if (sc) {
+ struct page_frag *frag = tdata->frags;
+
+ if ((tdata->flags & CXGBI_TASK_SGL_COPY) ||
+ (tdata->nr_frags > MAX_SKB_FRAGS) ||
+ (padlen && (tdata->nr_frags ==
+ MAX_SKB_FRAGS))) {
char *dst = skb->data + task->hdr_len;
- struct page_frag *frag = tdata->frags;
/* data fits in the skb's headroom */
for (i = 0; i < tdata->nr_frags; i++, frag++) {
char *src = kmap_atomic(frag->page);
- memcpy(dst, src+frag->offset, frag->size);
+ memcpy(dst, src + frag->offset, frag->size);
dst += frag->size;
kunmap_atomic(src);
}
+
if (padlen) {
memset(dst, 0, padlen);
padlen = 0;
}
skb_put(skb, count + padlen);
} else {
- /* data fit into frag_list */
- for (i = 0; i < tdata->nr_frags; i++) {
- __skb_fill_page_desc(skb, i,
- tdata->frags[i].page,
- tdata->frags[i].offset,
- tdata->frags[i].size);
- skb_frag_ref(skb, i);
+ for (i = 0; i < tdata->nr_frags; i++, frag++) {
+ get_page(frag->page);
+ skb_fill_page_desc(skb, i, frag->page,
+ frag->offset, frag->size);
}
- skb_shinfo(skb)->nr_frags = tdata->nr_frags;
+
skb->len += count;
skb->data_len += count;
skb->truesize += count;
}
-
} else {
- pg = virt_to_page(task->data);
-
+ pg = virt_to_head_page(task->data);
get_page(pg);
- skb_fill_page_desc(skb, 0, pg, offset_in_page(task->data),
- count);
+ skb_fill_page_desc(skb, 0, pg,
+ task->data - (char *)page_address(pg),
+ count);
skb->len += count;
skb->data_len += count;
skb->truesize += count;
}
if (padlen) {
- i = skb_shinfo(skb)->nr_frags;
+ get_page(rsvd_page);
skb_fill_page_desc(skb, skb_shinfo(skb)->nr_frags,
- virt_to_page(padding), offset_in_page(padding),
- padlen);
+ rsvd_page, 0, padlen);
skb->data_len += padlen;
skb->truesize += padlen;
skb->len += padlen;
}
+ if (likely(count > tdata->max_xmit_dlength))
+ cxgbi_prep_iso_info(task, skb, count);
+ else
+ cxgbi_skcb_clear_flag(skb, SKCBF_TX_ISO);
+
return 0;
}
EXPORT_SYMBOL_GPL(cxgbi_conn_init_pdu);
+static int cxgbi_sock_tx_queue_up(struct cxgbi_sock *csk, struct sk_buff *skb)
+{
+ struct cxgbi_device *cdev = csk->cdev;
+ struct cxgbi_iso_info *iso_cpl;
+ u32 frags = skb_shinfo(skb)->nr_frags;
+ u32 extra_len, num_pdu, hdr_len;
+ u32 iso_tx_rsvd = 0;
+
+ if (csk->state != CTP_ESTABLISHED) {
+ log_debug(1 << CXGBI_DBG_PDU_TX,
+ "csk 0x%p,%u,0x%lx,%u, EAGAIN.\n",
+ csk, csk->state, csk->flags, csk->tid);
+ return -EPIPE;
+ }
+
+ if (csk->err) {
+ log_debug(1 << CXGBI_DBG_PDU_TX,
+ "csk 0x%p,%u,0x%lx,%u, EPIPE %d.\n",
+ csk, csk->state, csk->flags, csk->tid, csk->err);
+ return -EPIPE;
+ }
+
+ if ((cdev->flags & CXGBI_FLAG_DEV_T3) &&
+ before((csk->snd_win + csk->snd_una), csk->write_seq)) {
+ log_debug(1 << CXGBI_DBG_PDU_TX,
+ "csk 0x%p,%u,0x%lx,%u, FULL %u-%u >= %u.\n",
+ csk, csk->state, csk->flags, csk->tid, csk->write_seq,
+ csk->snd_una, csk->snd_win);
+ return -ENOBUFS;
+ }
+
+ if (cxgbi_skcb_test_flag(skb, SKCBF_TX_ISO))
+ iso_tx_rsvd = cdev->skb_iso_txhdr;
+
+ if (unlikely(skb_headroom(skb) < (cdev->skb_tx_rsvd + iso_tx_rsvd))) {
+ pr_err("csk 0x%p, skb head %u < %u.\n",
+ csk, skb_headroom(skb), cdev->skb_tx_rsvd);
+ return -EINVAL;
+ }
+
+ if (skb->len != skb->data_len)
+ frags++;
+
+ if (frags >= SKB_WR_LIST_SIZE) {
+ pr_err("csk 0x%p, frags %u, %u,%u >%lu.\n",
+ csk, skb_shinfo(skb)->nr_frags, skb->len,
+ skb->data_len, SKB_WR_LIST_SIZE);
+ return -EINVAL;
+ }
+
+ cxgbi_skcb_set_flag(skb, SKCBF_TX_NEED_HDR);
+ skb_reset_transport_header(skb);
+ cxgbi_sock_skb_entail(csk, skb);
+
+ extra_len = cxgbi_ulp_extra_len(cxgbi_skcb_tx_ulp_mode(skb));
+
+ if (likely(cxgbi_skcb_test_flag(skb, SKCBF_TX_ISO))) {
+ iso_cpl = (struct cxgbi_iso_info *)skb->head;
+ num_pdu = iso_cpl->num_pdu;
+ hdr_len = cxgbi_skcb_tx_iscsi_hdrlen(skb);
+ extra_len = (cxgbi_ulp_extra_len(cxgbi_skcb_tx_ulp_mode(skb)) *
+ num_pdu) + (hdr_len * (num_pdu - 1));
+ }
+
+ csk->write_seq += (skb->len + extra_len);
+
+ return 0;
+}
+
+static int cxgbi_sock_send_skb(struct cxgbi_sock *csk, struct sk_buff *skb)
+{
+ struct cxgbi_device *cdev = csk->cdev;
+ int len = skb->len;
+ int err;
+
+ spin_lock_bh(&csk->lock);
+ err = cxgbi_sock_tx_queue_up(csk, skb);
+ if (err < 0) {
+ spin_unlock_bh(&csk->lock);
+ return err;
+ }
+
+ if (likely(skb_queue_len(&csk->write_queue)))
+ cdev->csk_push_tx_frames(csk, 0);
+ spin_unlock_bh(&csk->lock);
+ return len;
+}
+
int cxgbi_conn_xmit_pdu(struct iscsi_task *task)
{
struct iscsi_tcp_conn *tcp_conn = task->conn->dd_data;
struct cxgbi_conn *cconn = tcp_conn->dd_data;
+ struct iscsi_tcp_task *tcp_task = task->dd_data;
struct cxgbi_task_data *tdata = iscsi_task_cxgbi_data(task);
struct cxgbi_task_tag_info *ttinfo = &tdata->ttinfo;
- struct sk_buff *skb = tdata->skb;
+ struct sk_buff *skb;
struct cxgbi_sock *csk = NULL;
- unsigned int datalen;
+ u32 pdulen = 0;
+ u32 datalen;
int err;
+ if (!tcp_task || (tcp_task->dd_data != tdata)) {
+ pr_err("task 0x%p,0x%p, tcp_task 0x%p, tdata 0x%p/0x%p.\n",
+ task, task->sc, tcp_task,
+ tcp_task ? tcp_task->dd_data : NULL, tdata);
+ return -EINVAL;
+ }
+
+ skb = tdata->skb;
if (!skb) {
log_debug(1 << CXGBI_DBG_ISCSI | 1 << CXGBI_DBG_PDU_TX,
- "task 0x%p\n", task);
+ "task 0x%p, skb NULL.\n", task);
return 0;
}
if (cconn && cconn->cep)
csk = cconn->cep->csk;
+
if (!csk) {
log_debug(1 << CXGBI_DBG_ISCSI | 1 << CXGBI_DBG_PDU_TX,
"task 0x%p, csk gone.\n", task);
@@ -2101,13 +2411,12 @@
if (!task->sc)
memcpy(skb->data, task->hdr, SKB_TX_ISCSI_PDU_HEADER_MAX);
- err = cxgbi_sock_send_pdus(cconn->cep->csk, skb);
+ err = cxgbi_sock_send_skb(csk, skb);
if (err > 0) {
- int pdulen = err;
+ pdulen += err;
- log_debug(1 << CXGBI_DBG_PDU_TX,
- "task 0x%p,0x%p, skb 0x%p, len %u/%u, rv %d.\n",
- task, task->sc, skb, skb->len, skb->data_len, err);
+ log_debug(1 << CXGBI_DBG_PDU_TX, "task 0x%p,0x%p, rv %d.\n",
+ task, task->sc, err);
if (task->conn->hdrdgst_en)
pdulen += ISCSI_DIGEST_SIZE;
@@ -2116,24 +2425,42 @@
pdulen += ISCSI_DIGEST_SIZE;
task->conn->txdata_octets += pdulen;
+
+ if (unlikely(cxgbi_is_iso_config(csk) && cxgbi_is_iso_disabled(csk))) {
+ if (time_after(jiffies, csk->prev_iso_ts + HZ)) {
+ csk->disable_iso = false;
+ csk->prev_iso_ts = 0;
+ log_debug(1 << CXGBI_DBG_PDU_TX,
+ "enable iso: csk 0x%p\n", csk);
+ }
+ }
+
return 0;
}
if (err == -EAGAIN || err == -ENOBUFS) {
log_debug(1 << CXGBI_DBG_PDU_TX,
- "task 0x%p, skb 0x%p, len %u/%u, %d EAGAIN.\n",
- task, skb, skb->len, skb->data_len, err);
+ "task 0x%p, skb 0x%p, len %u/%u, %d EAGAIN.\n",
+ task, skb, skb->len, skb->data_len, err);
/* reset skb to send when we are called again */
tdata->skb = skb;
+
+ if (cxgbi_is_iso_config(csk) && !cxgbi_is_iso_disabled(csk) &&
+ (csk->no_tx_credits++ >= 2)) {
+ csk->disable_iso = true;
+ csk->prev_iso_ts = jiffies;
+ log_debug(1 << CXGBI_DBG_PDU_TX,
+ "disable iso:csk 0x%p, ts:%lu\n",
+ csk, csk->prev_iso_ts);
+ }
+
return err;
}
- log_debug(1 << CXGBI_DBG_ISCSI | 1 << CXGBI_DBG_PDU_TX,
- "itt 0x%x, skb 0x%p, len %u/%u, xmit err %d.\n",
- task->itt, skb, skb->len, skb->data_len, err);
-
__kfree_skb(skb);
-
+ log_debug(1 << CXGBI_DBG_ISCSI | 1 << CXGBI_DBG_PDU_TX,
+ "itt 0x%x, skb 0x%p, len %u/%u, xmit err %d.\n",
+ task->itt, skb, skb->len, skb->data_len, err);
iscsi_conn_printk(KERN_ERR, task->conn, "xmit err %d.\n", err);
iscsi_conn_failure(task->conn, ISCSI_ERR_XMIT_FAILED);
return err;
@@ -2145,7 +2472,7 @@
struct iscsi_tcp_task *tcp_task = task->dd_data;
struct cxgbi_task_data *tdata = iscsi_task_cxgbi_data(task);
- if (!tcp_task || !tdata || (tcp_task->dd_data != tdata)) {
+ if (!tcp_task || (tcp_task->dd_data != tdata)) {
pr_info("task 0x%p,0x%p, tcp_task 0x%p, tdata 0x%p/0x%p.\n",
task, task->sc, tcp_task,
tcp_task ? tcp_task->dd_data : NULL, tdata);
@@ -2749,12 +3076,17 @@
BUILD_BUG_ON(sizeof_field(struct sk_buff, cb) <
sizeof(struct cxgbi_skb_cb));
+ rsvd_page = alloc_page(GFP_KERNEL | __GFP_ZERO);
+ if (!rsvd_page)
+ return -ENOMEM;
+
return 0;
}
static void __exit libcxgbi_exit_module(void)
{
cxgbi_device_unregister_all(0xFF);
+ put_page(rsvd_page);
return;
}
diff --git a/drivers/scsi/cxgbi/libcxgbi.h b/drivers/scsi/cxgbi/libcxgbi.h
index 84b96af..fc7255f 100644
--- a/drivers/scsi/cxgbi/libcxgbi.h
+++ b/drivers/scsi/cxgbi/libcxgbi.h
@@ -76,6 +76,14 @@
#define ULP2_MAX_PDU_PAYLOAD \
(ULP2_MAX_PKT_SIZE - ISCSI_PDU_NONPAYLOAD_LEN)
+#define CXGBI_ULP2_MAX_ISO_PAYLOAD 65535
+
+#define CXGBI_MAX_ISO_DATA_IN_SKB \
+ min_t(u32, MAX_SKB_FRAGS << PAGE_SHIFT, CXGBI_ULP2_MAX_ISO_PAYLOAD)
+
+#define cxgbi_is_iso_config(csk) ((csk)->cdev->skb_iso_txhdr)
+#define cxgbi_is_iso_disabled(csk) ((csk)->disable_iso)
+
/*
* For iscsi connections HW may inserts digest bytes into the pdu. Those digest
* bytes are not sent by the host but are part of the TCP payload and therefore
@@ -162,6 +170,10 @@
u32 write_seq;
u32 snd_win;
u32 rcv_win;
+
+ bool disable_iso;
+ u32 no_tx_credits;
+ unsigned long prev_iso_ts;
};
/*
@@ -203,6 +215,8 @@
void *handle;
void *arp_err_handler;
struct sk_buff *wr_next;
+ u16 iscsi_hdr_len;
+ u8 ulp_mode;
};
enum cxgbi_skcb_flags {
@@ -218,6 +232,7 @@
SKCBF_RX_HCRC_ERR, /* header digest error */
SKCBF_RX_DCRC_ERR, /* data digest error */
SKCBF_RX_PAD_ERR, /* padding byte error */
+ SKCBF_TX_ISO, /* iso cpl in tx skb */
};
struct cxgbi_skb_cb {
@@ -225,18 +240,18 @@
struct cxgbi_skb_rx_cb rx;
struct cxgbi_skb_tx_cb tx;
};
- unsigned char ulp_mode;
unsigned long flags;
unsigned int seq;
};
#define CXGBI_SKB_CB(skb) ((struct cxgbi_skb_cb *)&((skb)->cb[0]))
#define cxgbi_skcb_flags(skb) (CXGBI_SKB_CB(skb)->flags)
-#define cxgbi_skcb_ulp_mode(skb) (CXGBI_SKB_CB(skb)->ulp_mode)
#define cxgbi_skcb_tcp_seq(skb) (CXGBI_SKB_CB(skb)->seq)
#define cxgbi_skcb_rx_ddigest(skb) (CXGBI_SKB_CB(skb)->rx.ddigest)
#define cxgbi_skcb_rx_pdulen(skb) (CXGBI_SKB_CB(skb)->rx.pdulen)
#define cxgbi_skcb_tx_wr_next(skb) (CXGBI_SKB_CB(skb)->tx.wr_next)
+#define cxgbi_skcb_tx_iscsi_hdrlen(skb) (CXGBI_SKB_CB(skb)->tx.iscsi_hdr_len)
+#define cxgbi_skcb_tx_ulp_mode(skb) (CXGBI_SKB_CB(skb)->tx.ulp_mode)
static inline void cxgbi_skcb_set_flag(struct sk_buff *skb,
enum cxgbi_skcb_flags flag)
@@ -458,6 +473,7 @@
#define CXGBI_FLAG_IPV4_SET 0x10
#define CXGBI_FLAG_USE_PPOD_OFLDQ 0x40
#define CXGBI_FLAG_DDP_OFF 0x100
+#define CXGBI_FLAG_DEV_ISO_OFF 0x400
struct cxgbi_device {
struct list_head list_head;
@@ -477,6 +493,7 @@
unsigned int pfvf;
unsigned int rx_credit_thres;
unsigned int skb_tx_rsvd;
+ u32 skb_iso_txhdr;
unsigned int skb_rx_extra; /* for msg coalesced mode */
unsigned int tx_max_size;
unsigned int rx_max_size;
@@ -523,20 +540,41 @@
struct cxgbi_sock *csk;
};
-#define MAX_PDU_FRAGS ((ULP2_MAX_PDU_PAYLOAD + 512 - 1) / 512)
struct cxgbi_task_data {
+#define CXGBI_TASK_SGL_CHECKED 0x1
+#define CXGBI_TASK_SGL_COPY 0x2
+ u8 flags;
unsigned short nr_frags;
- struct page_frag frags[MAX_PDU_FRAGS];
+ struct page_frag frags[MAX_SKB_FRAGS];
struct sk_buff *skb;
unsigned int dlen;
unsigned int offset;
unsigned int count;
unsigned int sgoffset;
+ u32 total_count;
+ u32 total_offset;
+ u32 max_xmit_dlength;
struct cxgbi_task_tag_info ttinfo;
};
#define iscsi_task_cxgbi_data(task) \
((task)->dd_data + sizeof(struct iscsi_tcp_task))
+struct cxgbi_iso_info {
+#define CXGBI_ISO_INFO_FSLICE 0x1
+#define CXGBI_ISO_INFO_LSLICE 0x2
+#define CXGBI_ISO_INFO_IMM_ENABLE 0x4
+ u8 flags;
+ u8 op;
+ u8 ahs;
+ u8 num_pdu;
+ u32 mpdu;
+ u32 burst_size;
+ u32 len;
+ u32 segment_offset;
+ u32 datasn_offset;
+ u32 buffer_offset;
+};
+
static inline void *cxgbi_alloc_big_mem(unsigned int size,
gfp_t gfp)
{
diff --git a/drivers/scsi/dpt_i2o.c b/drivers/scsi/dpt_i2o.c
index 0497ef6..f654ad8 100644
--- a/drivers/scsi/dpt_i2o.c
+++ b/drivers/scsi/dpt_i2o.c
@@ -1331,7 +1331,6 @@
printk(KERN_ERR"IOP reset failed - no free memory.\n");
return -ENOMEM;
}
- memset(status,0,4);
msg[0]=EIGHT_WORD_MSG_SIZE|SGL_OFFSET_0;
msg[1]=I2O_CMD_ADAPTER_RESET<<24|HOST_TID<<12|ADAPTER_TID;
@@ -2784,7 +2783,6 @@
pHba->name);
return -ENOMEM;
}
- memset(status, 0, 4);
writel(EIGHT_WORD_MSG_SIZE| SGL_OFFSET_6, &msg[0]);
writel(I2O_CMD_OUTBOUND_INIT<<24 | HOST_TID<<12 | ADAPTER_TID, &msg[1]);
@@ -2838,7 +2836,6 @@
printk(KERN_ERR "%s: Could not allocate reply pool\n", pHba->name);
return -ENOMEM;
}
- memset(pHba->reply_pool, 0 , pHba->reply_fifo_size * REPLY_FRAME_SIZE * 4);
for(i = 0; i < pHba->reply_fifo_size; i++) {
writel(pHba->reply_pool_pa + (i * REPLY_FRAME_SIZE * 4),
@@ -3073,7 +3070,6 @@
printk(KERN_WARNING "SysTab Set failed. Out of memory.\n");
return -ENOMEM;
}
- memset(sys_tbl, 0, sys_tbl_len);
sys_tbl->num_entries = hba_count;
sys_tbl->version = I2OVERSION;
diff --git a/drivers/scsi/esas2r/esas2r.h b/drivers/scsi/esas2r/esas2r.h
index 7f43b95..e30d2f1 100644
--- a/drivers/scsi/esas2r/esas2r.h
+++ b/drivers/scsi/esas2r/esas2r.h
@@ -1225,8 +1225,9 @@
/* req_table entry should be NULL at this point - if not, halt */
- if (a->req_table[LOWORD(vrq->scsi.handle)])
+ if (a->req_table[LOWORD(vrq->scsi.handle)]) {
esas2r_bugon();
+ }
/* fill in the table for this handle so we can get back to the
* request.
diff --git a/drivers/scsi/esas2r/esas2r_log.c b/drivers/scsi/esas2r/esas2r_log.c
index 65fdf22..b545798 100644
--- a/drivers/scsi/esas2r/esas2r_log.c
+++ b/drivers/scsi/esas2r/esas2r_log.c
@@ -75,7 +75,7 @@
/* A lock to protect the shared buffer used for formatting messages. */
static DEFINE_SPINLOCK(event_buffer_lock);
-/**
+/*
* translates an esas2r-defined logging event level to a kernel logging level.
*
* @param [in] level the esas2r-defined logging event level to translate
@@ -101,7 +101,7 @@
}
}
-/**
+/*
* the master logging function. this function will format the message as
* outlined by the formatting string, the input device information and the
* substitution arguments and output the resulting string to the system log.
@@ -170,7 +170,7 @@
return 0;
}
-/**
+/*
* formats and logs a message to the system log.
*
* @param [in] level the event level of the message
@@ -193,7 +193,7 @@
return retval;
}
-/**
+/*
* formats and logs a message to the system log. this message will include
* device information.
*
@@ -221,7 +221,7 @@
return retval;
}
-/**
+/*
* formats and logs a message to the system log. this message will include
* device information.
*
diff --git a/drivers/scsi/fcoe/fcoe.c b/drivers/scsi/fcoe/fcoe.c
index cb41d16..0f92749 100644
--- a/drivers/scsi/fcoe/fcoe.c
+++ b/drivers/scsi/fcoe/fcoe.c
@@ -645,7 +645,7 @@
return 0;
}
-/**
+/*
* fcoe_netdev_features_change - Updates the lport's offload flags based
* on the LLD netdev's FCoE feature flags
*/
@@ -2029,7 +2029,7 @@
/**
* fcoe_ctlr_mode() - Switch FIP mode
- * @cdev: The FCoE Controller that is being modified
+ * @ctlr_dev: The FCoE Controller that is being modified
*
* When the FIP mode has been changed we need to update
* the multicast addresses to ensure we get the correct
@@ -2136,9 +2136,7 @@
/**
* fcoe_dcb_create() - Initialize DCB attributes and hooks
- * @netdev: The net_device object of the L2 link that should be queried
- * @port: The fcoe_port to bind FCoE APP priority with
- * @
+ * @fcoe: The new FCoE interface
*/
static void fcoe_dcb_create(struct fcoe_interface *fcoe)
{
@@ -2609,7 +2607,7 @@
fc_lport_logo_resp(seq, fp, lport);
}
-/**
+/*
* fcoe_elsct_send - FCoE specific ELS handler
*
* This does special case handling of FIP encapsualted ELS exchanges for FCoE,
diff --git a/drivers/scsi/fcoe/fcoe_ctlr.c b/drivers/scsi/fcoe/fcoe_ctlr.c
index 1791a39..85c7959 100644
--- a/drivers/scsi/fcoe/fcoe_ctlr.c
+++ b/drivers/scsi/fcoe/fcoe_ctlr.c
@@ -134,6 +134,7 @@
/**
* fcoe_ctlr_init() - Initialize the FCoE Controller instance
* @fip: The FCoE controller to initialize
+ * @mode: FIP mode to set
*/
void fcoe_ctlr_init(struct fcoe_ctlr *fip, enum fip_mode mode)
{
@@ -336,7 +337,7 @@
printk(KERN_NOTICE "libfcoe: host%d: "
"FIP Fibre-Channel Forwarder MAC %pM deselected\n",
fip->lp->host->host_no, fip->dest_addr);
- memset(fip->dest_addr, 0, ETH_ALEN);
+ eth_zero_addr(fip->dest_addr);
}
if (sel) {
printk(KERN_INFO "libfcoe: host%d: FIP selected "
@@ -587,6 +588,7 @@
/**
* fcoe_ctlr_encaps() - Encapsulate an ELS frame for FIP, without sending it
* @fip: The FCoE controller for the ELS frame
+ * @lport: The local port
* @dtype: The FIP descriptor type for the frame
* @skb: The FCoE ELS frame including FC header but no FCoE headers
* @d_id: The destination port ID.
@@ -1302,7 +1304,7 @@
/**
* fcoe_ctlr_recv_els() - Handle an incoming link reset frame
* @fip: The FCoE controller that received the frame
- * @fh: The received FIP header
+ * @skb: The received FIP packet
*
* There may be multiple VN_Port descriptors.
* The overall length has already been checked.
@@ -1775,7 +1777,7 @@
/**
* fcoe_ctlr_timeout() - FIP timeout handler
- * @arg: The FCoE controller that timed out
+ * @t: Timer context use to obtain the controller reference
*/
static void fcoe_ctlr_timeout(struct timer_list *t)
{
@@ -1887,6 +1889,7 @@
/**
* fcoe_ctlr_recv_flogi() - Snoop pre-FIP receipt of FLOGI response
* @fip: The FCoE controller
+ * @lport: The local port
* @fp: The FC frame to snoop
*
* Snoop potential response to FLOGI or even incoming FLOGI.
@@ -2158,7 +2161,7 @@
/**
* fcoe_ctlr_disc_stop_locked() - stop discovery in VN2VN mode
- * @fip: The FCoE controller
+ * @lport: The local port
*
* Called with ctlr_mutex held.
*/
@@ -2179,7 +2182,7 @@
/**
* fcoe_ctlr_disc_stop() - stop discovery in VN2VN mode
- * @fip: The FCoE controller
+ * @lport: The local port
*
* Called through the local port template for discovery.
* Called without the ctlr_mutex held.
@@ -2195,7 +2198,7 @@
/**
* fcoe_ctlr_disc_stop_final() - stop discovery for shutdown in VN2VN mode
- * @fip: The FCoE controller
+ * @lport: The local port
*
* Called through the local port template for discovery.
* Called without the ctlr_mutex held.
@@ -2262,7 +2265,7 @@
* fcoe_ctlr_vn_parse - parse probe request or response
* @fip: The FCoE controller
* @skb: incoming packet
- * @rdata: buffer for resulting parsed VN entry plus fcoe_rport
+ * @frport: parsed FCoE rport from the probe request
*
* Returns non-zero error number on error.
* Does not consume the packet.
@@ -2793,7 +2796,7 @@
* fcoe_ctlr_vlan_parse - parse vlan discovery request or response
* @fip: The FCoE controller
* @skb: incoming packet
- * @rdata: buffer for resulting parsed VLAN entry plus fcoe_rport
+ * @frport: parsed FCoE rport from the probe request
*
* Returns non-zero error number on error.
* Does not consume the packet.
@@ -2892,7 +2895,6 @@
* @fip: The FCoE controller
* @sub: sub-opcode for vlan notification or vn2vn vlan notification
* @dest: The destination Ethernet MAC address
- * @min_len: minimum size of the Ethernet payload to be sent
*/
static void fcoe_ctlr_vlan_send(struct fcoe_ctlr *fip,
enum fip_vlan_subcode sub,
@@ -2969,9 +2971,8 @@
/**
* fcoe_ctlr_vlan_recv - vlan request receive handler for VN2VN mode.
- * @lport: The local port
- * @fp: The received frame
- *
+ * @fip: The FCoE controller
+ * @skb: The received FIP packet
*/
static int fcoe_ctlr_vlan_recv(struct fcoe_ctlr *fip, struct sk_buff *skb)
{
@@ -3015,9 +3016,8 @@
fc_frame_free(fp);
}
-/**
- * fcoe_ctlr_disc_recv - start discovery for VN2VN mode.
- * @fip: The FCoE controller
+/*
+ * fcoe_ctlr_disc_start - start discovery for VN2VN mode.
*
* This sets a flag indicating that remote ports should be created
* and started for the peers we discover. We use the disc_callback
diff --git a/drivers/scsi/fcoe/fcoe_transport.c b/drivers/scsi/fcoe/fcoe_transport.c
index a20ddc3..6e187d0 100644
--- a/drivers/scsi/fcoe/fcoe_transport.c
+++ b/drivers/scsi/fcoe/fcoe_transport.c
@@ -382,6 +382,7 @@
/**
* fcoe_check_wait_queue() - Attempt to clear the transmit backlog
* @lport: The local port whose backlog is to be cleared
+ * @skb: The received FIP packet
*
* This empties the wait_queue, dequeues the head of the wait_queue queue
* and calls fcoe_start_io() for each packet. If all skb have been
@@ -439,7 +440,7 @@
/**
* fcoe_queue_timer() - The fcoe queue timer
- * @lport: The local port
+ * @t: Timer context use to obtain the FCoE port
*
* Calls fcoe_check_wait_queue on timeout
*/
@@ -672,6 +673,7 @@
/**
* fcoe_netdev_map_lookup - find the fcoe transport that matches the netdev on which
* it was created
+ * @netdev: The net device that the FCoE interface is on
*
* Returns : ptr to the fcoe transport that supports this netdev or NULL
* if not found.
diff --git a/drivers/scsi/fdomain.h b/drivers/scsi/fdomain.h
index 6f63fc6..93afcee 100644
--- a/drivers/scsi/fdomain.h
+++ b/drivers/scsi/fdomain.h
@@ -103,7 +103,7 @@
#define REG_FIFO_COUNT 14 /* R: FIFO Data Count */
#ifdef CONFIG_PM_SLEEP
-static const struct dev_pm_ops fdomain_pm_ops;
+static const struct dev_pm_ops __maybe_unused fdomain_pm_ops;
#define FDOMAIN_PM_OPS (&fdomain_pm_ops)
#else
#define FDOMAIN_PM_OPS NULL
diff --git a/drivers/scsi/fnic/fnic_scsi.c b/drivers/scsi/fnic/fnic_scsi.c
index 27535c9..03b1805 100644
--- a/drivers/scsi/fnic/fnic_scsi.c
+++ b/drivers/scsi/fnic/fnic_scsi.c
@@ -23,6 +23,7 @@
#include <linux/scatterlist.h>
#include <linux/skbuff.h>
#include <linux/spinlock.h>
+#include <linux/etherdevice.h>
#include <linux/if_ether.h>
#include <linux/if_vlan.h>
#include <linux/delay.h>
@@ -275,7 +276,7 @@
}
if (fnic->ctlr.map_dest) {
- memset(gw_mac, 0xff, ETH_ALEN);
+ eth_broadcast_addr(gw_mac);
format = FCPIO_FLOGI_REG_DEF_DEST;
} else {
memcpy(gw_mac, fnic->ctlr.dest_addr, ETH_ALEN);
diff --git a/drivers/scsi/hisi_sas/hisi_sas_v1_hw.c b/drivers/scsi/hisi_sas/hisi_sas_v1_hw.c
index 09a7669..7922a9b 100644
--- a/drivers/scsi/hisi_sas/hisi_sas_v1_hw.c
+++ b/drivers/scsi/hisi_sas/hisi_sas_v1_hw.c
@@ -1258,8 +1258,10 @@
!(cmplt_hdr_data & CMPLT_HDR_RSPNS_XFRD_MSK)) {
slot_err_v1_hw(hisi_hba, task, slot);
- if (unlikely(slot->abort))
+ if (unlikely(slot->abort)) {
+ sas_task_abort(task);
return;
+ }
goto out;
}
diff --git a/drivers/scsi/hisi_sas/hisi_sas_v2_hw.c b/drivers/scsi/hisi_sas/hisi_sas_v2_hw.c
index 968d387..043f47b 100644
--- a/drivers/scsi/hisi_sas/hisi_sas_v2_hw.c
+++ b/drivers/scsi/hisi_sas/hisi_sas_v2_hw.c
@@ -2404,8 +2404,10 @@
error_info[0], error_info[1],
error_info[2], error_info[3]);
- if (unlikely(slot->abort))
+ if (unlikely(slot->abort)) {
+ sas_task_abort(task);
return;
+ }
goto out;
}
@@ -3300,7 +3302,7 @@
fatal_axi_int_v2_hw
};
-/**
+/*
* There is a limitation in the hip06 chipset that we need
* to map in all mbigen interrupts, even if they are not used.
*/
diff --git a/drivers/scsi/hisi_sas/hisi_sas_v3_hw.c b/drivers/scsi/hisi_sas/hisi_sas_v3_hw.c
index 55e2321..60adf5c 100644
--- a/drivers/scsi/hisi_sas/hisi_sas_v3_hw.c
+++ b/drivers/scsi/hisi_sas/hisi_sas_v3_hw.c
@@ -2235,8 +2235,10 @@
dw0, dw1, complete_hdr->act, dw3,
error_info[0], error_info[1],
error_info[2], error_info[3]);
- if (unlikely(slot->abort))
+ if (unlikely(slot->abort)) {
+ sas_task_abort(task);
return;
+ }
goto out;
}
diff --git a/drivers/scsi/hosts.c b/drivers/scsi/hosts.c
index 7ec91c3..37d1c55 100644
--- a/drivers/scsi/hosts.c
+++ b/drivers/scsi/hosts.c
@@ -272,8 +272,10 @@
if (shost->transportt->create_work_queue) {
snprintf(shost->work_q_name, sizeof(shost->work_q_name),
"scsi_wq_%d", shost->host_no);
- shost->work_q = create_singlethread_workqueue(
- shost->work_q_name);
+ shost->work_q = alloc_workqueue("%s",
+ WQ_SYSFS | __WQ_LEGACY | WQ_MEM_RECLAIM | WQ_UNBOUND,
+ 1, shost->work_q_name);
+
if (!shost->work_q) {
error = -EINVAL;
goto out_free_shost_data;
@@ -487,7 +489,7 @@
}
shost->tmf_work_q = alloc_workqueue("scsi_tmf_%d",
- WQ_UNBOUND | WQ_MEM_RECLAIM,
+ WQ_UNBOUND | WQ_MEM_RECLAIM | WQ_SYSFS,
1, shost->host_no);
if (!shost->tmf_work_q) {
shost_printk(KERN_WARNING, shost,
diff --git a/drivers/scsi/hpsa.c b/drivers/scsi/hpsa.c
index 81d0414..91794a5 100644
--- a/drivers/scsi/hpsa.c
+++ b/drivers/scsi/hpsa.c
@@ -59,7 +59,7 @@
* HPSA_DRIVER_VERSION must be 3 byte values (0-255) separated by '.'
* with an optional trailing '-' followed by a byte value (0-255).
*/
-#define HPSA_DRIVER_VERSION "3.4.20-170"
+#define HPSA_DRIVER_VERSION "3.4.20-200"
#define DRIVER_NAME "HP HPSA Driver (v " HPSA_DRIVER_VERSION ")"
#define HPSA "hpsa"
@@ -2134,6 +2134,7 @@
}
/* configure scsi device based on internal per-device structure */
+#define CTLR_TIMEOUT (120 * HZ)
static int hpsa_slave_configure(struct scsi_device *sdev)
{
struct hpsa_scsi_dev_t *sd;
@@ -2144,17 +2145,21 @@
if (sd) {
sd->was_removed = 0;
+ queue_depth = sd->queue_depth != 0 ?
+ sd->queue_depth : sdev->host->can_queue;
if (sd->external) {
queue_depth = EXTERNAL_QD;
sdev->eh_timeout = HPSA_EH_PTRAID_TIMEOUT;
blk_queue_rq_timeout(sdev->request_queue,
HPSA_EH_PTRAID_TIMEOUT);
- } else {
- queue_depth = sd->queue_depth != 0 ?
- sd->queue_depth : sdev->host->can_queue;
}
- } else
+ if (is_hba_lunid(sd->scsi3addr)) {
+ sdev->eh_timeout = CTLR_TIMEOUT;
+ blk_queue_rq_timeout(sdev->request_queue, CTLR_TIMEOUT);
+ }
+ } else {
queue_depth = sdev->host->can_queue;
+ }
scsi_change_queue_depth(sdev, queue_depth);
@@ -3443,9 +3448,14 @@
struct ErrorInfo *ei = NULL;
struct bmic_sense_storage_box_params *bssbp = NULL;
struct bmic_identify_physical_device *id_phys = NULL;
- struct ext_report_lun_entry *rle = &rlep->LUN[rle_index];
+ struct ext_report_lun_entry *rle;
u16 bmic_device_index = 0;
+ if (rle_index < 0 || rle_index >= HPSA_MAX_PHYS_LUN)
+ return;
+
+ rle = &rlep->LUN[rle_index];
+
encl_dev->eli =
hpsa_get_enclosure_logical_identifier(h, scsi3addr);
@@ -4174,6 +4184,9 @@
int rc;
struct ext_report_lun_entry *rle;
+ if (rle_index < 0 || rle_index >= HPSA_MAX_PHYS_LUN)
+ return;
+
rle = &rlep->LUN[rle_index];
dev->ioaccel_handle = rle->ioaccel_handle;
@@ -4198,7 +4211,12 @@
struct ReportExtendedLUNdata *rlep, int rle_index,
struct bmic_identify_physical_device *id_phys)
{
- struct ext_report_lun_entry *rle = &rlep->LUN[rle_index];
+ struct ext_report_lun_entry *rle;
+
+ if (rle_index < 0 || rle_index >= HPSA_MAX_PHYS_LUN)
+ return;
+
+ rle = &rlep->LUN[rle_index];
if ((rle->device_flags & 0x08) && this_device->ioaccel_handle)
this_device->hba_ioaccel_enabled = 1;
@@ -4420,7 +4438,8 @@
/*
* Skip over some devices such as a spare.
*/
- if (!tmpdevice->external && physical_device) {
+ if (phys_dev_index >= 0 && !tmpdevice->external &&
+ physical_device) {
skip_device = hpsa_skip_device(h, lunaddrbytes,
&physdev_list->LUN[phys_dev_index]);
if (skip_device)
diff --git a/drivers/scsi/hpsa.h b/drivers/scsi/hpsa.h
index f8c88fc..6b87d98 100644
--- a/drivers/scsi/hpsa.h
+++ b/drivers/scsi/hpsa.h
@@ -57,7 +57,7 @@
bool added_to_port;
};
-#define EXTERNAL_QD 7
+#define EXTERNAL_QD 128
struct hpsa_scsi_dev_t {
unsigned int devtype;
int bus, target, lun; /* as presented to the OS */
diff --git a/drivers/scsi/ibmvscsi/ibmvfc.c b/drivers/scsi/ibmvscsi/ibmvfc.c
index 635f6f9..77f4d37 100644
--- a/drivers/scsi/ibmvscsi/ibmvfc.c
+++ b/drivers/scsi/ibmvscsi/ibmvfc.c
@@ -1344,7 +1344,7 @@
}
/**
- * ibmvfc_map_sg_data - Maps dma for a scatterlist and initializes decriptor fields
+ * ibmvfc_map_sg_data - Maps dma for a scatterlist and initializes descriptor fields
* @scmd: struct scsi_cmnd with the scatterlist
* @evt: ibmvfc event struct
* @vfc_cmd: vfc_cmd that contains the memory descriptor
diff --git a/drivers/scsi/ibmvscsi/ibmvscsi.c b/drivers/scsi/ibmvscsi/ibmvscsi.c
index 14f687e..b1f3017 100644
--- a/drivers/scsi/ibmvscsi/ibmvscsi.c
+++ b/drivers/scsi/ibmvscsi/ibmvscsi.c
@@ -669,7 +669,7 @@
}
/**
- * map_sg_data: - Maps dma for a scatterlist and initializes decriptor fields
+ * map_sg_data: - Maps dma for a scatterlist and initializes descriptor fields
* @cmd: struct scsi_cmnd with the scatterlist
* @srp_cmd: srp_cmd that contains the memory descriptor
* @dev: device for which to map dma memory
diff --git a/drivers/scsi/imm.c b/drivers/scsi/imm.c
index 2519fb7..1459b14 100644
--- a/drivers/scsi/imm.c
+++ b/drivers/scsi/imm.c
@@ -903,7 +903,6 @@
w_ctr(ppb, 0x4);
}
return 0; /* Finished */
- break;
default:
printk("imm: Invalid scsi phase\n");
@@ -969,10 +968,8 @@
case 1: /* Have not connected to interface */
dev->cur_cmd = NULL; /* Forget the problem */
return SUCCESS;
- break;
default: /* SCSI command sent, can not abort */
return FAILED;
- break;
}
}
diff --git a/drivers/scsi/ipr.c b/drivers/scsi/ipr.c
index 7d86f4c..b0aa58d 100644
--- a/drivers/scsi/ipr.c
+++ b/drivers/scsi/ipr.c
@@ -670,6 +670,7 @@
/**
* ipr_init_ipr_cmnd - Initialize an IPR Cmnd block
* @ipr_cmd: ipr command struct
+ * @fast_done: fast done function call-back
*
* Return value:
* none
@@ -687,7 +688,7 @@
/**
* __ipr_get_free_ipr_cmnd - Get a free IPR Cmnd block
- * @ioa_cfg: ioa config struct
+ * @hrrq: hrr queue
*
* Return value:
* pointer to ipr command struct
@@ -737,7 +738,6 @@
static void ipr_mask_and_clear_interrupts(struct ipr_ioa_cfg *ioa_cfg,
u32 clr_ints)
{
- volatile u32 int_reg;
int i;
/* Stop new interrupts */
@@ -757,7 +757,7 @@
if (ioa_cfg->sis64)
writel(~0, ioa_cfg->regs.clr_interrupt_reg);
writel(clr_ints, ioa_cfg->regs.clr_interrupt_reg32);
- int_reg = readl(ioa_cfg->regs.sense_interrupt_reg);
+ readl(ioa_cfg->regs.sense_interrupt_reg);
}
/**
@@ -1287,7 +1287,7 @@
/**
* __ipr_format_res_path - Format the resource path for printing.
* @res_path: resource path
- * @buf: buffer
+ * @buffer: buffer
* @len: length of buffer provided
*
* Return value:
@@ -1310,7 +1310,7 @@
* ipr_format_res_path - Format the resource path for printing.
* @ioa_cfg: ioa config struct
* @res_path: resource path
- * @buf: buffer
+ * @buffer: buffer
* @len: length of buffer provided
*
* Return value:
@@ -1391,7 +1391,6 @@
* ipr_clear_res_target - Clear the bit in the bit map representing the target
* for the resource.
* @res: resource entry struct
- * @cfgtew: config table entry wrapper struct
*
* Return value:
* none
@@ -2667,7 +2666,7 @@
/**
* ipr_timeout - An internally generated op has timed out.
- * @ipr_cmd: ipr command struct
+ * @t: Timer context used to fetch ipr command struct
*
* This function blocks host requests and initiates an
* adapter reset.
@@ -2700,7 +2699,7 @@
/**
* ipr_oper_timeout - Adapter timed out transitioning to operational
- * @ipr_cmd: ipr command struct
+ * @t: Timer context used to fetch ipr command struct
*
* This function blocks host requests and initiates an
* adapter reset.
@@ -3484,6 +3483,7 @@
/**
* ipr_show_fw_version - Show the firmware version
* @dev: class device struct
+ * @attr: device attribute (unused)
* @buf: buffer
*
* Return value:
@@ -3518,6 +3518,7 @@
/**
* ipr_show_log_level - Show the adapter's error logging level
* @dev: class device struct
+ * @attr: device attribute (unused)
* @buf: buffer
*
* Return value:
@@ -3540,7 +3541,9 @@
/**
* ipr_store_log_level - Change the adapter's error logging level
* @dev: class device struct
+ * @attr: device attribute (unused)
* @buf: buffer
+ * @count: buffer size
*
* Return value:
* number of bytes printed to buffer
@@ -3571,6 +3574,7 @@
/**
* ipr_store_diagnostics - IOA Diagnostics interface
* @dev: device struct
+ * @attr: device attribute (unused)
* @buf: buffer
* @count: buffer size
*
@@ -3631,7 +3635,8 @@
/**
* ipr_show_adapter_state - Show the adapter's state
- * @class_dev: device struct
+ * @dev: device struct
+ * @attr: device attribute (unused)
* @buf: buffer
*
* Return value:
@@ -3657,6 +3662,7 @@
/**
* ipr_store_adapter_state - Change adapter state
* @dev: device struct
+ * @attr: device attribute (unused)
* @buf: buffer
* @count: buffer size
*
@@ -3708,6 +3714,7 @@
/**
* ipr_store_reset_adapter - Reset the adapter
* @dev: device struct
+ * @attr: device attribute (unused)
* @buf: buffer
* @count: buffer size
*
@@ -3749,6 +3756,7 @@
/**
* ipr_show_iopoll_weight - Show ipr polling mode
* @dev: class device struct
+ * @attr: device attribute (unused)
* @buf: buffer
*
* Return value:
@@ -3772,7 +3780,9 @@
/**
* ipr_store_iopoll_weight - Change the adapter's polling mode
* @dev: class device struct
+ * @attr: device attribute (unused)
* @buf: buffer
+ * @count: buffer size
*
* Return value:
* number of bytes printed to buffer
@@ -3871,7 +3881,7 @@
/**
* ipr_free_ucode_buffer - Frees a microcode download buffer
- * @p_dnld: scatter/gather list pointer
+ * @sglist: scatter/gather list pointer
*
* Free a DMA'able ucode download buffer previously allocated with
* ipr_alloc_ucode_buffer
@@ -4059,7 +4069,8 @@
/**
* ipr_store_update_fw - Update the firmware on the adapter
- * @class_dev: device struct
+ * @dev: device struct
+ * @attr: device attribute (unused)
* @buf: buffer
* @count: buffer size
*
@@ -4139,6 +4150,7 @@
/**
* ipr_show_fw_type - Show the adapter's firmware type.
* @dev: class device struct
+ * @attr: device attribute (unused)
* @buf: buffer
*
* Return value:
@@ -4480,7 +4492,6 @@
* ipr_change_queue_depth - Change the device's queue depth
* @sdev: scsi device struct
* @qdepth: depth to set
- * @reason: calling context
*
* Return value:
* actual depth set
@@ -4650,6 +4661,7 @@
/**
* ipr_show_raw_mode - Show the adapter's raw mode
* @dev: class device struct
+ * @attr: device attribute (unused)
* @buf: buffer
*
* Return value:
@@ -4677,7 +4689,9 @@
/**
* ipr_store_raw_mode - Change the adapter's raw mode
* @dev: class device struct
+ * @attr: device attribute (unused)
* @buf: buffer
+ * @count: buffer size
*
* Return value:
* number of bytes printed to buffer
@@ -5060,7 +5074,7 @@
/**
* ipr_cmnd_is_free - Check if a command is free or not
- * @ipr_cmd ipr command struct
+ * @ipr_cmd: ipr command struct
*
* Returns:
* true / false
@@ -5096,7 +5110,7 @@
/**
* ipr_wait_for_ops - Wait for matching commands to complete
- * @ipr_cmd: ipr command struct
+ * @ioa_cfg: ioa config struct
* @device: device to match (sdev)
* @match: match function to use
*
@@ -5261,6 +5275,7 @@
* ipr_sata_reset - Reset the SATA port
* @link: SATA link to reset
* @classes: class of the attached device
+ * @deadline: unused
*
* This function issues a SATA phy reset to the affected ATA link.
*
@@ -5440,7 +5455,7 @@
/**
* ipr_abort_timeout - An abort task has timed out
- * @ipr_cmd: ipr command struct
+ * @t: Timer context used to fetch ipr command struct
*
* This function handles when an abort task times out. If this
* happens we issue a bus reset since we have resources tied
@@ -5494,7 +5509,7 @@
struct ipr_ioa_cfg *ioa_cfg;
struct ipr_resource_entry *res;
struct ipr_cmd_pkt *cmd_pkt;
- u32 ioasc, int_reg;
+ u32 ioasc;
int i, op_found = 0;
struct ipr_hrr_queue *hrrq;
@@ -5517,7 +5532,7 @@
* by a still not detected EEH error. In such cases, reading a register will
* trigger the EEH recovery infrastructure.
*/
- int_reg = readl(ioa_cfg->regs.sense_interrupt_reg);
+ readl(ioa_cfg->regs.sense_interrupt_reg);
if (!ipr_is_gscsi(res))
return FAILED;
@@ -5569,7 +5584,8 @@
/**
* ipr_eh_abort - Abort a single op
- * @scsi_cmd: scsi command struct
+ * @shost: scsi host struct
+ * @elapsed_time: elapsed time
*
* Return value:
* 0 if scan in progress / 1 if scan is complete
@@ -5696,6 +5712,7 @@
* ipr_isr_eh - Interrupt service routine error handler
* @ioa_cfg: ioa config struct
* @msg: message to log
+ * @number: various meanings depending on the caller/message
*
* Return value:
* none
@@ -5762,7 +5779,6 @@
static int ipr_iopoll(struct irq_poll *iop, int budget)
{
- struct ipr_ioa_cfg *ioa_cfg;
struct ipr_hrr_queue *hrrq;
struct ipr_cmnd *ipr_cmd, *temp;
unsigned long hrrq_flags;
@@ -5770,7 +5786,6 @@
LIST_HEAD(doneq);
hrrq = container_of(iop, struct ipr_hrr_queue, iopoll);
- ioa_cfg = hrrq->ioa_cfg;
spin_lock_irqsave(hrrq->lock, hrrq_flags);
completed_ops = ipr_process_hrrq(hrrq, budget, &doneq);
@@ -6268,8 +6283,7 @@
/**
* ipr_gen_sense - Generate SCSI sense data from an IOASA
- * @ioasa: IOASA
- * @sense_buf: sense data buffer
+ * @ipr_cmd: ipr command struct
*
* Return value:
* none
@@ -6702,7 +6716,7 @@
/**
* ipr_info - Get information about the card/driver
- * @scsi_host: scsi host struct
+ * @host: scsi host struct
*
* Return value:
* pointer to buffer with description string
@@ -7592,7 +7606,7 @@
/**
* ipr_build_mode_sense - Builds a mode sense command
* @ipr_cmd: ipr command struct
- * @res: resource entry struct
+ * @res_handle: resource entry struct
* @parm: Byte 2 of mode sense command
* @dma_addr: DMA address of mode sense buffer
* @xfer_len: Size of DMA buffer
@@ -7939,6 +7953,7 @@
/**
* ipr_ioafp_set_caching_parameters - Issue Set Cache parameters service
* action
+ * @ipr_cmd: ipr command struct
*
* Return value:
* none
@@ -7975,6 +7990,10 @@
/**
* ipr_ioafp_inquiry - Send an Inquiry to the adapter.
* @ipr_cmd: ipr command struct
+ * @flags: flags to send
+ * @page: page to inquire
+ * @dma_addr: DMA address
+ * @xfer_len: transfer data length
*
* This utility function sends an inquiry to the adapter.
*
@@ -8265,7 +8284,7 @@
/**
* ipr_reset_timer_done - Adapter reset timer function
- * @ipr_cmd: ipr command struct
+ * @t: Timer context used to fetch ipr command struct
*
* Description: This function is used in adapter reset processing
* for timing events. If the reset_cmd pointer in the IOA
@@ -8659,7 +8678,6 @@
static int ipr_reset_restore_cfg_space(struct ipr_cmnd *ipr_cmd)
{
struct ipr_ioa_cfg *ioa_cfg = ipr_cmd->ioa_cfg;
- u32 int_reg;
ENTER;
ioa_cfg->pdev->state_saved = true;
@@ -8675,7 +8693,7 @@
if (ioa_cfg->sis64) {
/* Set the adapter to the correct endian mode. */
writel(IPR_ENDIAN_SWAP_KEY, ioa_cfg->regs.endian_swap_reg);
- int_reg = readl(ioa_cfg->regs.endian_swap_reg);
+ readl(ioa_cfg->regs.endian_swap_reg);
}
if (ioa_cfg->ioa_unit_checked) {
@@ -9483,7 +9501,6 @@
* Description: This is the second phase of adapter initialization
* This function takes care of initilizing the adapter to the point
* where it can accept new commands.
-
* Return value:
* 0 on success / -EIO on failure
**/
@@ -9597,7 +9614,7 @@
/**
* ipr_free_all_resources - Free all allocated resources for an adapter.
- * @ipr_cmd: ipr command struct
+ * @ioa_cfg: ioa config struct
*
* This function frees all allocated resources for the
* specified adapter.
@@ -10059,7 +10076,8 @@
/**
* ipr_test_intr - Handle the interrupt generated in ipr_test_msi().
- * @pdev: PCI device struct
+ * @devp: PCI device struct
+ * @irq: IRQ number
*
* Description: Simply set the msi_received flag to 1 indicating that
* Message Signaled Interrupts are supported.
@@ -10085,6 +10103,7 @@
/**
* ipr_test_msi - Test for Message Signaled Interrupt (MSI) support.
+ * @ioa_cfg: ioa config struct
* @pdev: PCI device struct
*
* Description: This routine sets up and initiates a test interrupt to determine
@@ -10097,7 +10116,6 @@
static int ipr_test_msi(struct ipr_ioa_cfg *ioa_cfg, struct pci_dev *pdev)
{
int rc;
- volatile u32 int_reg;
unsigned long lock_flags = 0;
int irq = pci_irq_vector(pdev, 0);
@@ -10108,7 +10126,7 @@
ioa_cfg->msi_received = 0;
ipr_mask_and_clear_interrupts(ioa_cfg, ~IPR_PCII_IOA_TRANS_TO_OPER);
writel(IPR_PCII_IO_DEBUG_ACKNOWLEDGE, ioa_cfg->regs.clr_interrupt_mask_reg32);
- int_reg = readl(ioa_cfg->regs.sense_interrupt_mask_reg);
+ readl(ioa_cfg->regs.sense_interrupt_mask_reg);
spin_unlock_irqrestore(ioa_cfg->host->host_lock, lock_flags);
rc = request_irq(irq, ipr_test_intr, 0, IPR_NAME, ioa_cfg);
@@ -10119,7 +10137,7 @@
dev_info(&pdev->dev, "IRQ assigned: %d\n", irq);
writel(IPR_PCII_IO_DEBUG_ACKNOWLEDGE, ioa_cfg->regs.sense_interrupt_reg32);
- int_reg = readl(ioa_cfg->regs.sense_interrupt_reg);
+ readl(ioa_cfg->regs.sense_interrupt_reg);
wait_event_timeout(ioa_cfg->msi_wait_q, ioa_cfg->msi_received, HZ);
spin_lock_irqsave(ioa_cfg->host->host_lock, lock_flags);
ipr_mask_and_clear_interrupts(ioa_cfg, ~IPR_PCII_IOA_TRANS_TO_OPER);
@@ -10530,6 +10548,8 @@
/**
* ipr_probe - Adapter hot plug add entry point
+ * @pdev: pci device struct
+ * @dev_id: pci device ID
*
* Return value:
* 0 on success / non-zero on failure
@@ -10786,6 +10806,7 @@
/**
* ipr_halt_done - Shutdown prepare completion
+ * @ipr_cmd: ipr command struct
*
* Return value:
* none
@@ -10797,6 +10818,9 @@
/**
* ipr_halt - Issue shutdown prepare to all adapters
+ * @nb: Notifier block
+ * @event: Notifier event
+ * @buf: Notifier data (unused)
*
* Return value:
* NOTIFY_OK on success / NOTIFY_DONE on failure
diff --git a/drivers/scsi/ipr.h b/drivers/scsi/ipr.h
index 9a0d3d7..783ee03 100644
--- a/drivers/scsi/ipr.h
+++ b/drivers/scsi/ipr.h
@@ -1684,7 +1684,7 @@
struct ipr_dump_location_entry {
struct ipr_dump_entry_header hdr;
u8 location[20];
-}__attribute__((packed));
+}__attribute__((packed, aligned (4)));
struct ipr_dump_trace_entry {
struct ipr_dump_entry_header hdr;
@@ -1708,7 +1708,7 @@
struct ipr_dump_location_entry location_entry;
struct ipr_dump_ioa_type_entry ioa_type_entry;
struct ipr_dump_trace_entry trace_entry;
-}__attribute__((packed));
+}__attribute__((packed, aligned (4)));
struct ipr_ioa_dump {
struct ipr_dump_entry_header hdr;
diff --git a/drivers/scsi/ips.c b/drivers/scsi/ips.c
index f256729..2e6077c 100644
--- a/drivers/scsi/ips.c
+++ b/drivers/scsi/ips.c
@@ -2239,7 +2239,7 @@
major = 0;
minor = 0;
- strncpy(ha->bios_version, " ?", 8);
+ memcpy(ha->bios_version, " ?", 8);
if (ha->pcidev->device == IPS_DEVICEID_COPPERHEAD) {
if (IPS_USE_MEMIO(ha)) {
@@ -3515,11 +3515,11 @@
inquiry.Flags[1] =
IPS_SCSI_INQ_WBus16 |
IPS_SCSI_INQ_Sync;
- strncpy(inquiry.VendorId, "IBM ",
+ memcpy(inquiry.VendorId, "IBM ",
8);
- strncpy(inquiry.ProductId,
+ memcpy(inquiry.ProductId,
"SERVERAID ", 16);
- strncpy(inquiry.ProductRevisionLevel,
+ memcpy(inquiry.ProductRevisionLevel,
"1.00", 4);
ips_scmd_buf_write(scb->scsi_cmd,
@@ -4036,9 +4036,9 @@
inquiry.Flags[0] = IPS_SCSI_INQ_Address16;
inquiry.Flags[1] =
IPS_SCSI_INQ_WBus16 | IPS_SCSI_INQ_Sync | IPS_SCSI_INQ_CmdQue;
- strncpy(inquiry.VendorId, "IBM ", 8);
- strncpy(inquiry.ProductId, "SERVERAID ", 16);
- strncpy(inquiry.ProductRevisionLevel, "1.00", 4);
+ memcpy(inquiry.VendorId, "IBM ", 8);
+ memcpy(inquiry.ProductId, "SERVERAID ", 16);
+ memcpy(inquiry.ProductRevisionLevel, "1.00", 4);
ips_scmd_buf_write(scb->scsi_cmd, &inquiry, sizeof (inquiry));
@@ -4697,7 +4697,6 @@
uint8_t Isr;
uint8_t Cbsp;
uint8_t PostByte[IPS_MAX_POST_BYTES];
- uint8_t ConfigByte[IPS_MAX_CONFIG_BYTES];
int i, j;
METHOD_TRACE("ips_init_copperhead", 1);
@@ -4742,7 +4741,7 @@
/* error occurred */
return (0);
- ConfigByte[i] = inb(ha->io_addr + IPS_REG_ISPR);
+ inb(ha->io_addr + IPS_REG_ISPR);
outb(Isr, ha->io_addr + IPS_REG_HISR);
}
@@ -4791,7 +4790,6 @@
uint8_t Isr = 0;
uint8_t Cbsp;
uint8_t PostByte[IPS_MAX_POST_BYTES];
- uint8_t ConfigByte[IPS_MAX_CONFIG_BYTES];
int i, j;
METHOD_TRACE("ips_init_copperhead_memio", 1);
@@ -4836,7 +4834,7 @@
/* error occurred */
return (0);
- ConfigByte[i] = readb(ha->mem_ptr + IPS_REG_ISPR);
+ readb(ha->mem_ptr + IPS_REG_ISPR);
writeb(Isr, ha->mem_ptr + IPS_REG_HISR);
}
@@ -5622,10 +5620,10 @@
/* change values (as needed) */
ha->nvram->operating_system = IPS_OS_LINUX;
ha->nvram->adapter_type = ha->ad_type;
- strncpy((char *) ha->nvram->driver_high, IPS_VERSION_HIGH, 4);
- strncpy((char *) ha->nvram->driver_low, IPS_VERSION_LOW, 4);
- strncpy((char *) ha->nvram->bios_high, ha->bios_version, 4);
- strncpy((char *) ha->nvram->bios_low, ha->bios_version + 4, 4);
+ memcpy((char *) ha->nvram->driver_high, IPS_VERSION_HIGH, 4);
+ memcpy((char *) ha->nvram->driver_low, IPS_VERSION_LOW, 4);
+ memcpy((char *) ha->nvram->bios_high, ha->bios_version, 4);
+ memcpy((char *) ha->nvram->bios_low, ha->bios_version + 4, 4);
ha->nvram->versioning = 0; /* Indicate the Driver Does Not Support Versioning */
@@ -6835,8 +6833,6 @@
uint32_t mem_addr;
uint32_t io_len;
uint32_t mem_len;
- uint8_t bus;
- uint8_t func;
int j;
int index;
dma_addr_t dma_address;
@@ -6856,10 +6852,6 @@
if (index >= IPS_MAX_ADAPTERS)
return -1;
- /* stuff that we get in dev */
- bus = pci_dev->bus->number;
- func = pci_dev->devfn;
-
/* Init MEM/IO addresses to 0 */
mem_addr = 0;
io_addr = 0;
diff --git a/drivers/scsi/isci/request.c b/drivers/scsi/isci/request.c
index 343d24c..6561a07 100644
--- a/drivers/scsi/isci/request.c
+++ b/drivers/scsi/isci/request.c
@@ -3444,7 +3444,7 @@
int isci_request_execute(struct isci_host *ihost, struct isci_remote_device *idev,
struct sas_task *task, u16 tag)
{
- enum sci_status status = SCI_FAILURE_UNSUPPORTED_PROTOCOL;
+ enum sci_status status;
struct isci_request *ireq;
unsigned long flags;
int ret = 0;
diff --git a/drivers/scsi/libfc/fc_disc.c b/drivers/scsi/libfc/fc_disc.c
index 2b865c6..19721db 100644
--- a/drivers/scsi/libfc/fc_disc.c
+++ b/drivers/scsi/libfc/fc_disc.c
@@ -337,7 +337,7 @@
/**
* fc_disc_gpn_ft_req() - Send Get Port Names by FC-4 type (GPN_FT) request
- * @lport: The discovery context
+ * @disc: The discovery context
*/
static void fc_disc_gpn_ft_req(struct fc_disc *disc)
{
@@ -370,7 +370,7 @@
/**
* fc_disc_gpn_ft_parse() - Parse the body of the dNS GPN_FT response.
- * @lport: The local port the GPN_FT was received on
+ * @disc: The discovery context
* @buf: The GPN_FT response buffer
* @len: The size of response buffer
*
@@ -488,7 +488,7 @@
* fc_disc_gpn_ft_resp() - Handle a response frame from Get Port Names (GPN_FT)
* @sp: The sequence that the GPN_FT response was received on
* @fp: The GPN_FT response frame
- * @lp_arg: The discovery context
+ * @disc_arg: The discovery context
*
* Locking Note: This function is called without disc mutex held, and
* should do all its processing with the mutex held
diff --git a/drivers/scsi/libfc/fc_exch.c b/drivers/scsi/libfc/fc_exch.c
index 52e8666..16eb3b6 100644
--- a/drivers/scsi/libfc/fc_exch.c
+++ b/drivers/scsi/libfc/fc_exch.c
@@ -49,6 +49,8 @@
* @total_exches: Total allocated exchanges
* @lock: Exch pool lock
* @ex_list: List of exchanges
+ * @left: Cache of free slot in exch array
+ * @right: Cache of free slot in exch array
*
* This structure manages per cpu exchanges in array of exchange pointers.
* This array is allocated followed by struct fc_exch_pool memory for
@@ -60,7 +62,6 @@
u16 next_index;
u16 total_exches;
- /* two cache of free slot in exch array */
u16 left;
u16 right;
} ____cacheline_aligned_in_smp;
@@ -74,6 +75,7 @@
* @ep_pool: Reserved exchange pointers
* @pool_max_index: Max exch array index in exch pool
* @pool: Per cpu exch pool
+ * @lport: Local exchange port
* @stats: Statistics structure
*
* This structure is the center for creating exchanges and sequences.
@@ -702,6 +704,9 @@
/**
* fc_invoke_resp() - invoke ep->resp()
+ * @ep: The exchange to be operated on
+ * @fp: The frame pointer to pass through to ->resp()
+ * @sp: The sequence pointer to pass through to ->resp()
*
* Notes:
* It is assumed that after initialization finished (this means the
diff --git a/drivers/scsi/libfc/fc_fcp.c b/drivers/scsi/libfc/fc_fcp.c
index bf2cc96..e11d4f0 100644
--- a/drivers/scsi/libfc/fc_fcp.c
+++ b/drivers/scsi/libfc/fc_fcp.c
@@ -289,6 +289,7 @@
/**
* fc_fcp_retry_cmd() - Retry a fcp_pkt
* @fsp: The FCP packet to be retried
+ * @status_code: The FCP status code to set
*
* Sets the status code to be FC_ERROR and then calls
* fc_fcp_complete_locked() which in turn calls fc_io_compl().
@@ -580,7 +581,7 @@
/**
* fc_fcp_send_data() - Send SCSI data to a target
* @fsp: The FCP packet the data is on
- * @sp: The sequence the data is to be sent on
+ * @seq: The sequence the data is to be sent on
* @offset: The starting offset for this data request
* @seq_blen: The burst length for this data request
*
@@ -1283,7 +1284,7 @@
/**
* fc_lun_reset_send() - Send LUN reset command
- * @data: The FCP packet that identifies the LUN to be reset
+ * @t: Timer context used to fetch the FSP packet
*/
static void fc_lun_reset_send(struct timer_list *t)
{
@@ -1409,7 +1410,7 @@
/**
* fc_fcp_timeout() - Handler for fcp_pkt timeouts
- * @data: The FCP packet that has timed out
+ * @t: Timer context used to fetch the FSP packet
*
* If REC is supported then just issue it and return. The REC exchange will
* complete or time out and recovery can continue at that point. Otherwise,
@@ -1691,6 +1692,7 @@
/**
* fc_fcp_recovery() - Handler for fcp_pkt recovery
* @fsp: The FCP pkt that needs to be aborted
+ * @code: The FCP status code to set
*/
static void fc_fcp_recovery(struct fc_fcp_pkt *fsp, u8 code)
{
@@ -1709,6 +1711,7 @@
* fc_fcp_srr() - Send a SRR request (Sequence Retransmission Request)
* @fsp: The FCP packet the SRR is to be sent on
* @r_ctl: The R_CTL field for the SRR request
+ * @offset: The SRR relative offset
* This is called after receiving status but insufficient data, or
* when expecting status but the request has timed out.
*/
@@ -1851,7 +1854,7 @@
/**
* fc_queuecommand() - The queuecommand function of the SCSI template
* @shost: The Scsi_Host that the command was issued to
- * @cmd: The scsi_cmnd to be executed
+ * @sc_cmd: The scsi_cmnd to be executed
*
* This is the i/o strategy routine, called by the SCSI layer.
*/
diff --git a/drivers/scsi/libfc/fc_lport.c b/drivers/scsi/libfc/fc_lport.c
index 684c5e3..b84dbc3 100644
--- a/drivers/scsi/libfc/fc_lport.c
+++ b/drivers/scsi/libfc/fc_lport.c
@@ -405,7 +405,7 @@
/**
* fc_lport_recv_echo_req() - Handle received ECHO request
* @lport: The local port receiving the ECHO
- * @fp: ECHO request frame
+ * @in_fp: ECHO request frame
*/
static void fc_lport_recv_echo_req(struct fc_lport *lport,
struct fc_frame *in_fp)
@@ -440,7 +440,7 @@
/**
* fc_lport_recv_rnid_req() - Handle received Request Node ID data request
* @lport: The local port receiving the RNID
- * @fp: The RNID request frame
+ * @in_fp: The RNID request frame
*/
static void fc_lport_recv_rnid_req(struct fc_lport *lport,
struct fc_frame *in_fp)
@@ -1325,6 +1325,7 @@
/**
* fc_lport_enter_ns() - register some object with the name server
* @lport: Fibre Channel local port to register
+ * @state: Local port state
*/
static void fc_lport_enter_ns(struct fc_lport *lport, enum fc_lport_state state)
{
@@ -1423,6 +1424,7 @@
/**
* fc_lport_enter_ms() - management server commands
* @lport: Fibre Channel local port to register
+ * @state: Local port state
*/
static void fc_lport_enter_ms(struct fc_lport *lport, enum fc_lport_state state)
{
@@ -1932,6 +1934,7 @@
* @job: The BSG Passthrough job
* @lport: The local port sending the request
* @did: The destination port id
+ * @tov: The timeout period (in ms)
*/
static int fc_lport_els_request(struct bsg_job *job,
struct fc_lport *lport,
diff --git a/drivers/scsi/libfc/fc_rport.c b/drivers/scsi/libfc/fc_rport.c
index 278d15f..18663a8 100644
--- a/drivers/scsi/libfc/fc_rport.c
+++ b/drivers/scsi/libfc/fc_rport.c
@@ -121,7 +121,7 @@
/**
* fc_rport_create() - Create a new remote port
* @lport: The local port this remote port will be associated with
- * @ids: The identifiers for the new remote port
+ * @port_id: The identifiers for the new remote port
*
* The remote port will start in the INIT state.
*/
@@ -1445,7 +1445,7 @@
* fc_rport_logo_resp() - Handler for logout (LOGO) responses
* @sp: The sequence the LOGO was on
* @fp: The LOGO response frame
- * @lport_arg: The local port
+ * @rdata_arg: The remote port
*/
static void fc_rport_logo_resp(struct fc_seq *sp, struct fc_frame *fp,
void *rdata_arg)
diff --git a/drivers/scsi/libsas/sas_ata.c b/drivers/scsi/libsas/sas_ata.c
index 5d716d3..1b93332 100644
--- a/drivers/scsi/libsas/sas_ata.c
+++ b/drivers/scsi/libsas/sas_ata.c
@@ -507,10 +507,23 @@
spin_unlock_irqrestore(&ha->lock, flags);
}
+static int sas_ata_prereset(struct ata_link *link, unsigned long deadline)
+{
+ struct ata_port *ap = link->ap;
+ struct domain_device *dev = ap->private_data;
+ struct sas_phy *local_phy = sas_get_local_phy(dev);
+ int res = 0;
+
+ if (!local_phy->enabled || test_bit(SAS_DEV_GONE, &dev->state))
+ res = -ENOENT;
+ sas_put_local_phy(local_phy);
+
+ return res;
+}
+
static struct ata_port_operations sas_sata_ops = {
- .prereset = ata_std_prereset,
+ .prereset = sas_ata_prereset,
.hardreset = sas_ata_hard_reset,
- .postreset = ata_std_postreset,
.error_handler = ata_std_error_handler,
.post_internal_cmd = sas_ata_post_internal,
.qc_defer = ata_std_qc_defer,
diff --git a/drivers/scsi/libsas/sas_expander.c b/drivers/scsi/libsas/sas_expander.c
index ab671cd..b7d1b1e 100644
--- a/drivers/scsi/libsas/sas_expander.c
+++ b/drivers/scsi/libsas/sas_expander.c
@@ -427,7 +427,7 @@
static int sas_expander_discover(struct domain_device *dev)
{
struct expander_device *ex = &dev->ex_dev;
- int res = -ENOMEM;
+ int res;
ex->ex_phy = kcalloc(ex->num_phys, sizeof(*ex->ex_phy), GFP_KERNEL);
if (!ex->ex_phy)
diff --git a/drivers/scsi/lpfc/lpfc.h b/drivers/scsi/lpfc/lpfc.h
index c3ceb6e..549adfa 100644
--- a/drivers/scsi/lpfc/lpfc.h
+++ b/drivers/scsi/lpfc/lpfc.h
@@ -627,6 +627,14 @@
enum ras_state state; /* RAS logging running state */
};
+#define DBG_LOG_STR_SZ 256
+#define DBG_LOG_SZ 256
+
+struct dbg_log_ent {
+ char log[DBG_LOG_STR_SZ];
+ u64 t_ns;
+};
+
enum lpfc_irq_chann_mode {
/* Assign IRQs to all possible cpus that have hardware queues */
NORMAL_MODE,
@@ -709,6 +717,9 @@
struct workqueue_struct *wq;
struct delayed_work eq_delay_work;
+#define LPFC_IDLE_STAT_DELAY 1000
+ struct delayed_work idle_stat_delay_work;
+
struct lpfc_sli sli;
uint8_t pci_dev_grp; /* lpfc PCI dev group: 0x0, 0x1, 0x2,... */
uint32_t sli_rev; /* SLI2, SLI3, or SLI4 */
@@ -1237,6 +1248,10 @@
struct scsi_host_template port_template;
/* SCSI host template information - for all vports */
struct scsi_host_template vport_template;
+ atomic_t dbg_log_idx;
+ atomic_t dbg_log_cnt;
+ atomic_t dbg_log_dmping;
+ struct dbg_log_ent dbg_log[DBG_LOG_SZ];
};
static inline struct Scsi_Host *
diff --git a/drivers/scsi/lpfc/lpfc_bsg.c b/drivers/scsi/lpfc/lpfc_bsg.c
index 0ea03ae..1d88fed 100644
--- a/drivers/scsi/lpfc/lpfc_bsg.c
+++ b/drivers/scsi/lpfc/lpfc_bsg.c
@@ -2404,33 +2404,27 @@
union lpfc_sli4_cfg_shdr *shdr;
uint32_t shdr_status, shdr_add_status;
struct diag_status *diag_status_reply;
- int mbxstatus, rc = 0;
+ int mbxstatus, rc = -ENODEV, rc1 = 0;
shost = fc_bsg_to_shost(job);
- if (!shost) {
- rc = -ENODEV;
+ if (!shost)
goto job_error;
- }
- vport = shost_priv(shost);
- if (!vport) {
- rc = -ENODEV;
- goto job_error;
- }
- phba = vport->phba;
- if (!phba) {
- rc = -ENODEV;
- goto job_error;
- }
- if (phba->sli_rev < LPFC_SLI_REV4) {
- rc = -ENODEV;
+ vport = shost_priv(shost);
+ if (!vport)
goto job_error;
- }
+
+ phba = vport->phba;
+ if (!phba)
+ goto job_error;
+
+
+ if (phba->sli_rev < LPFC_SLI_REV4)
+ goto job_error;
+
if (bf_get(lpfc_sli_intf_if_type, &phba->sli4_hba.sli_intf) <
- LPFC_SLI_INTF_IF_TYPE_2) {
- rc = -ENODEV;
+ LPFC_SLI_INTF_IF_TYPE_2)
goto job_error;
- }
if (job->request_len < sizeof(struct fc_bsg_request) +
sizeof(struct sli4_link_diag)) {
@@ -2465,8 +2459,10 @@
alloc_len = lpfc_sli4_config(phba, pmboxq, LPFC_MBOX_SUBSYSTEM_FCOE,
LPFC_MBOX_OPCODE_FCOE_LINK_DIAG_STATE,
req_len, LPFC_SLI4_MBX_EMBED);
- if (alloc_len != req_len)
+ if (alloc_len != req_len) {
+ rc = -ENOMEM;
goto link_diag_test_exit;
+ }
run_link_diag_test = &pmboxq->u.mqe.un.link_diag_test;
bf_set(lpfc_mbx_run_diag_test_link_num, &run_link_diag_test->u.req,
@@ -2515,7 +2511,7 @@
diag_status_reply->shdr_add_status = shdr_add_status;
link_diag_test_exit:
- rc = lpfc_sli4_bsg_set_link_diag_state(phba, 0);
+ rc1 = lpfc_sli4_bsg_set_link_diag_state(phba, 0);
if (pmboxq)
mempool_free(pmboxq, phba->mbox_mem_pool);
@@ -2524,6 +2520,8 @@
job_error:
/* make error code available to userspace */
+ if (rc1 && !rc)
+ rc = rc1;
bsg_reply->result = rc;
/* complete the job back to userspace if no error */
if (rc == 0)
@@ -4306,6 +4304,7 @@
case COMN_OPCODE_GET_CNTL_ADDL_ATTRIBUTES:
case COMN_OPCODE_GET_CNTL_ATTRIBUTES:
case COMN_OPCODE_GET_PROFILE_CONFIG:
+ case COMN_OPCODE_SET_FEATURES:
lpfc_printf_log(phba, KERN_INFO, LOG_LIBDFC,
"3106 Handled SLI_CONFIG "
"subsys_comn, opcode:x%x\n",
diff --git a/drivers/scsi/lpfc/lpfc_bsg.h b/drivers/scsi/lpfc/lpfc_bsg.h
index d170813..2dc7124 100644
--- a/drivers/scsi/lpfc/lpfc_bsg.h
+++ b/drivers/scsi/lpfc/lpfc_bsg.h
@@ -225,6 +225,10 @@
uint32_t reserved5;
};
+#define LPFC_CSF_BOOT_DEV 0x1D
+#define LPFC_CSF_QUERY 0
+#define LPFC_CSF_SAVE 1
+
struct lpfc_sli_config_emb0_subsys {
struct lpfc_sli_config_hdr sli_config_hdr;
#define LPFC_MBX_SLI_CONFIG_MAX_MSE 19
@@ -243,6 +247,15 @@
#define FCOE_OPCODE_ADD_FCF 0x09
#define FCOE_OPCODE_SET_DPORT_MODE 0x27
#define FCOE_OPCODE_GET_DPORT_RESULTS 0x28
+ uint32_t timeout; /* comn_set_feature timeout */
+ uint32_t request_length; /* comn_set_feature request len */
+ uint32_t version; /* comn_set_feature version */
+ uint32_t csf_feature; /* comn_set_feature feature */
+ uint32_t word69; /* comn_set_feature parameter len */
+ uint32_t word70; /* comn_set_feature parameter val0 */
+#define lpfc_emb0_subcmnd_csf_p0_SHIFT 0
+#define lpfc_emb0_subcmnd_csf_p0_MASK 0x3
+#define lpfc_emb0_subcmnd_csf_p0_WORD word70
};
struct lpfc_sli_config_emb1_subsys {
@@ -261,6 +274,7 @@
#define COMN_OPCODE_WRITE_OBJECT 0xAC
#define COMN_OPCODE_READ_OBJECT_LIST 0xAD
#define COMN_OPCODE_DELETE_OBJECT 0xAE
+#define COMN_OPCODE_SET_FEATURES 0xBF
#define COMN_OPCODE_GET_CNTL_ADDL_ATTRIBUTES 0x79
#define COMN_OPCODE_GET_CNTL_ATTRIBUTES 0x20
uint32_t timeout;
diff --git a/drivers/scsi/lpfc/lpfc_crtn.h b/drivers/scsi/lpfc/lpfc_crtn.h
index 9ee6b93..782f6f7 100644
--- a/drivers/scsi/lpfc/lpfc_crtn.h
+++ b/drivers/scsi/lpfc/lpfc_crtn.h
@@ -386,7 +386,7 @@
int lpfc_link_reset(struct lpfc_vport *vport);
/* Function prototypes. */
-int lpfc_check_pci_resettable(const struct lpfc_hba *phba);
+int lpfc_check_pci_resettable(struct lpfc_hba *phba);
const char* lpfc_info(struct Scsi_Host *);
int lpfc_scan_finished(struct Scsi_Host *, unsigned long);
diff --git a/drivers/scsi/lpfc/lpfc_ct.c b/drivers/scsi/lpfc/lpfc_ct.c
index 69d4710d..dd9f2bf 100644
--- a/drivers/scsi/lpfc/lpfc_ct.c
+++ b/drivers/scsi/lpfc/lpfc_ct.c
@@ -300,7 +300,7 @@
return 0;
}
-/**
+/*
* lpfc_gen_req - Build and issue a GEN_REQUEST command to the SLI Layer
* @vport: pointer to a host virtual N_Port data structure.
* @bmp: Pointer to BPL for SLI command
@@ -394,7 +394,7 @@
return 0;
}
-/**
+/*
* lpfc_ct_cmd - Build and issue a CT command
* @vport: pointer to a host virtual N_Port data structure.
* @inmp: Pointer to data buffer for response data.
@@ -750,7 +750,7 @@
if (vport->fc_flag & FC_RSCN_MODE)
lpfc_els_flush_rscn(vport);
lpfc_vport_set_state(vport, FC_VPORT_FAILED);
- lpfc_printf_vlog(vport, KERN_ERR, LOG_ELS,
+ lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
"0257 GID_FT Query error: 0x%x 0x%x\n",
irsp->ulpStatus, vport->fc_ns_retry);
} else {
@@ -811,7 +811,7 @@
} else {
/* NameServer Rsp Error */
- lpfc_printf_vlog(vport, KERN_ERR, LOG_DISCOVERY,
+ lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
"0241 NameServer Rsp Error "
"Data: x%x x%x x%x x%x\n",
CTrsp->CommandResponse.bits.CmdRsp,
@@ -951,7 +951,7 @@
if (vport->fc_flag & FC_RSCN_MODE)
lpfc_els_flush_rscn(vport);
lpfc_vport_set_state(vport, FC_VPORT_FAILED);
- lpfc_printf_vlog(vport, KERN_ERR, LOG_ELS,
+ lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
"4103 GID_FT Query error: 0x%x 0x%x\n",
irsp->ulpStatus, vport->fc_ns_retry);
} else {
@@ -1012,7 +1012,7 @@
}
} else {
/* NameServer Rsp Error */
- lpfc_printf_vlog(vport, KERN_ERR, LOG_DISCOVERY,
+ lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
"4109 NameServer Rsp Error "
"Data: x%x x%x x%x x%x\n",
CTrsp->CommandResponse.bits.CmdRsp,
@@ -1143,7 +1143,7 @@
}
}
}
- lpfc_printf_vlog(vport, KERN_ERR, LOG_DISCOVERY,
+ lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
"0267 NameServer GFF Rsp "
"x%x Error (%d %d) Data: x%x x%x\n",
did, irsp->ulpStatus, irsp->un.ulpWord[4],
@@ -1271,7 +1271,7 @@
}
}
} else
- lpfc_printf_vlog(vport, KERN_ERR, LOG_DISCOVERY,
+ lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
"3065 GFT_ID failed x%08x\n", irsp->ulpStatus);
lpfc_ct_free_iocb(phba, cmdiocb);
@@ -1320,7 +1320,7 @@
irsp->ulpStatus, irsp->un.ulpWord[4], cmdcode);
if (irsp->ulpStatus) {
- lpfc_printf_vlog(vport, KERN_ERR, LOG_DISCOVERY,
+ lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
"0268 NS cmd x%x Error (x%x x%x)\n",
cmdcode, irsp->ulpStatus, irsp->un.ulpWord[4]);
@@ -1843,7 +1843,7 @@
ns_cmd_free_mp:
kfree(mp);
ns_cmd_exit:
- lpfc_printf_vlog(vport, KERN_ERR, LOG_DISCOVERY,
+ lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
"0266 Issue NameServer Req x%x err %d Data: x%x x%x\n",
cmdcode, rc, vport->fc_flag, vport->fc_rscn_id_cnt);
return 1;
@@ -3019,8 +3019,8 @@
* lpfc_fdmi_cmd - Build and send a FDMI cmd to the specified NPort
* @vport: pointer to a host virtual N_Port data structure.
* @ndlp: ndlp to send FDMI cmd to (if NULL use FDMI_DID)
- * cmdcode: FDMI command to send
- * mask: Mask of HBA or PORT Attributes to send
+ * @cmdcode: FDMI command to send
+ * @new_mask: Mask of HBA or PORT Attributes to send
*
* Builds and sends a FDMI command using the CT subsystem.
*/
@@ -3262,7 +3262,7 @@
/**
* lpfc_delayed_disc_tmo - Timeout handler for delayed discovery timer.
- * @ptr - Context object of the timer.
+ * @t: Context object of the timer.
*
* This function set the WORKER_DELAYED_DISC_TMO flag and wake up
* the worker thread.
diff --git a/drivers/scsi/lpfc/lpfc_els.c b/drivers/scsi/lpfc/lpfc_els.c
index 3d67056..85d4e40 100644
--- a/drivers/scsi/lpfc/lpfc_els.c
+++ b/drivers/scsi/lpfc/lpfc_els.c
@@ -100,7 +100,7 @@
return 0;
/* Pending Link Event during Discovery */
- lpfc_printf_vlog(vport, KERN_ERR, LOG_DISCOVERY,
+ lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
"0237 Pending Link Event during "
"Discovery: State x%x\n",
phba->pport->port_state);
@@ -440,8 +440,9 @@
fail:
lpfc_vport_set_state(vport, FC_VPORT_FAILED);
- lpfc_printf_vlog(vport, KERN_ERR, LOG_ELS,
- "0249 Cannot issue Register Fabric login: Err %d\n", err);
+ lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
+ "0249 Cannot issue Register Fabric login: Err %d\n",
+ err);
return -ENXIO;
}
@@ -524,8 +525,8 @@
}
lpfc_vport_set_state(vport, FC_VPORT_FAILED);
- lpfc_printf_vlog(vport, KERN_ERR, LOG_ELS,
- "0289 Issue Register VFI failed: Err %d\n", rc);
+ lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
+ "0289 Issue Register VFI failed: Err %d\n", rc);
return rc;
}
@@ -550,7 +551,7 @@
mboxq = mempool_alloc(phba->mbox_mem_pool, GFP_KERNEL);
if (!mboxq) {
- lpfc_printf_log(phba, KERN_ERR, LOG_DISCOVERY|LOG_MBOX,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"2556 UNREG_VFI mbox allocation failed"
"HBA state x%x\n", phba->pport->port_state);
return -ENOMEM;
@@ -562,7 +563,7 @@
rc = lpfc_sli_issue_mbox(phba, mboxq, MBX_NOWAIT);
if (rc == MBX_NOT_FINISHED) {
- lpfc_printf_log(phba, KERN_ERR, LOG_DISCOVERY|LOG_MBOX,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"2557 UNREG_VFI issue mbox failed rc x%x "
"HBA state x%x\n",
rc, phba->pport->port_state);
@@ -1041,18 +1042,18 @@
if (!(irsp->ulpStatus == IOSTAT_LOCAL_REJECT &&
((irsp->un.ulpWord[4] & IOERR_PARAM_MASK) ==
IOERR_LOOP_OPEN_FAILURE)))
- lpfc_printf_vlog(vport, KERN_ERR, LOG_ELS,
- "2858 FLOGI failure Status:x%x/x%x "
- "TMO:x%x Data x%x x%x\n",
- irsp->ulpStatus, irsp->un.ulpWord[4],
- irsp->ulpTimeout, phba->hba_flag,
- phba->fcf.fcf_flag);
+ lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
+ "2858 FLOGI failure Status:x%x/x%x TMO"
+ ":x%x Data x%x x%x\n",
+ irsp->ulpStatus, irsp->un.ulpWord[4],
+ irsp->ulpTimeout, phba->hba_flag,
+ phba->fcf.fcf_flag);
/* Check for retry */
if (lpfc_els_retry(phba, cmdiocb, rspiocb))
goto out;
- lpfc_printf_vlog(vport, KERN_WARNING, LOG_ELS,
+ lpfc_printf_vlog(vport, KERN_WARNING, LOG_TRACE_EVENT,
"0150 FLOGI failure Status:x%x/x%x "
"xri x%x TMO:x%x\n",
irsp->ulpStatus, irsp->un.ulpWord[4],
@@ -1132,8 +1133,7 @@
else if (!(phba->hba_flag & HBA_FCOE_MODE))
rc = lpfc_cmpl_els_flogi_nport(vport, ndlp, sp);
else {
- lpfc_printf_vlog(vport, KERN_ERR,
- LOG_FIP | LOG_ELS,
+ lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
"2831 FLOGI response with cleared Fabric "
"bit fcf_index 0x%x "
"Switch Name %02x%02x%02x%02x%02x%02x%02x%02x "
@@ -1934,7 +1934,7 @@
ndlp = lpfc_findnode_did(vport, irsp->un.elsreq64.remoteID);
if (!ndlp || !NLP_CHK_NODE_ACT(ndlp) || ndlp != rrq->ndlp) {
- lpfc_printf_vlog(vport, KERN_ERR, LOG_ELS,
+ lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
"2882 RRQ completes to NPort x%x "
"with no ndlp. Data: x%x x%x x%x\n",
irsp->un.elsreq64.remoteID,
@@ -1957,10 +1957,11 @@
(((irsp->un.ulpWord[4]) >> 16 != LSRJT_INVALID_CMD) &&
((irsp->un.ulpWord[4]) >> 16 != LSRJT_UNABLE_TPC)) ||
(phba)->pport->cfg_log_verbose & LOG_ELS)
- lpfc_printf_vlog(vport, KERN_ERR, LOG_ELS,
- "2881 RRQ failure DID:%06X Status:x%x/x%x\n",
- ndlp->nlp_DID, irsp->ulpStatus,
- irsp->un.ulpWord[4]);
+ lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
+ "2881 RRQ failure DID:%06X Status:"
+ "x%x/x%x\n",
+ ndlp->nlp_DID, irsp->ulpStatus,
+ irsp->un.ulpWord[4]);
}
out:
if (rrq)
@@ -2010,7 +2011,7 @@
ndlp = lpfc_findnode_did(vport, irsp->un.elsreq64.remoteID);
if (!ndlp || !NLP_CHK_NODE_ACT(ndlp)) {
- lpfc_printf_vlog(vport, KERN_ERR, LOG_ELS,
+ lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
"0136 PLOGI completes to NPort x%x "
"with no ndlp. Data: x%x x%x x%x\n",
irsp->un.elsreq64.remoteID,
@@ -2059,7 +2060,7 @@
(((irsp->un.ulpWord[4]) >> 16 != LSRJT_INVALID_CMD) &&
((irsp->un.ulpWord[4]) >> 16 != LSRJT_UNABLE_TPC)) ||
(phba)->pport->cfg_log_verbose & LOG_ELS)
- lpfc_printf_vlog(vport, KERN_ERR, LOG_ELS,
+ lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
"2753 PLOGI failure DID:%06X Status:x%x/x%x\n",
ndlp->nlp_DID, irsp->ulpStatus,
irsp->un.ulpWord[4]);
@@ -2237,6 +2238,7 @@
IOCB_t *irsp;
struct lpfc_nodelist *ndlp;
char *mode;
+ u32 loglevel;
/* we pass cmdiocb to state machine which needs rspiocb as well */
cmdiocb->context_un.rsp_iocb = rspiocb;
@@ -2278,13 +2280,16 @@
* could be expected.
*/
if ((vport->fc_flag & FC_FABRIC) ||
- (vport->cfg_enable_fc4_type != LPFC_ENABLE_BOTH))
+ (vport->cfg_enable_fc4_type != LPFC_ENABLE_BOTH)) {
mode = KERN_ERR;
- else
+ loglevel = LOG_TRACE_EVENT;
+ } else {
mode = KERN_INFO;
+ loglevel = LOG_ELS;
+ }
/* PRLI failed */
- lpfc_printf_vlog(vport, mode, LOG_ELS,
+ lpfc_printf_vlog(vport, mode, loglevel,
"2754 PRLI failure DID:%06X Status:x%x/x%x, "
"data: x%x\n",
ndlp->nlp_DID, irsp->ulpStatus,
@@ -2695,7 +2700,7 @@
goto out;
}
/* ADISC failed */
- lpfc_printf_vlog(vport, KERN_ERR, LOG_ELS,
+ lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
"2755 ADISC failure DID:%06X Status:x%x/x%x\n",
ndlp->nlp_DID, irsp->ulpStatus,
irsp->un.ulpWord[4]);
@@ -2853,7 +2858,7 @@
*/
if (irsp->ulpStatus) {
/* LOGO failed */
- lpfc_printf_vlog(vport, KERN_ERR, LOG_ELS,
+ lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
"2756 LOGO failure, No Retry DID:%06X Status:x%x/x%x\n",
ndlp->nlp_DID, irsp->ulpStatus,
irsp->un.ulpWord[4]);
@@ -3597,7 +3602,7 @@
/**
* lpfc_els_retry_delay - Timer function with a ndlp delayed function timer
- * @ptr: holder for the pointer to the timer function associated data (ndlp).
+ * @t: pointer to the timer function associated data (ndlp).
*
* This routine is invoked by the ndlp delayed-function timer to check
* whether there is any pending ELS retry event(s) with the node. If not, it
@@ -3734,7 +3739,7 @@
"2851 Attempt link reset\n");
mbox = mempool_alloc(phba->mbox_mem_pool, GFP_KERNEL);
if (!mbox) {
- lpfc_printf_log(phba, KERN_ERR, LOG_MBOX,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"2852 Failed to allocate mbox memory");
return 1;
}
@@ -3756,7 +3761,7 @@
mbox->vport = vport;
rc = lpfc_sli_issue_mbox(phba, mbox, MBX_NOWAIT);
if ((rc != MBX_BUSY) && (rc != MBX_SUCCESS)) {
- lpfc_printf_log(phba, KERN_ERR, LOG_MBOX,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"2853 Failed to issue INIT_LINK "
"mbox command, rc:x%x\n", rc);
mempool_free(mbox, phba->mbox_mem_pool);
@@ -3860,7 +3865,7 @@
break;
case IOERR_ILLEGAL_COMMAND:
- lpfc_printf_vlog(vport, KERN_ERR, LOG_ELS,
+ lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
"0124 Retry illegal cmd x%x "
"retry:x%x delay:x%x\n",
cmd, cmdiocb->retry, delay);
@@ -3970,7 +3975,8 @@
if ((phba->sli3_options & LPFC_SLI3_NPIV_ENABLED) &&
(cmd == ELS_CMD_FDISC) &&
(stat.un.b.lsRjtRsnCodeExp == LSEXP_OUT_OF_RESOURCE)){
- lpfc_printf_vlog(vport, KERN_ERR, LOG_ELS,
+ lpfc_printf_vlog(vport, KERN_ERR,
+ LOG_TRACE_EVENT,
"0125 FDISC Failed (x%x). "
"Fabric out of resources\n",
stat.un.lsRjtError);
@@ -4009,7 +4015,8 @@
LSEXP_NOTHING_MORE) {
vport->fc_sparam.cmn.bbRcvSizeMsb &= 0xf;
retry = 1;
- lpfc_printf_vlog(vport, KERN_ERR, LOG_ELS,
+ lpfc_printf_vlog(vport, KERN_ERR,
+ LOG_TRACE_EVENT,
"0820 FLOGI Failed (x%x). "
"BBCredit Not Supported\n",
stat.un.lsRjtError);
@@ -4022,7 +4029,8 @@
((stat.un.b.lsRjtRsnCodeExp == LSEXP_INVALID_PNAME) ||
(stat.un.b.lsRjtRsnCodeExp == LSEXP_INVALID_NPORT_ID))
) {
- lpfc_printf_vlog(vport, KERN_ERR, LOG_ELS,
+ lpfc_printf_vlog(vport, KERN_ERR,
+ LOG_TRACE_EVENT,
"0122 FDISC Failed (x%x). "
"Fabric Detected Bad WWN\n",
stat.un.lsRjtError);
@@ -4200,7 +4208,7 @@
}
/* No retry ELS command <elsCmd> to remote NPORT <did> */
if (logerr) {
- lpfc_printf_vlog(vport, KERN_ERR, LOG_ELS,
+ lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
"0137 No retry ELS command x%x to remote "
"NPORT x%x: Out of Resources: Error:x%x/%x\n",
cmd, did, irsp->ulpStatus,
@@ -4499,7 +4507,7 @@
irsp = &rspiocb->iocb;
if (!vport) {
- lpfc_printf_log(phba, KERN_ERR, LOG_ELS,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"3177 ELS response failed\n");
goto out;
}
@@ -4605,7 +4613,7 @@
ndlp->nlp_flag &= ~NLP_REG_LOGIN_SEND;
/* ELS rsp: Cannot issue reg_login for <NPortid> */
- lpfc_printf_vlog(vport, KERN_ERR, LOG_ELS,
+ lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
"0138 ELS rsp: Cannot issue reg_login for x%x "
"Data: x%x x%x x%x\n",
ndlp->nlp_DID, ndlp->nlp_flag, ndlp->nlp_state,
@@ -4843,7 +4851,7 @@
/**
* lpfc_els_rsp_reject - Propare and issue a rjt response iocb command
* @vport: pointer to a virtual N_Port data structure.
- * @rejectError:
+ * @rejectError: reject response to issue
* @oldiocb: pointer to the original lpfc command iocb data structure.
* @ndlp: pointer to a node-list data structure.
* @mbox: pointer to the driver internal queue element for mailbox command.
@@ -6411,8 +6419,8 @@
lcb_context->rx_id = cmdiocb->iocb.ulpContext;
lcb_context->ndlp = lpfc_nlp_get(ndlp);
if (lpfc_sli4_set_beacon(vport, lcb_context, state)) {
- lpfc_printf_vlog(ndlp->vport, KERN_ERR,
- LOG_ELS, "0193 failed to send mail box");
+ lpfc_printf_vlog(ndlp->vport, KERN_ERR, LOG_TRACE_EVENT,
+ "0193 failed to send mail box");
kfree(lcb_context);
lpfc_nlp_put(ndlp);
rjt_err = LSRJT_UNABLE_TPC;
@@ -6621,7 +6629,7 @@
rscn_event_data = kmalloc(sizeof(struct lpfc_rscn_event_header) +
payload_len, GFP_KERNEL);
if (!rscn_event_data) {
- lpfc_printf_vlog(vport, KERN_ERR, LOG_ELS,
+ lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
"0147 Failed to allocate memory for RSCN event\n");
return;
}
@@ -6998,7 +7006,7 @@
/* An FLOGI ELS command <elsCmd> was received from DID <did> in
Loop Mode */
- lpfc_printf_vlog(vport, KERN_ERR, LOG_ELS,
+ lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
"0113 An FLOGI ELS command x%x was "
"received from DID x%x in Loop Mode\n",
cmd, did);
@@ -7879,7 +7887,7 @@
/**
* lpfc_els_timeout - Handler funciton to the els timer
- * @ptr: holder for the timer function associated data.
+ * @t: timer context used to obtain the vport.
*
* This routine is invoked by the ELS timer after timeout. It posts the ELS
* timer timeout event by setting the WORKER_ELS_TMO bit to the work port
@@ -7988,7 +7996,7 @@
list_for_each_entry_safe(piocb, tmp_iocb, &abort_list, dlist) {
cmd = &piocb->iocb;
- lpfc_printf_vlog(vport, KERN_ERR, LOG_ELS,
+ lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
"0127 ELS timeout Data: x%x x%x x%x "
"x%x\n", els_command,
remote_ID, cmd->ulpCommand, cmd->ulpIoTag);
@@ -8098,7 +8106,7 @@
spin_unlock_irqrestore(&phba->hbalock, iflags);
}
if (!list_empty(&abort_list))
- lpfc_printf_vlog(vport, KERN_ERR, LOG_ELS,
+ lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
"3387 abort list for txq not empty\n");
INIT_LIST_HEAD(&abort_list);
@@ -8252,7 +8260,7 @@
* lpfc_send_els_event - Posts unsolicited els event
* @vport: Pointer to vport object.
* @ndlp: Pointer FC node object.
- * @cmd: ELS command code.
+ * @payload: ELS command code type.
*
* This function posts an event when there is an incoming
* unsolicited ELS command.
@@ -8269,7 +8277,7 @@
if (*payload == ELS_CMD_LOGO) {
logo_data = kmalloc(sizeof(struct lpfc_logo_event), GFP_KERNEL);
if (!logo_data) {
- lpfc_printf_vlog(vport, KERN_ERR, LOG_ELS,
+ lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
"0148 Failed to allocate memory "
"for LOGO event\n");
return;
@@ -8279,7 +8287,7 @@
els_data = kmalloc(sizeof(struct lpfc_els_event_header),
GFP_KERNEL);
if (!els_data) {
- lpfc_printf_vlog(vport, KERN_ERR, LOG_ELS,
+ lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
"0149 Failed to allocate memory "
"for ELS event\n");
return;
@@ -8337,7 +8345,7 @@
/**
* lpfc_els_rcv_fpin_li - Process an FPIN Link Integrity Event.
* @vport: Pointer to vport object.
- * @lnk_not: Pointer to the Link Integrity Notification Descriptor.
+ * @tlv: Pointer to the Link Integrity Notification Descriptor.
*
* This function processes a link integrity FPIN event by
* logging a message
@@ -8396,7 +8404,7 @@
break;
default:
dtag_nm = lpfc_get_tlv_dtag_nm(dtag);
- lpfc_printf_vlog(vport, KERN_ERR, LOG_ELS,
+ lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
"4678 skipped FPIN descriptor[%d]: "
"tag x%x (%s)\n",
desc_cnt, dtag, dtag_nm);
@@ -8811,7 +8819,7 @@
rjt_exp = LSEXP_NOTHING_MORE;
/* Unknown ELS command <elsCmd> received from NPORT <did> */
- lpfc_printf_vlog(vport, KERN_ERR, LOG_ELS,
+ lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
"0115 Unknown ELS command x%x "
"received from NPORT x%x\n", cmd, did);
if (newnode)
@@ -8856,7 +8864,7 @@
dropit:
if (vport && !(vport->load_flag & FC_UNLOADING))
- lpfc_printf_vlog(vport, KERN_ERR, LOG_ELS,
+ lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
"0111 Dropping received ELS cmd "
"Data: x%x x%x x%x\n",
icmd->ulpStatus, icmd->un.ulpWord[4], icmd->ulpTimeout);
@@ -9006,7 +9014,7 @@
spin_lock_irq(shost->host_lock);
if (vport->fc_flag & FC_DISC_DELAYED) {
spin_unlock_irq(shost->host_lock);
- lpfc_printf_log(phba, KERN_ERR, LOG_DISCOVERY,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"3334 Delay fc port discovery for %d seconds\n",
phba->fc_ratov);
mod_timer(&vport->delayed_disc_tmo,
@@ -9024,7 +9032,7 @@
return;
}
lpfc_vport_set_state(vport, FC_VPORT_FAILED);
- lpfc_printf_vlog(vport, KERN_ERR, LOG_ELS,
+ lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
"0251 NameServer login: no memory\n");
return;
}
@@ -9036,7 +9044,7 @@
return;
}
lpfc_vport_set_state(vport, FC_VPORT_FAILED);
- lpfc_printf_vlog(vport, KERN_ERR, LOG_ELS,
+ lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
"0348 NameServer login: node freed\n");
return;
}
@@ -9047,7 +9055,7 @@
if (lpfc_issue_els_plogi(vport, ndlp->nlp_DID, 0)) {
lpfc_vport_set_state(vport, FC_VPORT_FAILED);
- lpfc_printf_vlog(vport, KERN_ERR, LOG_ELS,
+ lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
"0252 Cannot issue NameServer login\n");
return;
}
@@ -9084,7 +9092,7 @@
spin_unlock_irq(shost->host_lock);
if (mb->mbxStatus) {
- lpfc_printf_vlog(vport, KERN_ERR, LOG_MBOX,
+ lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
"0915 Register VPI failed : Status: x%x"
" upd bit: x%x \n", mb->mbxStatus,
mb->un.varRegVpi.upd);
@@ -9114,8 +9122,8 @@
rc = lpfc_sli_issue_mbox(phba, pmb,
MBX_NOWAIT);
if (rc == MBX_NOT_FINISHED) {
- lpfc_printf_vlog(vport,
- KERN_ERR, LOG_MBOX,
+ lpfc_printf_vlog(vport, KERN_ERR,
+ LOG_TRACE_EVENT,
"2732 Failed to issue INIT_VPI"
" mailbox command\n");
} else {
@@ -9203,12 +9211,12 @@
lpfc_nlp_put(ndlp);
mempool_free(mbox, phba->mbox_mem_pool);
- lpfc_printf_vlog(vport, KERN_ERR, LOG_MBOX,
+ lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
"0253 Register VPI: Can't send mbox\n");
goto mbox_err_exit;
}
} else {
- lpfc_printf_vlog(vport, KERN_ERR, LOG_MBOX,
+ lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
"0254 Register VPI: no memory\n");
goto mbox_err_exit;
}
@@ -9370,7 +9378,7 @@
if (lpfc_els_retry(phba, cmdiocb, rspiocb))
goto out;
/* FDISC failed */
- lpfc_printf_vlog(vport, KERN_ERR, LOG_ELS,
+ lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
"0126 FDISC failed. (x%x/x%x)\n",
irsp->ulpStatus, irsp->un.ulpWord[4]);
goto fdisc_failed;
@@ -9492,7 +9500,7 @@
ELS_CMD_FDISC);
if (!elsiocb) {
lpfc_vport_set_state(vport, FC_VPORT_FAILED);
- lpfc_printf_vlog(vport, KERN_ERR, LOG_ELS,
+ lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
"0255 Issue FDISC: no IOCB\n");
return 1;
}
@@ -9546,7 +9554,7 @@
if (rc == IOCB_ERROR) {
lpfc_els_free_iocb(phba, elsiocb);
lpfc_vport_set_state(vport, FC_VPORT_FAILED);
- lpfc_printf_vlog(vport, KERN_ERR, LOG_ELS,
+ lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
"0256 Issue FDISC: Cannot send IOCB\n");
return 1;
}
@@ -9666,7 +9674,7 @@
/**
* lpfc_fabric_block_timeout - Handler function to the fabric block timer
- * @ptr: holder for the timer function associated data.
+ * @t: timer context used to obtain the lpfc hba.
*
* This routine is invoked by the fabric iocb block timer after
* timeout. It posts the fabric iocb block timeout event by setting the
@@ -10127,8 +10135,7 @@
"rport in state 0x%x\n", ndlp->nlp_state);
return;
}
- lpfc_printf_log(phba, KERN_ERR,
- LOG_ELS | LOG_FCP_ERROR | LOG_NVME_IOERR,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"3094 Start rport recovery on shost id 0x%x "
"fc_id 0x%06x vpi 0x%x rpi 0x%x state 0x%x "
"flags 0x%x\n",
diff --git a/drivers/scsi/lpfc/lpfc_hbadisc.c b/drivers/scsi/lpfc/lpfc_hbadisc.c
index 4084f7f..142a021 100644
--- a/drivers/scsi/lpfc/lpfc_hbadisc.c
+++ b/drivers/scsi/lpfc/lpfc_hbadisc.c
@@ -155,17 +155,17 @@
return;
if (rport->port_name != wwn_to_u64(ndlp->nlp_portname.u.wwn))
- lpfc_printf_vlog(vport, KERN_ERR, LOG_NODE,
- "6789 rport name %llx != node port name %llx",
- rport->port_name,
- wwn_to_u64(ndlp->nlp_portname.u.wwn));
+ lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
+ "6789 rport name %llx != node port name %llx",
+ rport->port_name,
+ wwn_to_u64(ndlp->nlp_portname.u.wwn));
evtp = &ndlp->dev_loss_evt;
if (!list_empty(&evtp->evt_listp)) {
- lpfc_printf_vlog(vport, KERN_ERR, LOG_NODE,
- "6790 rport name %llx dev_loss_evt pending",
- rport->port_name);
+ lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
+ "6790 rport name %llx dev_loss_evt pending",
+ rport->port_name);
return;
}
@@ -295,7 +295,7 @@
}
if (warn_on) {
- lpfc_printf_vlog(vport, KERN_ERR, LOG_DISCOVERY,
+ lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
"0203 Devloss timeout on "
"WWPN %02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x "
"NPort x%06x Data: x%x x%x x%x\n",
@@ -304,7 +304,7 @@
ndlp->nlp_DID, ndlp->nlp_flag,
ndlp->nlp_state, ndlp->nlp_rpi);
} else {
- lpfc_printf_vlog(vport, KERN_INFO, LOG_DISCOVERY,
+ lpfc_printf_vlog(vport, KERN_INFO, LOG_TRACE_EVENT,
"0204 Devloss timeout on "
"WWPN %02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x "
"NPort x%06x Data: x%x x%x x%x\n",
@@ -755,7 +755,7 @@
|| kthread_should_stop()));
/* Signal wakeup shall terminate the worker thread */
if (rc) {
- lpfc_printf_log(phba, KERN_ERR, LOG_ELS,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"0433 Wakeup on signal: rc=x%x\n", rc);
break;
}
@@ -1092,7 +1092,7 @@
/* Check for error */
if ((mb->mbxStatus) && (mb->mbxStatus != 0x1601)) {
/* CLEAR_LA mbox error <mbxStatus> state <hba_state> */
- lpfc_printf_vlog(vport, KERN_ERR, LOG_MBOX,
+ lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
"0320 CLEAR_LA mbxStatus error x%x hba "
"state x%x\n",
mb->mbxStatus, vport->port_state);
@@ -1180,7 +1180,7 @@
return;
out:
- lpfc_printf_vlog(vport, KERN_ERR, LOG_MBOX,
+ lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
"0306 CONFIG_LINK mbxStatus error x%x "
"HBA state x%x\n",
pmb->u.mb.mbxStatus, vport->port_state);
@@ -1188,7 +1188,7 @@
lpfc_linkdown(phba);
- lpfc_printf_vlog(vport, KERN_ERR, LOG_DISCOVERY,
+ lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
"0200 CONFIG_LINK bad hba state x%x\n",
vport->port_state);
@@ -1198,7 +1198,7 @@
/**
* lpfc_sli4_clear_fcf_rr_bmask
- * @phba pointer to the struct lpfc_hba for this port.
+ * @phba: pointer to the struct lpfc_hba for this port.
* This fucnction resets the round robin bit mask and clears the
* fcf priority list. The list deletions are done while holding the
* hbalock. The ON_LIST flag and the FLOGI_FAILED flags are cleared
@@ -1224,10 +1224,10 @@
struct lpfc_vport *vport = mboxq->vport;
if (mboxq->u.mb.mbxStatus) {
- lpfc_printf_vlog(vport, KERN_ERR, LOG_MBOX,
- "2017 REG_FCFI mbxStatus error x%x "
- "HBA state x%x\n",
- mboxq->u.mb.mbxStatus, vport->port_state);
+ lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
+ "2017 REG_FCFI mbxStatus error x%x "
+ "HBA state x%x\n", mboxq->u.mb.mbxStatus,
+ vport->port_state);
goto fail_out;
}
@@ -1297,7 +1297,7 @@
/**
* lpfc_sw_name_match - Check if the fcf switch name match.
- * @fab_name: pointer to fabric name.
+ * @sw_name: pointer to switch name.
* @new_fcf_record: pointer to fcf record.
*
* This routine compare the fcf record's switch name with provided
@@ -1385,7 +1385,7 @@
/**
* lpfc_copy_fcf_record - Copy fcf information to lpfc_hba.
- * @fcf: pointer to driver fcf record.
+ * @fcf_rec: pointer to driver fcf record.
* @new_fcf_record: pointer to fcf record.
*
* This routine copies the FCF information from the FCF
@@ -1848,7 +1848,7 @@
*/
lpfc_sli4_mbx_sge_get(mboxq, 0, &sge);
if (unlikely(!mboxq->sge_array)) {
- lpfc_printf_log(phba, KERN_ERR, LOG_MBOX,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"2524 Failed to get the non-embedded SGE "
"virtual address\n");
return NULL;
@@ -1864,11 +1864,12 @@
if (shdr_status || shdr_add_status) {
if (shdr_status == STATUS_FCF_TABLE_EMPTY ||
if_type == LPFC_SLI_INTF_IF_TYPE_2)
- lpfc_printf_log(phba, KERN_ERR, LOG_FIP,
+ lpfc_printf_log(phba, KERN_ERR,
+ LOG_TRACE_EVENT,
"2726 READ_FCF_RECORD Indicates empty "
"FCF table.\n");
else
- lpfc_printf_log(phba, KERN_ERR, LOG_FIP,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"2521 READ_FCF_RECORD mailbox failed "
"with status x%x add_status x%x, "
"mbx\n", shdr_status, shdr_add_status);
@@ -1952,7 +1953,7 @@
}
/**
- lpfc_sli4_fcf_record_match - testing new FCF record for matching existing FCF
+ * lpfc_sli4_fcf_record_match - testing new FCF record for matching existing FCF
* @phba: pointer to lpfc hba data structure.
* @fcf_rec: pointer to an existing FCF record.
* @new_fcf_record: pointer to a new FCF record.
@@ -2066,7 +2067,7 @@
/**
* lpfc_sli4_fcf_pri_list_del
* @phba: pointer to lpfc hba data structure.
- * @fcf_index the index of the fcf record to delete
+ * @fcf_index: the index of the fcf record to delete
* This routine checks the on list flag of the fcf_index to be deleted.
* If it is one the list then it is removed from the list, and the flag
* is cleared. This routine grab the hbalock before removing the fcf
@@ -2096,7 +2097,7 @@
/**
* lpfc_sli4_set_fcf_flogi_fail
* @phba: pointer to lpfc hba data structure.
- * @fcf_index the index of the fcf record to update
+ * @fcf_index: the index of the fcf record to update
* This routine acquires the hbalock and then set the LPFC_FCF_FLOGI_FAILED
* flag so the the round robin slection for the particular priority level
* will try a different fcf record that does not have this bit set.
@@ -2116,7 +2117,8 @@
/**
* lpfc_sli4_fcf_pri_list_add
* @phba: pointer to lpfc hba data structure.
- * @fcf_index the index of the fcf record to add
+ * @fcf_index: the index of the fcf record to add
+ * @new_fcf_record: pointer to a new FCF record.
* This routine checks the priority of the fcf_index to be added.
* If it is a lower priority than the current head of the fcf_pri list
* then it is added to the list in the right order.
@@ -2246,7 +2248,7 @@
new_fcf_record = lpfc_sli4_fcf_rec_mbox_parse(phba, mboxq,
&next_fcf_index);
if (!new_fcf_record) {
- lpfc_printf_log(phba, KERN_ERR, LOG_FIP,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"2765 Mailbox command READ_FCF_RECORD "
"failed to retrieve a FCF record.\n");
/* Let next new FCF event trigger fast failover */
@@ -2290,7 +2292,8 @@
new_fcf_record, LPFC_FCOE_IGNORE_VID)) {
if (bf_get(lpfc_fcf_record_fcf_index, new_fcf_record) !=
phba->fcf.current_rec.fcf_indx) {
- lpfc_printf_log(phba, KERN_ERR, LOG_FIP,
+ lpfc_printf_log(phba, KERN_ERR,
+ LOG_TRACE_EVENT,
"2862 FCF (x%x) matches property "
"of in-use FCF (x%x)\n",
bf_get(lpfc_fcf_record_fcf_index,
@@ -2360,7 +2363,7 @@
phba->pport->fc_flag);
goto out;
} else
- lpfc_printf_log(phba, KERN_ERR, LOG_FIP,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"2863 New FCF (x%x) matches "
"property of in-use FCF (x%x)\n",
bf_get(lpfc_fcf_record_fcf_index,
@@ -2774,10 +2777,9 @@
(bf_get(lpfc_sli_intf_if_type, &phba->sli4_hba.sli_intf) !=
LPFC_SLI_INTF_IF_TYPE_0) &&
mboxq->u.mb.mbxStatus != MBX_VFI_IN_USE) {
- lpfc_printf_vlog(vport, KERN_ERR,
- LOG_MBOX,
- "2891 Init VFI mailbox failed 0x%x\n",
- mboxq->u.mb.mbxStatus);
+ lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
+ "2891 Init VFI mailbox failed 0x%x\n",
+ mboxq->u.mb.mbxStatus);
mempool_free(mboxq, phba->mbox_mem_pool);
lpfc_vport_set_state(vport, FC_VPORT_FAILED);
return;
@@ -2805,7 +2807,7 @@
mboxq = mempool_alloc(phba->mbox_mem_pool, GFP_KERNEL);
if (!mboxq) {
lpfc_printf_vlog(vport, KERN_ERR,
- LOG_MBOX, "2892 Failed to allocate "
+ LOG_TRACE_EVENT, "2892 Failed to allocate "
"init_vfi mailbox\n");
return;
}
@@ -2813,8 +2815,8 @@
mboxq->mbox_cmpl = lpfc_init_vfi_cmpl;
rc = lpfc_sli_issue_mbox(phba, mboxq, MBX_NOWAIT);
if (rc == MBX_NOT_FINISHED) {
- lpfc_printf_vlog(vport, KERN_ERR,
- LOG_MBOX, "2893 Failed to issue init_vfi mailbox\n");
+ lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
+ "2893 Failed to issue init_vfi mailbox\n");
mempool_free(mboxq, vport->phba->mbox_mem_pool);
}
}
@@ -2834,10 +2836,9 @@
struct Scsi_Host *shost = lpfc_shost_from_vport(vport);
if (mboxq->u.mb.mbxStatus) {
- lpfc_printf_vlog(vport, KERN_ERR,
- LOG_MBOX,
- "2609 Init VPI mailbox failed 0x%x\n",
- mboxq->u.mb.mbxStatus);
+ lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
+ "2609 Init VPI mailbox failed 0x%x\n",
+ mboxq->u.mb.mbxStatus);
mempool_free(mboxq, phba->mbox_mem_pool);
lpfc_vport_set_state(vport, FC_VPORT_FAILED);
return;
@@ -2851,7 +2852,7 @@
ndlp = lpfc_findnode_did(vport, Fabric_DID);
if (!ndlp)
lpfc_printf_vlog(vport, KERN_ERR,
- LOG_DISCOVERY,
+ LOG_TRACE_EVENT,
"2731 Cannot find fabric "
"controller node\n");
else
@@ -2864,7 +2865,7 @@
lpfc_initial_fdisc(vport);
else {
lpfc_vport_set_state(vport, FC_VPORT_NO_FABRIC_SUPP);
- lpfc_printf_vlog(vport, KERN_ERR, LOG_ELS,
+ lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
"2606 No NPIV Fabric support\n");
}
mempool_free(mboxq, phba->mbox_mem_pool);
@@ -2887,8 +2888,7 @@
if ((vport->port_type != LPFC_PHYSICAL_PORT) && (!vport->vpi)) {
vpi = lpfc_alloc_vpi(vport->phba);
if (!vpi) {
- lpfc_printf_vlog(vport, KERN_ERR,
- LOG_MBOX,
+ lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
"3303 Failed to obtain vport vpi\n");
lpfc_vport_set_state(vport, FC_VPORT_FAILED);
return;
@@ -2899,7 +2899,7 @@
mboxq = mempool_alloc(vport->phba->mbox_mem_pool, GFP_KERNEL);
if (!mboxq) {
lpfc_printf_vlog(vport, KERN_ERR,
- LOG_MBOX, "2607 Failed to allocate "
+ LOG_TRACE_EVENT, "2607 Failed to allocate "
"init_vpi mailbox\n");
return;
}
@@ -2908,8 +2908,8 @@
mboxq->mbox_cmpl = lpfc_init_vpi_cmpl;
rc = lpfc_sli_issue_mbox(vport->phba, mboxq, MBX_NOWAIT);
if (rc == MBX_NOT_FINISHED) {
- lpfc_printf_vlog(vport, KERN_ERR,
- LOG_MBOX, "2608 Failed to issue init_vpi mailbox\n");
+ lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
+ "2608 Failed to issue init_vpi mailbox\n");
mempool_free(mboxq, vport->phba->mbox_mem_pool);
}
}
@@ -2953,7 +2953,7 @@
lpfc_vport_set_state(vports[i],
FC_VPORT_NO_FABRIC_SUPP);
lpfc_printf_vlog(vports[i], KERN_ERR,
- LOG_ELS,
+ LOG_TRACE_EVENT,
"0259 No NPIV "
"Fabric support\n");
}
@@ -2977,10 +2977,10 @@
(bf_get(lpfc_sli_intf_if_type, &phba->sli4_hba.sli_intf) !=
LPFC_SLI_INTF_IF_TYPE_0) &&
mboxq->u.mb.mbxStatus != MBX_VFI_IN_USE) {
- lpfc_printf_vlog(vport, KERN_ERR, LOG_MBOX,
- "2018 REG_VFI mbxStatus error x%x "
- "HBA state x%x\n",
- mboxq->u.mb.mbxStatus, vport->port_state);
+ lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
+ "2018 REG_VFI mbxStatus error x%x "
+ "HBA state x%x\n",
+ mboxq->u.mb.mbxStatus, vport->port_state);
if (phba->fc_topology == LPFC_TOPOLOGY_LOOP) {
/* FLOGI failed, use loop map to make discovery list */
lpfc_disc_list_loopmap(vport);
@@ -3067,7 +3067,7 @@
/* Check for error */
if (mb->mbxStatus) {
/* READ_SPARAM mbox error <mbxStatus> state <hba_state> */
- lpfc_printf_vlog(vport, KERN_ERR, LOG_MBOX,
+ lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
"0319 READ_SPARAM mbxStatus error x%x "
"hba state x%x>\n",
mb->mbxStatus, vport->port_state);
@@ -3286,7 +3286,7 @@
GFP_KERNEL);
if (unlikely(!fcf_record)) {
lpfc_printf_log(phba, KERN_ERR,
- LOG_MBOX | LOG_SLI,
+ LOG_TRACE_EVENT,
"2554 Could not allocate memory for "
"fcf record\n");
rc = -ENODEV;
@@ -3298,7 +3298,7 @@
rc = lpfc_sli4_add_fcf_record(phba, fcf_record);
if (unlikely(rc)) {
lpfc_printf_log(phba, KERN_ERR,
- LOG_MBOX | LOG_SLI,
+ LOG_TRACE_EVENT,
"2013 Could not manually add FCF "
"record 0, status %d\n", rc);
rc = -ENODEV;
@@ -3344,7 +3344,7 @@
return;
out:
lpfc_vport_set_state(vport, FC_VPORT_FAILED);
- lpfc_printf_vlog(vport, KERN_ERR, LOG_MBOX,
+ lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
"0263 Discovery Mailbox error: state: 0x%x : x%px x%px\n",
vport->port_state, sparam_mbox, cfglink_mbox);
lpfc_issue_clear_la(phba, vport);
@@ -3617,7 +3617,7 @@
break;
/* If VPI is busy, reset the HBA */
case 0x9700:
- lpfc_printf_vlog(vport, KERN_ERR, LOG_NODE,
+ lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
"2798 Unreg_vpi failed vpi 0x%x, mb status = 0x%x\n",
vport->vpi, mb->mbxStatus);
if (!(phba->pport->load_flag & FC_UNLOADING))
@@ -3655,7 +3655,7 @@
mbox->mbox_cmpl = lpfc_mbx_cmpl_unreg_vpi;
rc = lpfc_sli_issue_mbox(phba, mbox, MBX_NOWAIT);
if (rc == MBX_NOT_FINISHED) {
- lpfc_printf_vlog(vport, KERN_ERR, LOG_MBOX | LOG_VPORT,
+ lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
"1800 Could not issue unreg_vpi\n");
mempool_free(mbox, phba->mbox_mem_pool);
vport->unreg_vpi_cmpl = VPORT_ERROR;
@@ -3742,7 +3742,7 @@
pmb = mempool_alloc(phba->mbox_mem_pool, GFP_KERNEL);
if (!pmb) {
- lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"0542 lpfc_create_static_vport failed to"
" allocate mailbox memory\n");
return;
@@ -3752,7 +3752,7 @@
vport_info = kzalloc(sizeof(struct static_vport_info), GFP_KERNEL);
if (!vport_info) {
- lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"0543 lpfc_create_static_vport failed to"
" allocate vport_info\n");
mempool_free(pmb, phba->mbox_mem_pool);
@@ -3813,11 +3813,12 @@
if ((le32_to_cpu(vport_info->signature) != VPORT_INFO_SIG) ||
((le32_to_cpu(vport_info->rev) & VPORT_INFO_REV_MASK)
!= VPORT_INFO_REV)) {
- lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
- "0545 lpfc_create_static_vport bad"
- " information header 0x%x 0x%x\n",
- le32_to_cpu(vport_info->signature),
- le32_to_cpu(vport_info->rev) & VPORT_INFO_REV_MASK);
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
+ "0545 lpfc_create_static_vport bad"
+ " information header 0x%x 0x%x\n",
+ le32_to_cpu(vport_info->signature),
+ le32_to_cpu(vport_info->rev) &
+ VPORT_INFO_REV_MASK);
goto out;
}
@@ -3881,7 +3882,7 @@
pmb->ctx_buf = NULL;
if (mb->mbxStatus) {
- lpfc_printf_vlog(vport, KERN_ERR, LOG_MBOX,
+ lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
"0258 Register Fabric login error: 0x%x\n",
mb->mbxStatus);
lpfc_mbuf_free(phba, mp->virt, mp->phys);
@@ -3954,7 +3955,8 @@
/* Cannot issue NameServer FCP Query, so finish up
* discovery
*/
- lpfc_printf_vlog(vport, KERN_ERR, LOG_SLI,
+ lpfc_printf_vlog(vport, KERN_ERR,
+ LOG_TRACE_EVENT,
"0604 %s FC TYPE %x %s\n",
"Failed to issue GID_FT to ",
FC_TYPE_FCP,
@@ -3970,7 +3972,8 @@
/* Cannot issue NameServer NVME Query, so finish up
* discovery
*/
- lpfc_printf_vlog(vport, KERN_ERR, LOG_SLI,
+ lpfc_printf_vlog(vport, KERN_ERR,
+ LOG_TRACE_EVENT,
"0605 %s FC_TYPE %x %s %d\n",
"Failed to issue GID_FT to ",
FC_TYPE_NVME,
@@ -4002,7 +4005,7 @@
/* Cannot issue NameServer FCP Query, so finish up
* discovery
*/
- lpfc_printf_vlog(vport, KERN_ERR, LOG_SLI,
+ lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
"0606 %s Port TYPE %x %s\n",
"Failed to issue GID_PT to ",
GID_PT_N_PORT,
@@ -4032,7 +4035,7 @@
vport->gidft_inp = 0;
if (mb->mbxStatus) {
- lpfc_printf_vlog(vport, KERN_ERR, LOG_ELS,
+ lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
"0260 Register NameServer error: 0x%x\n",
mb->mbxStatus);
@@ -4344,7 +4347,7 @@
GFP_KERNEL);
if (!ndlp->lat_data)
- lpfc_printf_vlog(vport, KERN_ERR, LOG_NODE,
+ lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
"0286 lpfc_nlp_state_cleanup failed to "
"allocate statistical data buffer DID "
"0x%x\n", ndlp->nlp_DID);
@@ -5013,8 +5016,8 @@
vports = lpfc_create_vport_work_array(phba);
if (!vports) {
- lpfc_printf_log(phba, KERN_ERR, LOG_DISCOVERY,
- "2884 Vport array allocation failed \n");
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
+ "2884 Vport array allocation failed \n");
return;
}
for (i = 0; i <= phba->max_vports && vports[i] != NULL; i++) {
@@ -5057,9 +5060,10 @@
mempool_free(mbox, phba->mbox_mem_pool);
if ((rc == MBX_TIMEOUT) || (rc == MBX_NOT_FINISHED))
- lpfc_printf_vlog(vport, KERN_ERR, LOG_MBOX | LOG_VPORT,
- "1836 Could not issue "
- "unreg_login(all_rpis) status %d\n", rc);
+ lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
+ "1836 Could not issue "
+ "unreg_login(all_rpis) status %d\n",
+ rc);
}
}
@@ -5086,7 +5090,7 @@
mempool_free(mbox, phba->mbox_mem_pool);
if ((rc == MBX_TIMEOUT) || (rc == MBX_NOT_FINISHED))
- lpfc_printf_vlog(vport, KERN_ERR, LOG_MBOX | LOG_VPORT,
+ lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
"1815 Could not issue "
"unreg_did (default rpis) status %d\n",
rc);
@@ -5907,7 +5911,8 @@
case LPFC_FLOGI:
/* port_state is identically LPFC_FLOGI while waiting for FLOGI cmpl */
/* Initial FLOGI timeout */
- lpfc_printf_vlog(vport, KERN_ERR, LOG_DISCOVERY,
+ lpfc_printf_vlog(vport, KERN_ERR,
+ LOG_TRACE_EVENT,
"0222 Initial %s timeout\n",
vport->vpi ? "FDISC" : "FLOGI");
@@ -5925,7 +5930,8 @@
case LPFC_FABRIC_CFG_LINK:
/* hba_state is identically LPFC_FABRIC_CFG_LINK while waiting for
NameServer login */
- lpfc_printf_vlog(vport, KERN_ERR, LOG_DISCOVERY,
+ lpfc_printf_vlog(vport, KERN_ERR,
+ LOG_TRACE_EVENT,
"0223 Timeout while waiting for "
"NameServer login\n");
/* Next look for NameServer ndlp */
@@ -5938,7 +5944,8 @@
case LPFC_NS_QRY:
/* Check for wait for NameServer Rsp timeout */
- lpfc_printf_vlog(vport, KERN_ERR, LOG_DISCOVERY,
+ lpfc_printf_vlog(vport, KERN_ERR,
+ LOG_TRACE_EVENT,
"0224 NameServer Query timeout "
"Data: x%x x%x\n",
vport->fc_ns_retry, LPFC_MAX_NS_RETRY);
@@ -5971,7 +5978,8 @@
/* Setup and issue mailbox INITIALIZE LINK command */
initlinkmbox = mempool_alloc(phba->mbox_mem_pool, GFP_KERNEL);
if (!initlinkmbox) {
- lpfc_printf_vlog(vport, KERN_ERR, LOG_DISCOVERY,
+ lpfc_printf_vlog(vport, KERN_ERR,
+ LOG_TRACE_EVENT,
"0206 Device Discovery "
"completion error\n");
phba->link_state = LPFC_HBA_ERROR;
@@ -5993,7 +6001,8 @@
case LPFC_DISC_AUTH:
/* Node Authentication timeout */
- lpfc_printf_vlog(vport, KERN_ERR, LOG_DISCOVERY,
+ lpfc_printf_vlog(vport, KERN_ERR,
+ LOG_TRACE_EVENT,
"0227 Node Authentication timeout\n");
lpfc_disc_flush_list(vport);
@@ -6013,7 +6022,8 @@
case LPFC_VPORT_READY:
if (vport->fc_flag & FC_RSCN_MODE) {
- lpfc_printf_vlog(vport, KERN_ERR, LOG_DISCOVERY,
+ lpfc_printf_vlog(vport, KERN_ERR,
+ LOG_TRACE_EVENT,
"0231 RSCN timeout Data: x%x "
"x%x\n",
vport->fc_ns_retry, LPFC_MAX_NS_RETRY);
@@ -6027,7 +6037,8 @@
break;
default:
- lpfc_printf_vlog(vport, KERN_ERR, LOG_DISCOVERY,
+ lpfc_printf_vlog(vport, KERN_ERR,
+ LOG_TRACE_EVENT,
"0273 Unexpected discovery timeout, "
"vport State x%x\n", vport->port_state);
break;
@@ -6036,7 +6047,8 @@
switch (phba->link_state) {
case LPFC_CLEAR_LA:
/* CLEAR LA timeout */
- lpfc_printf_vlog(vport, KERN_ERR, LOG_DISCOVERY,
+ lpfc_printf_vlog(vport, KERN_ERR,
+ LOG_TRACE_EVENT,
"0228 CLEAR LA timeout\n");
clrlaerr = 1;
break;
@@ -6050,7 +6062,8 @@
case LPFC_INIT_MBX_CMDS:
case LPFC_LINK_DOWN:
case LPFC_HBA_ERROR:
- lpfc_printf_vlog(vport, KERN_ERR, LOG_DISCOVERY,
+ lpfc_printf_vlog(vport, KERN_ERR,
+ LOG_TRACE_EVENT,
"0230 Unexpected timeout, hba link "
"state x%x\n", phba->link_state);
clrlaerr = 1;
@@ -6241,9 +6254,9 @@
}
if (i >= phba->max_vpi) {
- lpfc_printf_log(phba, KERN_ERR, LOG_ELS,
- "2936 Could not find Vport mapped "
- "to vpi %d\n", vpi);
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
+ "2936 Could not find Vport mapped "
+ "to vpi %d\n", vpi);
return NULL;
}
}
@@ -6547,10 +6560,10 @@
struct Scsi_Host *shost = lpfc_shost_from_vport(vport);
if (mboxq->u.mb.mbxStatus) {
- lpfc_printf_log(phba, KERN_ERR, LOG_DISCOVERY|LOG_MBOX,
- "2555 UNREG_VFI mbxStatus error x%x "
- "HBA state x%x\n",
- mboxq->u.mb.mbxStatus, vport->port_state);
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
+ "2555 UNREG_VFI mbxStatus error x%x "
+ "HBA state x%x\n",
+ mboxq->u.mb.mbxStatus, vport->port_state);
}
spin_lock_irq(shost->host_lock);
phba->pport->fc_flag &= ~FC_VFI_REGISTERED;
@@ -6572,10 +6585,10 @@
struct lpfc_vport *vport = mboxq->vport;
if (mboxq->u.mb.mbxStatus) {
- lpfc_printf_log(phba, KERN_ERR, LOG_DISCOVERY|LOG_MBOX,
- "2550 UNREG_FCFI mbxStatus error x%x "
- "HBA state x%x\n",
- mboxq->u.mb.mbxStatus, vport->port_state);
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
+ "2550 UNREG_FCFI mbxStatus error x%x "
+ "HBA state x%x\n",
+ mboxq->u.mb.mbxStatus, vport->port_state);
}
mempool_free(mboxq, phba->mbox_mem_pool);
return;
@@ -6664,7 +6677,7 @@
mbox = mempool_alloc(phba->mbox_mem_pool, GFP_KERNEL);
if (!mbox) {
- lpfc_printf_log(phba, KERN_ERR, LOG_DISCOVERY|LOG_MBOX,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"2551 UNREG_FCFI mbox allocation failed"
"HBA state x%x\n", phba->pport->port_state);
return -ENOMEM;
@@ -6675,7 +6688,7 @@
rc = lpfc_sli_issue_mbox(phba, mbox, MBX_NOWAIT);
if (rc == MBX_NOT_FINISHED) {
- lpfc_printf_log(phba, KERN_ERR, LOG_SLI,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"2552 Unregister FCFI command failed rc x%x "
"HBA state x%x\n",
rc, phba->pport->port_state);
@@ -6699,7 +6712,7 @@
/* Preparation for unregistering fcf */
rc = lpfc_unregister_fcf_prep(phba);
if (rc) {
- lpfc_printf_log(phba, KERN_ERR, LOG_DISCOVERY,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"2748 Failed to prepare for unregistering "
"HBA's FCF record: rc=%d\n", rc);
return;
@@ -6735,7 +6748,7 @@
spin_lock_irq(&phba->hbalock);
phba->fcf.fcf_flag &= ~FCF_INIT_DISC;
spin_unlock_irq(&phba->hbalock);
- lpfc_printf_log(phba, KERN_ERR, LOG_DISCOVERY|LOG_MBOX,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"2553 lpfc_unregister_unused_fcf failed "
"to read FCF record HBA state x%x\n",
phba->pport->port_state);
@@ -6757,7 +6770,7 @@
/* Preparation for unregistering fcf */
rc = lpfc_unregister_fcf_prep(phba);
if (rc) {
- lpfc_printf_log(phba, KERN_ERR, LOG_DISCOVERY,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"2749 Failed to prepare for unregistering "
"HBA's FCF record: rc=%d\n", rc);
return;
@@ -6844,9 +6857,9 @@
conn_entry = kzalloc(sizeof(struct lpfc_fcf_conn_entry),
GFP_KERNEL);
if (!conn_entry) {
- lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
- "2566 Failed to allocate connection"
- " table entry\n");
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
+ "2566 Failed to allocate connection"
+ " table entry\n");
return;
}
@@ -6990,7 +7003,7 @@
/* Check the region signature first */
if (memcmp(buff, LPFC_REGION23_SIGNATURE, 4)) {
- lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"2567 Config region 23 has bad signature\n");
return;
}
@@ -6999,8 +7012,8 @@
/* Check the data structure version */
if (buff[offset] != LPFC_REGION23_VERSION) {
- lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
- "2568 Config region 23 has bad version\n");
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
+ "2568 Config region 23 has bad version\n");
return;
}
offset += 4;
diff --git a/drivers/scsi/lpfc/lpfc_hw4.h b/drivers/scsi/lpfc/lpfc_hw4.h
index 6dfff03..c4ba827 100644
--- a/drivers/scsi/lpfc/lpfc_hw4.h
+++ b/drivers/scsi/lpfc/lpfc_hw4.h
@@ -650,6 +650,9 @@
#define lpfc_sliport_status_oti_SHIFT 29
#define lpfc_sliport_status_oti_MASK 0x1
#define lpfc_sliport_status_oti_WORD word0
+#define lpfc_sliport_status_dip_SHIFT 25
+#define lpfc_sliport_status_dip_MASK 0x1
+#define lpfc_sliport_status_dip_WORD word0
#define lpfc_sliport_status_rn_SHIFT 24
#define lpfc_sliport_status_rn_MASK 0x1
#define lpfc_sliport_status_rn_WORD word0
@@ -3531,7 +3534,7 @@
};
#define LPFC_SET_UE_RECOVERY 0x10
-#define LPFC_SET_MDS_DIAGS 0x11
+#define LPFC_SET_MDS_DIAGS 0x12
#define LPFC_SET_DUAL_DUMP 0x1e
struct lpfc_mbx_set_feature {
struct mbox_header header;
diff --git a/drivers/scsi/lpfc/lpfc_init.c b/drivers/scsi/lpfc/lpfc_init.c
index 6637f84..c4a7e82 100644
--- a/drivers/scsi/lpfc/lpfc_init.c
+++ b/drivers/scsi/lpfc/lpfc_init.c
@@ -153,7 +153,7 @@
rc = lpfc_sli_issue_mbox(phba, pmb, MBX_POLL);
if (rc != MBX_SUCCESS) {
- lpfc_printf_log(phba, KERN_ERR, LOG_MBOX,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"0324 Config Port initialization "
"error, mbxCmd x%x READ_NVPARM, "
"mbxStatus x%x\n",
@@ -177,7 +177,7 @@
lpfc_read_rev(phba, pmb);
rc = lpfc_sli_issue_mbox(phba, pmb, MBX_POLL);
if (rc != MBX_SUCCESS) {
- lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"0439 Adapter failed to init, mbxCmd x%x "
"READ_REV, mbxStatus x%x\n",
mb->mbxCommand, mb->mbxStatus);
@@ -192,7 +192,7 @@
*/
if (mb->un.varRdRev.rr == 0) {
vp->rev.rBit = 0;
- lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"0440 Adapter failed to init, READ_REV has "
"missing revision information.\n");
mempool_free(pmb, phba->mbox_mem_pool);
@@ -253,13 +253,15 @@
*/
if (mb->un.varDmp.word_cnt == 0)
break;
- if (mb->un.varDmp.word_cnt > DMP_VPD_SIZE - offset)
- mb->un.varDmp.word_cnt = DMP_VPD_SIZE - offset;
+
+ i = mb->un.varDmp.word_cnt * sizeof(uint32_t);
+ if (offset + i > DMP_VPD_SIZE)
+ i = DMP_VPD_SIZE - offset;
lpfc_sli_pcimem_bcopy(((uint8_t *)mb) + DMP_RSP_OFFSET,
- lpfc_vpd_data + offset,
- mb->un.varDmp.word_cnt);
- offset += mb->un.varDmp.word_cnt;
- } while (mb->un.varDmp.word_cnt && offset < DMP_VPD_SIZE);
+ lpfc_vpd_data + offset, i);
+ offset += i;
+ } while (offset < DMP_VPD_SIZE);
+
lpfc_parse_vpd(phba, lpfc_vpd_data, offset);
kfree(lpfc_vpd_data);
@@ -442,7 +444,7 @@
pmb->vport = vport;
if (lpfc_sli_issue_mbox(phba, pmb, MBX_POLL) != MBX_SUCCESS) {
- lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"0448 Adapter failed init, mbxCmd x%x "
"READ_SPARM mbxStatus x%x\n",
mb->mbxCommand, mb->mbxStatus);
@@ -496,7 +498,7 @@
lpfc_read_config(phba, pmb);
pmb->vport = vport;
if (lpfc_sli_issue_mbox(phba, pmb, MBX_POLL) != MBX_SUCCESS) {
- lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"0453 Adapter failed to init, mbxCmd x%x "
"READ_CONFIG, mbxStatus x%x\n",
mb->mbxCommand, mb->mbxStatus);
@@ -545,7 +547,7 @@
}
rc = lpfc_sli_issue_mbox(phba, pmb, MBX_POLL);
if (rc != MBX_SUCCESS) {
- lpfc_printf_log(phba, KERN_ERR, LOG_MBOX,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"0352 Config MSI mailbox command "
"failed, mbxCmd x%x, mbxStatus x%x\n",
pmb->u.mb.mbxCommand,
@@ -596,17 +598,15 @@
jiffies + msecs_to_jiffies(1000 * phba->eratt_poll_interval));
if (phba->hba_flag & LINK_DISABLED) {
- lpfc_printf_log(phba,
- KERN_ERR, LOG_INIT,
- "2598 Adapter Link is disabled.\n");
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
+ "2598 Adapter Link is disabled.\n");
lpfc_down_link(phba, pmb);
pmb->mbox_cmpl = lpfc_sli_def_mbox_cmpl;
rc = lpfc_sli_issue_mbox(phba, pmb, MBX_NOWAIT);
if ((rc != MBX_SUCCESS) && (rc != MBX_BUSY)) {
- lpfc_printf_log(phba,
- KERN_ERR, LOG_INIT,
- "2599 Adapter failed to issue DOWN_LINK"
- " mbox command rc 0x%x\n", rc);
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
+ "2599 Adapter failed to issue DOWN_LINK"
+ " mbox command rc 0x%x\n", rc);
mempool_free(pmb, phba->mbox_mem_pool);
return -EIO;
@@ -630,9 +630,7 @@
rc = lpfc_sli_issue_mbox(phba, pmb, MBX_NOWAIT);
if ((rc != MBX_BUSY) && (rc != MBX_SUCCESS)) {
- lpfc_printf_log(phba,
- KERN_ERR,
- LOG_INIT,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"0456 Adapter failed to issue "
"ASYNCEVT_ENABLE mbox status x%x\n",
rc);
@@ -652,7 +650,8 @@
rc = lpfc_sli_issue_mbox(phba, pmb, MBX_NOWAIT);
if ((rc != MBX_BUSY) && (rc != MBX_SUCCESS)) {
- lpfc_printf_log(phba, KERN_ERR, LOG_INIT, "0435 Adapter failed "
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
+ "0435 Adapter failed "
"to get Option ROM version status x%x\n", rc);
mempool_free(pmb, phba->mbox_mem_pool);
}
@@ -730,10 +729,10 @@
((phba->cfg_link_speed == LPFC_USER_LINK_SPEED_64G) &&
!(phba->lmt & LMT_64Gb))) {
/* Reset link speed to auto */
- lpfc_printf_log(phba, KERN_ERR, LOG_LINK_EVENT,
- "1302 Invalid speed for this board:%d "
- "Reset link speed to auto.\n",
- phba->cfg_link_speed);
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
+ "1302 Invalid speed for this board:%d "
+ "Reset link speed to auto.\n",
+ phba->cfg_link_speed);
phba->cfg_link_speed = LPFC_USER_LINK_SPEED_AUTO;
}
lpfc_init_link(phba, pmb, fc_topology, phba->cfg_link_speed);
@@ -742,10 +741,10 @@
lpfc_set_loopback_flag(phba);
rc = lpfc_sli_issue_mbox(phba, pmb, flag);
if ((rc != MBX_BUSY) && (rc != MBX_SUCCESS)) {
- lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
- "0498 Adapter failed to init, mbxCmd x%x "
- "INIT_LINK, mbxStatus x%x\n",
- mb->mbxCommand, mb->mbxStatus);
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
+ "0498 Adapter failed to init, mbxCmd x%x "
+ "INIT_LINK, mbxStatus x%x\n",
+ mb->mbxCommand, mb->mbxStatus);
if (phba->sli_rev <= LPFC_SLI_REV3) {
/* Clear all interrupt enable conditions */
writel(0, phba->HCregaddr);
@@ -791,17 +790,15 @@
return -ENOMEM;
}
- lpfc_printf_log(phba,
- KERN_ERR, LOG_INIT,
- "0491 Adapter Link is disabled.\n");
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
+ "0491 Adapter Link is disabled.\n");
lpfc_down_link(phba, pmb);
pmb->mbox_cmpl = lpfc_sli_def_mbox_cmpl;
rc = lpfc_sli_issue_mbox(phba, pmb, flag);
if ((rc != MBX_SUCCESS) && (rc != MBX_BUSY)) {
- lpfc_printf_log(phba,
- KERN_ERR, LOG_INIT,
- "2522 Adapter failed to issue DOWN_LINK"
- " mbox command rc 0x%x\n", rc);
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
+ "2522 Adapter failed to issue DOWN_LINK"
+ " mbox command rc 0x%x\n", rc);
mempool_free(pmb, phba->mbox_mem_pool);
return -EIO;
@@ -996,7 +993,6 @@
/**
* lpfc_hba_down_post_s3 - Perform lpfc uninitialization after HBA reset
- int i;
* @phba: pointer to lpfc HBA data structure.
*
* This routine will do uninitialization after the HBA is reset when bring
@@ -1124,7 +1120,7 @@
/**
* lpfc_hb_timeout - The HBA-timer timeout handler
- * @ptr: unsigned long holds the pointer to lpfc hba data structure.
+ * @t: timer context used to obtain the pointer to lpfc hba data structure.
*
* This is the HBA-timer timeout handler registered to the lpfc driver. When
* this timer fires, a HBA timeout event shall be posted to the lpfc driver
@@ -1158,7 +1154,7 @@
/**
* lpfc_rrq_timeout - The RRQ-timer timeout handler
- * @ptr: unsigned long holds the pointer to lpfc hba data structure.
+ * @t: timer context used to obtain the pointer to lpfc hba data structure.
*
* This is the RRQ-timer timeout handler registered to the lpfc driver. When
* this timer fires, a RRQ timeout event shall be posted to the lpfc driver
@@ -1222,6 +1218,75 @@
return;
}
+/*
+ * lpfc_idle_stat_delay_work - idle_stat tracking
+ *
+ * This routine tracks per-cq idle_stat and determines polling decisions.
+ *
+ * Return codes:
+ * None
+ **/
+static void
+lpfc_idle_stat_delay_work(struct work_struct *work)
+{
+ struct lpfc_hba *phba = container_of(to_delayed_work(work),
+ struct lpfc_hba,
+ idle_stat_delay_work);
+ struct lpfc_queue *cq;
+ struct lpfc_sli4_hdw_queue *hdwq;
+ struct lpfc_idle_stat *idle_stat;
+ u32 i, idle_percent;
+ u64 wall, wall_idle, diff_wall, diff_idle, busy_time;
+
+ if (phba->pport->load_flag & FC_UNLOADING)
+ return;
+
+ if (phba->link_state == LPFC_HBA_ERROR ||
+ phba->pport->fc_flag & FC_OFFLINE_MODE)
+ goto requeue;
+
+ for_each_present_cpu(i) {
+ hdwq = &phba->sli4_hba.hdwq[phba->sli4_hba.cpu_map[i].hdwq];
+ cq = hdwq->io_cq;
+
+ /* Skip if we've already handled this cq's primary CPU */
+ if (cq->chann != i)
+ continue;
+
+ idle_stat = &phba->sli4_hba.idle_stat[i];
+
+ /* get_cpu_idle_time returns values as running counters. Thus,
+ * to know the amount for this period, the prior counter values
+ * need to be subtracted from the current counter values.
+ * From there, the idle time stat can be calculated as a
+ * percentage of 100 - the sum of the other consumption times.
+ */
+ wall_idle = get_cpu_idle_time(i, &wall, 1);
+ diff_idle = wall_idle - idle_stat->prev_idle;
+ diff_wall = wall - idle_stat->prev_wall;
+
+ if (diff_wall <= diff_idle)
+ busy_time = 0;
+ else
+ busy_time = diff_wall - diff_idle;
+
+ idle_percent = div64_u64(100 * busy_time, diff_wall);
+ idle_percent = 100 - idle_percent;
+
+ if (idle_percent < 15)
+ cq->poll_mode = LPFC_QUEUE_WORK;
+ else
+ cq->poll_mode = LPFC_IRQ_POLL;
+
+ idle_stat->prev_idle = wall_idle;
+ idle_stat->prev_wall = wall;
+ }
+
+requeue:
+ schedule_delayed_work(&phba->idle_stat_delay_work,
+ msecs_to_jiffies(LPFC_IDLE_STAT_DELAY));
+}
+
static void
lpfc_hb_eq_delay_work(struct work_struct *work)
{
@@ -1538,11 +1603,11 @@
return;
}
- lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
- "0479 Deferred Adapter Hardware Error "
- "Data: x%x x%x x%x\n",
- phba->work_hs,
- phba->work_status[0], phba->work_status[1]);
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
+ "0479 Deferred Adapter Hardware Error "
+ "Data: x%x x%x x%x\n",
+ phba->work_hs, phba->work_status[0],
+ phba->work_status[1]);
spin_lock_irq(&phba->hbalock);
psli->sli_flag &= ~LPFC_SLI_ACTIVE;
@@ -1693,7 +1758,7 @@
temp_event_data.event_code = LPFC_CRIT_TEMP;
temp_event_data.data = (uint32_t)temperature;
- lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"0406 Adapter maximum temperature exceeded "
"(%ld), taking this port offline "
"Data: x%x x%x x%x\n",
@@ -1717,7 +1782,7 @@
* failure is a value other than FFER6. Do not call the offline
* twice. This is the adapter hardware error path.
*/
- lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"0457 Adapter Hardware Error "
"Data: x%x x%x x%x\n",
phba->work_hs,
@@ -1738,7 +1803,7 @@
* lpfc_sli4_port_sta_fn_reset - The SLI4 function reset due to port status reg
* @phba: pointer to lpfc hba data structure.
* @mbx_action: flag for mailbox shutdown action.
- *
+ * @en_rn_msg: send reset/port recovery message.
* This routine is invoked to perform an SLI4 port PCI function reset in
* response to port status register polling attention. It waits for port
* status register (ERR, RDY, RN) bits before proceeding with function reset.
@@ -1765,7 +1830,7 @@
/* need reset: attempt for port recovery */
if (en_rn_msg)
- lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"2887 Reset Needed: Attempting Port "
"Recovery...\n");
lpfc_offline_prep(phba, mbx_action);
@@ -1775,14 +1840,14 @@
lpfc_sli4_disable_intr(phba);
rc = lpfc_sli_brdrestart(phba);
if (rc) {
- lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"6309 Failed to restart board\n");
return rc;
}
/* request and enable interrupt */
intr_mode = lpfc_sli4_enable_intr(phba, phba->intr_mode);
if (intr_mode == LPFC_INTR_ERROR) {
- lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"3175 Failed to enable interrupt\n");
return -EIO;
}
@@ -1821,7 +1886,7 @@
* we cannot communicate with the pci card anyway.
*/
if (pci_channel_offline(phba->pcidev)) {
- lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"3166 pci channel is offline\n");
lpfc_sli4_offline_eratt(phba);
return;
@@ -1844,7 +1909,7 @@
lpfc_sli4_offline_eratt(phba);
return;
}
- lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"7623 Checking UE recoverable");
for (i = 0; i < phba->sli4_hba.ue_to_sr / 1000; i++) {
@@ -1861,7 +1926,7 @@
msleep(1000);
}
- lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"4827 smphr_port_status x%x : Waited %dSec",
smphr_port_status, i);
@@ -1879,14 +1944,14 @@
LPFC_MBX_NO_WAIT, en_rn_msg);
if (rc == 0)
return;
- lpfc_printf_log(phba,
- KERN_ERR, LOG_INIT,
+ lpfc_printf_log(phba, KERN_ERR,
+ LOG_TRACE_EVENT,
"4215 Failed to recover UE");
break;
}
}
}
- lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"7624 Firmware not ready: Failing UE recovery,"
" waited %dSec", i);
phba->link_state = LPFC_HBA_ERROR;
@@ -1899,7 +1964,7 @@
&portstat_reg.word0);
/* consider PCI bus read error as pci_channel_offline */
if (pci_rd_rc1 == -EIO) {
- lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"3151 PCI bus read access failure: x%x\n",
readl(phba->sli4_hba.u.if_type2.STATUSregaddr));
lpfc_sli4_offline_eratt(phba);
@@ -1908,10 +1973,10 @@
reg_err1 = readl(phba->sli4_hba.u.if_type2.ERR1regaddr);
reg_err2 = readl(phba->sli4_hba.u.if_type2.ERR2regaddr);
if (bf_get(lpfc_sliport_status_oti, &portstat_reg)) {
- lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
- "2889 Port Overtemperature event, "
- "taking port offline Data: x%x x%x\n",
- reg_err1, reg_err2);
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
+ "2889 Port Overtemperature event, "
+ "taking port offline Data: x%x x%x\n",
+ reg_err1, reg_err2);
phba->sfp_alarm |= LPFC_TRANSGRESSION_HIGH_TEMPERATURE;
temp_event_data.event_type = FC_REG_TEMPERATURE_EVENT;
@@ -1933,17 +1998,17 @@
}
if (reg_err1 == SLIPORT_ERR1_REG_ERR_CODE_2 &&
reg_err2 == SLIPORT_ERR2_REG_FW_RESTART) {
- lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"3143 Port Down: Firmware Update "
"Detected\n");
en_rn_msg = false;
} else if (reg_err1 == SLIPORT_ERR1_REG_ERR_CODE_2 &&
reg_err2 == SLIPORT_ERR2_REG_FORCED_DUMP)
- lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"3144 Port Down: Debug Dump\n");
else if (reg_err1 == SLIPORT_ERR1_REG_ERR_CODE_2 &&
reg_err2 == SLIPORT_ERR2_REG_FUNC_PROVISON)
- lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"3145 Port Down: Provisioning\n");
/* If resets are disabled then leave the HBA alone and return */
@@ -1962,7 +2027,7 @@
break;
}
/* fall through for not able to recover */
- lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"3152 Unrecoverable error\n");
phba->link_state = LPFC_HBA_ERROR;
break;
@@ -2080,8 +2145,8 @@
lpfc_linkdown(phba);
phba->link_state = LPFC_HBA_ERROR;
- lpfc_printf_log(phba, KERN_ERR, LOG_MBOX,
- "0300 LATT: Cannot issue READ_LA: Data:%d\n", rc);
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
+ "0300 LATT: Cannot issue READ_LA: Data:%d\n", rc);
return;
}
@@ -2830,12 +2895,13 @@
*/
while (!list_empty(&vport->fc_nodes)) {
if (i++ > 3000) {
- lpfc_printf_vlog(vport, KERN_ERR, LOG_DISCOVERY,
+ lpfc_printf_vlog(vport, KERN_ERR,
+ LOG_TRACE_EVENT,
"0233 Nodelist not empty\n");
list_for_each_entry_safe(ndlp, next_ndlp,
&vport->fc_nodes, nlp_listp) {
lpfc_printf_vlog(ndlp->vport, KERN_ERR,
- LOG_NODE,
+ LOG_TRACE_EVENT,
"0282 did:x%x ndlp:x%px "
"usgmap:x%x refcnt:%d\n",
ndlp->nlp_DID, (void *)ndlp,
@@ -2922,6 +2988,7 @@
if (phba->pport)
lpfc_stop_vport_timers(phba->pport);
cancel_delayed_work_sync(&phba->eq_delay_work);
+ cancel_delayed_work_sync(&phba->idle_stat_delay_work);
del_timer_sync(&phba->sli.mbox_tmo);
del_timer_sync(&phba->fabric_block_timer);
del_timer_sync(&phba->eratt_poll);
@@ -2942,7 +3009,7 @@
lpfc_sli4_stop_fcf_redisc_wait_timer(phba);
break;
default:
- lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"0297 Invalid device group (x%x)\n",
phba->pci_dev_grp);
break;
@@ -2953,6 +3020,7 @@
/**
* lpfc_block_mgmt_io - Mark a HBA's management interface as blocked
* @phba: pointer to lpfc hba data structure.
+ * @mbx_action: flag for mailbox no wait action.
*
* This routine marks a HBA's management interface as blocked. Once the HBA's
* management interface is marked as blocked, all the user space access to
@@ -2989,10 +3057,10 @@
/* Check active mailbox complete status every 2ms */
msleep(2);
if (time_after(jiffies, timeout)) {
- lpfc_printf_log(phba, KERN_ERR, LOG_SLI,
- "2813 Mgmt IO is Blocked %x "
- "- mbox cmd %x still active\n",
- phba->sli.sli_flag, actcmd);
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
+ "2813 Mgmt IO is Blocked %x "
+ "- mbox cmd %x still active\n",
+ phba->sli.sli_flag, actcmd);
break;
}
}
@@ -3337,7 +3405,7 @@
!phba->nvmet_support) {
error = lpfc_nvme_create_localport(phba->pport);
if (error)
- lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"6132 NVME restore reg failed "
"on nvmei error x%x\n", error);
}
@@ -3403,6 +3471,7 @@
/**
* lpfc_offline_prep - Prepare a HBA to be brought offline
* @phba: pointer to lpfc hba data structure.
+ * @mbx_action: flag for mailbox shutdown action.
*
* This routine is invoked to prepare a HBA to be brought offline. It performs
* unregistration login to all the nodes on all vports and flushes the mailbox
@@ -3677,7 +3746,8 @@
sglq_entry = kzalloc(sizeof(struct lpfc_sglq),
GFP_KERNEL);
if (sglq_entry == NULL) {
- lpfc_printf_log(phba, KERN_ERR, LOG_SLI,
+ lpfc_printf_log(phba, KERN_ERR,
+ LOG_TRACE_EVENT,
"2562 Failure to allocate an "
"ELS sgl entry:%d\n", i);
rc = -ENOMEM;
@@ -3688,7 +3758,8 @@
&sglq_entry->phys);
if (sglq_entry->virt == NULL) {
kfree(sglq_entry);
- lpfc_printf_log(phba, KERN_ERR, LOG_SLI,
+ lpfc_printf_log(phba, KERN_ERR,
+ LOG_TRACE_EVENT,
"2563 Failure to allocate an "
"ELS mbuf:%d\n", i);
rc = -ENOMEM;
@@ -3743,7 +3814,8 @@
&phba->sli4_hba.lpfc_els_sgl_list, list) {
lxri = lpfc_sli4_next_xritag(phba);
if (lxri == NO_XRI) {
- lpfc_printf_log(phba, KERN_ERR, LOG_SLI,
+ lpfc_printf_log(phba, KERN_ERR,
+ LOG_TRACE_EVENT,
"2400 Failed to allocate xri for "
"ELS sgl\n");
rc = -ENOMEM;
@@ -3798,7 +3870,8 @@
sglq_entry = kzalloc(sizeof(struct lpfc_sglq),
GFP_KERNEL);
if (sglq_entry == NULL) {
- lpfc_printf_log(phba, KERN_ERR, LOG_SLI,
+ lpfc_printf_log(phba, KERN_ERR,
+ LOG_TRACE_EVENT,
"6303 Failure to allocate an "
"NVMET sgl entry:%d\n", i);
rc = -ENOMEM;
@@ -3809,7 +3882,8 @@
&sglq_entry->phys);
if (sglq_entry->virt == NULL) {
kfree(sglq_entry);
- lpfc_printf_log(phba, KERN_ERR, LOG_SLI,
+ lpfc_printf_log(phba, KERN_ERR,
+ LOG_TRACE_EVENT,
"6304 Failure to allocate an "
"NVMET buf:%d\n", i);
rc = -ENOMEM;
@@ -3865,7 +3939,8 @@
&phba->sli4_hba.lpfc_nvmet_sgl_list, list) {
lxri = lpfc_sli4_next_xritag(phba);
if (lxri == NO_XRI) {
- lpfc_printf_log(phba, KERN_ERR, LOG_SLI,
+ lpfc_printf_log(phba, KERN_ERR,
+ LOG_TRACE_EVENT,
"6307 Failed to allocate xri for "
"NVMET sgl\n");
rc = -ENOMEM;
@@ -4039,7 +4114,8 @@
&io_sgl_list, list) {
lxri = lpfc_sli4_next_xritag(phba);
if (lxri == NO_XRI) {
- lpfc_printf_log(phba, KERN_ERR, LOG_SLI,
+ lpfc_printf_log(phba, KERN_ERR,
+ LOG_TRACE_EVENT,
"6075 Failed to allocate xri for "
"nvme buffer\n");
rc = -ENOMEM;
@@ -4058,8 +4134,8 @@
/**
* lpfc_new_io_buf - IO buffer allocator for HBA with SLI4 IF spec
- * @vport: The virtual port for which this call being executed.
- * @num_to_allocate: The requested number of buffers to allocate.
+ * @phba: Pointer to lpfc hba data structure.
+ * @num_to_alloc: The requested number of buffers to allocate.
*
* This routine allocates nvme buffers for device with SLI-4 interface spec,
* the nvme buffer contains all the necessary information needed to initiate
@@ -4109,7 +4185,8 @@
if ((phba->sli3_options & LPFC_SLI3_BG_ENABLED) &&
(((unsigned long)(lpfc_ncmd->data) &
(unsigned long)(SLI4_PAGE_SIZE - 1)) != 0)) {
- lpfc_printf_log(phba, KERN_ERR, LOG_FCP,
+ lpfc_printf_log(phba, KERN_ERR,
+ LOG_TRACE_EVENT,
"3369 Memory alignment err: "
"addr=%lx\n",
(unsigned long)lpfc_ncmd->data);
@@ -4138,7 +4215,7 @@
dma_pool_free(phba->lpfc_sg_dma_buf_pool,
lpfc_ncmd->data, lpfc_ncmd->dma_handle);
kfree(lpfc_ncmd);
- lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"6121 Failed to allocate IOTAG for"
" XRI:0x%x\n", lxri);
lpfc_sli4_free_xri(phba, lxri);
@@ -4189,7 +4266,7 @@
lpfc_read_nv(phba, mboxq);
rc = lpfc_sli_issue_mbox(phba, mboxq, MBX_POLL);
if (rc != MBX_SUCCESS) {
- lpfc_printf_log(phba, KERN_ERR, LOG_SLI,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"6019 Mailbox failed , mbxCmd x%x "
"READ_NV, mbxStatus x%x\n",
bf_get(lpfc_mqe_command, &mboxq->u.mqe),
@@ -4249,7 +4326,8 @@
for (i = 0; i < lpfc_no_hba_reset_cnt; i++) {
if (wwn == lpfc_no_hba_reset[i]) {
- lpfc_printf_log(phba, KERN_ERR, LOG_SLI,
+ lpfc_printf_log(phba, KERN_ERR,
+ LOG_TRACE_EVENT,
"6020 Setting use_no_reset port=%llx\n",
wwn);
use_no_reset_hba = true;
@@ -4646,7 +4724,7 @@
/**
* lpfc_sli4_fcf_redisc_wait_tmo - FCF table rediscover wait timeout
- * @ptr: Map to lpfc_hba data structure pointer.
+ * @t: Timer context used to obtain the pointer to lpfc hba data structure.
*
* This routine is invoked when waiting for FCF table rediscover has been
* timed out. If new FCF record(s) has (have) been discovered during the
@@ -4694,7 +4772,7 @@
case LPFC_ASYNC_LINK_FAULT_LR_LRR:
break;
default:
- lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"0398 Unknown link fault code: x%x\n",
bf_get(lpfc_acqe_link_fault, acqe_link));
break;
@@ -4730,7 +4808,7 @@
att_type = LPFC_ATT_LINK_UP;
break;
default:
- lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"0399 Invalid link attention type: x%x\n",
bf_get(lpfc_acqe_link_status, acqe_link));
att_type = LPFC_ATT_RESERVED;
@@ -4902,19 +4980,19 @@
phba->fcoe_eventtag = acqe_link->event_tag;
pmb = (LPFC_MBOXQ_t *)mempool_alloc(phba->mbox_mem_pool, GFP_KERNEL);
if (!pmb) {
- lpfc_printf_log(phba, KERN_ERR, LOG_SLI,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"0395 The mboxq allocation failed\n");
return;
}
mp = kmalloc(sizeof(struct lpfc_dmabuf), GFP_KERNEL);
if (!mp) {
- lpfc_printf_log(phba, KERN_ERR, LOG_SLI,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"0396 The lpfc_dmabuf allocation failed\n");
goto out_free_pmb;
}
mp->virt = lpfc_mbuf_alloc(phba, 0, &mp->phys);
if (!mp->virt) {
- lpfc_printf_log(phba, KERN_ERR, LOG_SLI,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"0397 The mbuf allocation failed\n");
goto out_free_dmabuf;
}
@@ -5013,7 +5091,6 @@
* lpfc_async_link_speed_to_read_top - Parse async evt link speed code to read
* topology.
* @phba: pointer to lpfc hba data structure.
- * @evt_code: asynchronous event code.
* @speed_code: asynchronous event link speed code.
*
* This routine is to parse the giving SLI4 async event link speed code into
@@ -5115,7 +5192,7 @@
phba->trunk_link.link3.fault = port_fault & 0x8 ? err : 0;
}
- lpfc_printf_log(phba, KERN_ERR, LOG_SLI,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"2910 Async FC Trunking Event - Speed:%d\n"
"\tLogical speed:%d "
"port0: %s port1: %s port2: %s port3: %s\n",
@@ -5125,7 +5202,7 @@
trunk_link_status(2), trunk_link_status(3));
if (port_fault)
- lpfc_printf_log(phba, KERN_ERR, LOG_SLI,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"3202 trunk error:0x%x (%s) seen on port0:%s "
/*
* SLI-4: We have only 0xA error codes
@@ -5159,7 +5236,7 @@
if (bf_get(lpfc_trailer_type, acqe_fc) !=
LPFC_FC_LA_EVENT_TYPE_FC_LINK) {
- lpfc_printf_log(phba, KERN_ERR, LOG_SLI,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"2895 Non FC link Event detected.(%d)\n",
bf_get(lpfc_trailer_type, acqe_fc));
return;
@@ -5207,19 +5284,19 @@
phba->sli4_hba.link_state.fault);
pmb = (LPFC_MBOXQ_t *)mempool_alloc(phba->mbox_mem_pool, GFP_KERNEL);
if (!pmb) {
- lpfc_printf_log(phba, KERN_ERR, LOG_SLI,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"2897 The mboxq allocation failed\n");
return;
}
mp = kmalloc(sizeof(struct lpfc_dmabuf), GFP_KERNEL);
if (!mp) {
- lpfc_printf_log(phba, KERN_ERR, LOG_SLI,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"2898 The lpfc_dmabuf allocation failed\n");
goto out_free_pmb;
}
mp->virt = lpfc_mbuf_alloc(phba, 0, &mp->phys);
if (!mp->virt) {
- lpfc_printf_log(phba, KERN_ERR, LOG_SLI,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"2899 The mbuf allocation failed\n");
goto out_free_dmabuf;
}
@@ -5291,7 +5368,7 @@
/**
* lpfc_sli4_async_sli_evt - Process the asynchronous SLI link event
* @phba: pointer to lpfc hba data structure.
- * @acqe_fc: pointer to the async SLI completion queue entry.
+ * @acqe_sli: pointer to the async SLI completion queue entry.
*
* This routine is to handle the SLI4 asynchronous SLI events.
**/
@@ -5386,7 +5463,7 @@
&misconfigured->theEvent);
break;
default:
- lpfc_printf_log(phba, KERN_ERR, LOG_SLI,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"3296 "
"LPFC_SLI_EVENT_TYPE_MISCONFIGURED "
"event: Invalid link %d",
@@ -5438,7 +5515,8 @@
rc = lpfc_sli4_read_config(phba);
if (rc) {
phba->lmt = 0;
- lpfc_printf_log(phba, KERN_ERR, LOG_SLI,
+ lpfc_printf_log(phba, KERN_ERR,
+ LOG_TRACE_EVENT,
"3194 Unable to retrieve supported "
"speeds, rc = 0x%x\n", rc);
}
@@ -5546,7 +5624,7 @@
/**
* lpfc_sli4_perform_all_vport_cvl - Perform clear virtual link on all vports
- * @vport: pointer to lpfc hba data structure.
+ * @phba: pointer to lpfc hba data structure.
*
* This routine is to perform Clear Virtual Link (CVL) on all vports in
* response to a FCF dead event.
@@ -5567,7 +5645,7 @@
/**
* lpfc_sli4_async_fip_evt - Process the asynchronous FCoE FIP event
* @phba: pointer to lpfc hba data structure.
- * @acqe_link: pointer to the async fcoe completion queue entry.
+ * @acqe_fip: pointer to the async fcoe completion queue entry.
*
* This routine is to handle the SLI4 asynchronous fcoe event.
**/
@@ -5590,8 +5668,7 @@
case LPFC_FIP_EVENT_TYPE_NEW_FCF:
case LPFC_FIP_EVENT_TYPE_FCF_PARAM_MOD:
if (event_type == LPFC_FIP_EVENT_TYPE_NEW_FCF)
- lpfc_printf_log(phba, KERN_ERR, LOG_FIP |
- LOG_DISCOVERY,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"2546 New FCF event, evt_tag:x%x, "
"index:x%x\n",
acqe_fip->event_tag,
@@ -5644,23 +5721,24 @@
rc = lpfc_sli4_fcf_scan_read_fcf_rec(phba,
LPFC_FCOE_FCF_GET_FIRST);
if (rc)
- lpfc_printf_log(phba, KERN_ERR, LOG_FIP | LOG_DISCOVERY,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"2547 Issue FCF scan read FCF mailbox "
"command failed (x%x)\n", rc);
break;
case LPFC_FIP_EVENT_TYPE_FCF_TABLE_FULL:
- lpfc_printf_log(phba, KERN_ERR, LOG_SLI,
- "2548 FCF Table full count 0x%x tag 0x%x\n",
- bf_get(lpfc_acqe_fip_fcf_count, acqe_fip),
- acqe_fip->event_tag);
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
+ "2548 FCF Table full count 0x%x tag 0x%x\n",
+ bf_get(lpfc_acqe_fip_fcf_count, acqe_fip),
+ acqe_fip->event_tag);
break;
case LPFC_FIP_EVENT_TYPE_FCF_DEAD:
phba->fcoe_cvl_eventtag = acqe_fip->event_tag;
- lpfc_printf_log(phba, KERN_ERR, LOG_FIP | LOG_DISCOVERY,
- "2549 FCF (x%x) disconnected from network, "
- "tag:x%x\n", acqe_fip->index, acqe_fip->event_tag);
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
+ "2549 FCF (x%x) disconnected from network, "
+ "tag:x%x\n", acqe_fip->index,
+ acqe_fip->event_tag);
/*
* If we are in the middle of FCF failover process, clear
* the corresponding FCF bit in the roundrobin bitmap.
@@ -5697,7 +5775,7 @@
rc = lpfc_sli4_redisc_fcf_table(phba);
if (rc) {
lpfc_printf_log(phba, KERN_ERR, LOG_FIP |
- LOG_DISCOVERY,
+ LOG_TRACE_EVENT,
"2772 Issue FCF rediscover mailbox "
"command failed, fail through to FCF "
"dead event\n");
@@ -5721,7 +5799,8 @@
break;
case LPFC_FIP_EVENT_TYPE_CVL:
phba->fcoe_cvl_eventtag = acqe_fip->event_tag;
- lpfc_printf_log(phba, KERN_ERR, LOG_FIP | LOG_DISCOVERY,
+ lpfc_printf_log(phba, KERN_ERR,
+ LOG_TRACE_EVENT,
"2718 Clear Virtual Link Received for VPI 0x%x"
" tag 0x%x\n", acqe_fip->index, acqe_fip->event_tag);
@@ -5788,7 +5867,7 @@
rc = lpfc_sli4_redisc_fcf_table(phba);
if (rc) {
lpfc_printf_log(phba, KERN_ERR, LOG_FIP |
- LOG_DISCOVERY,
+ LOG_TRACE_EVENT,
"2774 Issue FCF rediscover "
"mailbox command failed, "
"through to CVL event\n");
@@ -5809,9 +5888,9 @@
}
break;
default:
- lpfc_printf_log(phba, KERN_ERR, LOG_SLI,
- "0288 Unknown FCoE event type 0x%x event tag "
- "0x%x\n", event_type, acqe_fip->event_tag);
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
+ "0288 Unknown FCoE event type 0x%x event tag "
+ "0x%x\n", event_type, acqe_fip->event_tag);
break;
}
}
@@ -5819,7 +5898,7 @@
/**
* lpfc_sli4_async_dcbx_evt - Process the asynchronous dcbx event
* @phba: pointer to lpfc hba data structure.
- * @acqe_link: pointer to the async dcbx completion queue entry.
+ * @acqe_dcbx: pointer to the async dcbx completion queue entry.
*
* This routine is to handle the SLI4 asynchronous dcbx event.
**/
@@ -5828,7 +5907,7 @@
struct lpfc_acqe_dcbx *acqe_dcbx)
{
phba->fc_eventTag = acqe_dcbx->event_tag;
- lpfc_printf_log(phba, KERN_ERR, LOG_SLI,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"0290 The SLI4 DCBX asynchronous event is not "
"handled yet\n");
}
@@ -5836,7 +5915,7 @@
/**
* lpfc_sli4_async_grp5_evt - Process the asynchronous group5 event
* @phba: pointer to lpfc hba data structure.
- * @acqe_link: pointer to the async grp5 completion queue entry.
+ * @acqe_grp5: pointer to the async grp5 completion queue entry.
*
* This routine is to handle the SLI4 asynchronous grp5 event. A grp5 event
* is an asynchronous notified of a logical link speed change. The Port
@@ -5905,7 +5984,8 @@
lpfc_sli4_async_sli_evt(phba, &cq_event->cqe.acqe_sli);
break;
default:
- lpfc_printf_log(phba, KERN_ERR, LOG_SLI,
+ lpfc_printf_log(phba, KERN_ERR,
+ LOG_TRACE_EVENT,
"1804 Invalid asynchronous event code: "
"x%x\n", bf_get(lpfc_trailer_code,
&cq_event->cqe.mcqe_cmpl));
@@ -5941,7 +6021,7 @@
"2777 Start post-quiescent FCF table scan\n");
rc = lpfc_sli4_fcf_scan_read_fcf_rec(phba, LPFC_FCOE_FCF_GET_FIRST);
if (rc)
- lpfc_printf_log(phba, KERN_ERR, LOG_FIP | LOG_DISCOVERY,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"2747 Issue FCF scan read FCF mailbox "
"command failed 0x%x\n", rc);
}
@@ -6012,7 +6092,7 @@
"0480 Enabled MSI-X interrupt mode.\n");
break;
default:
- lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"0482 Illegal interrupt mode.\n");
break;
}
@@ -6060,7 +6140,7 @@
out_disable_device:
pci_disable_device(pdev);
out_error:
- lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"1401 Failed to enable pci device\n");
return -ENODEV;
}
@@ -6161,7 +6241,7 @@
max_nr_vfn = lpfc_sli_sriov_nr_virtfn_get(phba);
if (nr_vfn > max_nr_vfn) {
- lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"3057 Requested vfs (%d) greater than "
"supported vfs (%d)", nr_vfn, max_nr_vfn);
return -EINVAL;
@@ -6200,6 +6280,9 @@
* Driver resources common to all SLI revisions
*/
atomic_set(&phba->fast_event_count, 0);
+ atomic_set(&phba->dbg_log_idx, 0);
+ atomic_set(&phba->dbg_log_cnt, 0);
+ atomic_set(&phba->dbg_log_dmping, 0);
spin_lock_init(&phba->hbalock);
/* Initialize ndlp management spinlock */
@@ -6253,6 +6336,9 @@
INIT_DELAYED_WORK(&phba->eq_delay_work, lpfc_hb_eq_delay_work);
+ INIT_DELAYED_WORK(&phba->idle_stat_delay_work,
+ lpfc_idle_stat_delay_work);
+
return 0;
}
@@ -6624,7 +6710,8 @@
lpfc_read_nv(phba, mboxq);
rc = lpfc_sli_issue_mbox(phba, mboxq, MBX_POLL);
if (rc != MBX_SUCCESS) {
- lpfc_printf_log(phba, KERN_ERR, LOG_SLI,
+ lpfc_printf_log(phba, KERN_ERR,
+ LOG_TRACE_EVENT,
"6016 Mailbox failed , mbxCmd x%x "
"READ_NV, mbxStatus x%x\n",
bf_get(lpfc_mqe_command, &mboxq->u.mqe),
@@ -6653,11 +6740,13 @@
phba->nvmet_support = 1; /* a match */
- lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+ lpfc_printf_log(phba, KERN_ERR,
+ LOG_TRACE_EVENT,
"6017 NVME Target %016llx\n",
wwn);
#else
- lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+ lpfc_printf_log(phba, KERN_ERR,
+ LOG_TRACE_EVENT,
"6021 Can't enable NVME Target."
" NVME_TARGET_FC infrastructure"
" is not in kernel\n");
@@ -6717,9 +6806,9 @@
&phba->sli4_hba.sli_intf);
if (phba->sli4_hba.extents_in_use &&
phba->sli4_hba.rpi_hdrs_in_use) {
- lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
- "2999 Unsupported SLI4 Parameters "
- "Extents and RPI headers enabled.\n");
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
+ "2999 Unsupported SLI4 Parameters "
+ "Extents and RPI headers enabled.\n");
if (if_type == LPFC_SLI_INTF_IF_TYPE_0 &&
if_fam == LPFC_SLI_INTF_FAMILY_BE2) {
mempool_free(mboxq, phba->mbox_mem_pool);
@@ -6879,13 +6968,13 @@
/* Allocate and initialize active sgl array */
rc = lpfc_init_active_sgl_array(phba);
if (rc) {
- lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"1430 Failed to initialize sgl list.\n");
goto out_destroy_cq_event_pool;
}
rc = lpfc_sli4_init_rpi_hdrs(phba);
if (rc) {
- lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"1432 Failed to initialize rpi headers.\n");
goto out_free_active_sgl;
}
@@ -6895,7 +6984,7 @@
phba->fcf.fcf_rr_bmask = kcalloc(longs, sizeof(unsigned long),
GFP_KERNEL);
if (!phba->fcf.fcf_rr_bmask) {
- lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"2759 Failed allocate memory for FCF round "
"robin failover bmask\n");
rc = -ENOMEM;
@@ -6906,7 +6995,7 @@
sizeof(struct lpfc_hba_eq_hdl),
GFP_KERNEL);
if (!phba->sli4_hba.hba_eq_hdl) {
- lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"2572 Failed allocate memory for "
"fast-path per-EQ handle array\n");
rc = -ENOMEM;
@@ -6917,7 +7006,7 @@
sizeof(struct lpfc_vector_map_info),
GFP_KERNEL);
if (!phba->sli4_hba.cpu_map) {
- lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"3327 Failed allocate memory for msi-x "
"interrupt vector mapping\n");
rc = -ENOMEM;
@@ -6926,19 +7015,29 @@
phba->sli4_hba.eq_info = alloc_percpu(struct lpfc_eq_intr_info);
if (!phba->sli4_hba.eq_info) {
- lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"3321 Failed allocation for per_cpu stats\n");
rc = -ENOMEM;
goto out_free_hba_cpu_map;
}
+ phba->sli4_hba.idle_stat = kcalloc(phba->sli4_hba.num_possible_cpu,
+ sizeof(*phba->sli4_hba.idle_stat),
+ GFP_KERNEL);
+ if (!phba->sli4_hba.idle_stat) {
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
+ "3390 Failed allocation for idle_stat\n");
+ rc = -ENOMEM;
+ goto out_free_hba_eq_info;
+ }
+
#ifdef CONFIG_SCSI_LPFC_DEBUG_FS
phba->sli4_hba.c_stat = alloc_percpu(struct lpfc_hdwq_stat);
if (!phba->sli4_hba.c_stat) {
- lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"3332 Failed allocating per cpu hdwq stats\n");
rc = -ENOMEM;
- goto out_free_hba_eq_info;
+ goto out_free_hba_idle_stat;
}
#endif
@@ -6962,9 +7061,11 @@
return 0;
#ifdef CONFIG_SCSI_LPFC_DEBUG_FS
+out_free_hba_idle_stat:
+ kfree(phba->sli4_hba.idle_stat);
+#endif
out_free_hba_eq_info:
free_percpu(phba->sli4_hba.eq_info);
-#endif
out_free_hba_cpu_map:
kfree(phba->sli4_hba.cpu_map);
out_free_hba_eq_hdl:
@@ -7006,6 +7107,7 @@
#ifdef CONFIG_SCSI_LPFC_DEBUG_FS
free_percpu(phba->sli4_hba.c_stat);
#endif
+ kfree(phba->sli4_hba.idle_stat);
/* Free memory allocated for msi-x interrupt vector to CPU mapping */
kfree(phba->sli4_hba.cpu_map);
@@ -7080,7 +7182,7 @@
phba->lpfc_stop_port = lpfc_stop_port_s4;
break;
default:
- lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"1431 Invalid HBA PCI-device group: 0x%x\n",
dev_grp);
return -ENODEV;
@@ -7164,6 +7266,7 @@
/**
* lpfc_init_iocb_list - Allocate and initialize iocb list.
* @phba: pointer to lpfc hba data structure.
+ * @iocb_count: number of requested iocbs
*
* This routine is invoked to allocate and initizlize the driver's IOCB
* list and set up the IOCB tag array accordingly.
@@ -7375,7 +7478,7 @@
rpi_hdr = lpfc_sli4_create_rpi_hdr(phba);
if (!rpi_hdr) {
- lpfc_printf_log(phba, KERN_ERR, LOG_MBOX | LOG_SLI,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"0391 Error during rpi post operation\n");
lpfc_sli4_remove_rpis(phba);
rc = -ENODEV;
@@ -7687,7 +7790,7 @@
if (phba->cfg_prot_mask && phba->cfg_prot_guard) {
if ((old_mask != phba->cfg_prot_mask) ||
(old_guard != phba->cfg_prot_guard))
- lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"1475 Registering BlockGuard with the "
"SCSI layer: mask %d guard %d\n",
phba->cfg_prot_mask,
@@ -7696,7 +7799,7 @@
scsi_host_set_prot(shost, phba->cfg_prot_mask);
scsi_host_set_guard(shost, phba->cfg_prot_guard);
} else
- lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"1479 Not Registering BlockGuard with the SCSI "
"layer, Bad protection parameters: %d %d\n",
old_mask, old_guard);
@@ -7927,7 +8030,7 @@
* other register reads as the data may not be valid. Just exit.
*/
if (port_error) {
- lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"1408 Port Failed POST - portsmphr=0x%x, "
"perr=x%x, sfi=x%x, nip=x%x, ipc=x%x, scr1=x%x, "
"scr2=x%x, hscratch=x%x, pstatus=x%x\n",
@@ -7976,7 +8079,8 @@
readl(phba->sli4_hba.u.if_type0.UERRHIregaddr);
if ((~phba->sli4_hba.ue_mask_lo & uerrlo_reg.word0) ||
(~phba->sli4_hba.ue_mask_hi & uerrhi_reg.word0)) {
- lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+ lpfc_printf_log(phba, KERN_ERR,
+ LOG_TRACE_EVENT,
"1422 Unrecoverable Error "
"Detected during POST "
"uerr_lo_reg=0x%x, "
@@ -8003,7 +8107,7 @@
phba->work_status[1] =
readl(phba->sli4_hba.u.if_type2.
ERR2regaddr);
- lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"2888 Unrecoverable port error "
"following POST: port status reg "
"0x%x, port_smphr reg 0x%x, "
@@ -8116,6 +8220,7 @@
/**
* lpfc_sli4_bar1_register_memmap - Set up SLI4 BAR1 register memory map.
* @phba: pointer to lpfc hba data structure.
+ * @if_type: sli if type to operate on.
*
* This routine is invoked to set up SLI4 BAR1 register memory map.
**/
@@ -8297,20 +8402,19 @@
"P2P then Loop",
};
+#define LINK_FLAGS_DEF 0x0
+#define LINK_FLAGS_P2P 0x1
+#define LINK_FLAGS_LOOP 0x2
/**
* lpfc_map_topology - Map the topology read from READ_CONFIG
* @phba: pointer to lpfc hba data structure.
- * @rdconf: pointer to read config data
+ * @rd_config: pointer to read config data
*
* This routine is invoked to map the topology values as read
* from the read config mailbox command. If the persistent
* topology feature is supported, the firmware will provide the
* saved topology information to be used in INIT_LINK
- *
**/
-#define LINK_FLAGS_DEF 0x0
-#define LINK_FLAGS_P2P 0x1
-#define LINK_FLAGS_LOOP 0x2
static void
lpfc_map_topology(struct lpfc_hba *phba, struct lpfc_mbx_read_config *rd_config)
{
@@ -8397,7 +8501,7 @@
pmb = (LPFC_MBOXQ_t *) mempool_alloc(phba->mbox_mem_pool, GFP_KERNEL);
if (!pmb) {
- lpfc_printf_log(phba, KERN_ERR, LOG_SLI,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"2011 Unable to allocate memory for issuing "
"SLI_CONFIG_SPECIAL mailbox command\n");
return -ENOMEM;
@@ -8407,11 +8511,11 @@
rc = lpfc_sli_issue_mbox(phba, pmb, MBX_POLL);
if (rc != MBX_SUCCESS) {
- lpfc_printf_log(phba, KERN_ERR, LOG_SLI,
- "2012 Mailbox failed , mbxCmd x%x "
- "READ_CONFIG, mbxStatus x%x\n",
- bf_get(lpfc_mqe_command, &pmb->u.mqe),
- bf_get(lpfc_mqe_status, &pmb->u.mqe));
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
+ "2012 Mailbox failed , mbxCmd x%x "
+ "READ_CONFIG, mbxStatus x%x\n",
+ bf_get(lpfc_mqe_command, &pmb->u.mqe),
+ bf_get(lpfc_mqe_status, &pmb->u.mqe));
rc = -EIO;
} else {
rd_config = &pmb->u.mqe.un.rd_config;
@@ -8521,7 +8625,7 @@
/* Check to see if there is enough for NVME */
if ((phba->cfg_irq_chann > qmin) ||
(phba->cfg_hdw_queue > qmin)) {
- lpfc_printf_log(phba, KERN_ERR, LOG_SLI,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"2005 Reducing Queues: "
"WQ %d CQ %d EQ %d: min %d: "
"IRQ %d HDWQ %d\n",
@@ -8587,7 +8691,8 @@
LPFC_USER_LINK_SPEED_AUTO;
break;
default:
- lpfc_printf_log(phba, KERN_ERR, LOG_SLI,
+ lpfc_printf_log(phba, KERN_ERR,
+ LOG_TRACE_EVENT,
"0047 Unrecognized link "
"speed : %d\n",
forced_link_speed);
@@ -8624,7 +8729,7 @@
shdr_status = bf_get(lpfc_mbox_hdr_status, &shdr->response);
shdr_add_status = bf_get(lpfc_mbox_hdr_add_status, &shdr->response);
if (rc2 || shdr_status || shdr_add_status) {
- lpfc_printf_log(phba, KERN_ERR, LOG_SLI,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"3026 Mailbox failed , mbxCmd x%x "
"GET_FUNCTION_CONFIG, mbxStatus x%x\n",
bf_get(lpfc_mqe_command, &pmb->u.mqe),
@@ -8661,7 +8766,7 @@
"vf_number:%d\n", phba->sli4_hba.iov.pf_number,
phba->sli4_hba.iov.vf_number);
else
- lpfc_printf_log(phba, KERN_ERR, LOG_SLI,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"3028 GET_FUNCTION_CONFIG: failed to find "
"Resource Descriptor:x%x\n",
LPFC_RSRC_DESC_TYPE_FCFCOE);
@@ -8698,7 +8803,7 @@
mboxq = (LPFC_MBOXQ_t *) mempool_alloc(phba->mbox_mem_pool,
GFP_KERNEL);
if (!mboxq) {
- lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"0492 Unable to allocate memory for "
"issuing SLI_CONFIG_SPECIAL mailbox "
"command\n");
@@ -8713,7 +8818,7 @@
memcpy(&mboxq->u.mqe, &endian_mb_data, sizeof(endian_mb_data));
rc = lpfc_sli_issue_mbox(phba, mboxq, MBX_POLL);
if (rc != MBX_SUCCESS) {
- lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"0493 SLI_CONFIG_SPECIAL mailbox "
"failed with status x%x\n",
rc);
@@ -8793,8 +8898,9 @@
phba->sli4_hba.cq_esize,
phba->sli4_hba.cq_ecount, cpu);
if (!qdesc) {
- lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
- "0499 Failed allocate fast-path IO CQ (%d)\n", idx);
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
+ "0499 Failed allocate fast-path IO CQ (%d)\n",
+ idx);
return 1;
}
qdesc->qe_valid = 1;
@@ -8816,7 +8922,7 @@
phba->sli4_hba.wq_ecount, cpu);
if (!qdesc) {
- lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"0503 Failed allocate fast-path IO WQ (%d)\n",
idx);
return 1;
@@ -8872,7 +8978,7 @@
phba->cfg_hdw_queue, sizeof(struct lpfc_sli4_hdw_queue),
GFP_KERNEL);
if (!phba->sli4_hba.hdwq) {
- lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"6427 Failed allocate memory for "
"fast-path Hardware Queue array\n");
goto out_error;
@@ -8904,7 +9010,7 @@
sizeof(struct lpfc_queue *),
GFP_KERNEL);
if (!phba->sli4_hba.nvmet_cqset) {
- lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"3121 Fail allocate memory for "
"fast-path CQ set array\n");
goto out_error;
@@ -8914,7 +9020,7 @@
sizeof(struct lpfc_queue *),
GFP_KERNEL);
if (!phba->sli4_hba.nvmet_mrq_hdr) {
- lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"3122 Fail allocate memory for "
"fast-path RQ set hdr array\n");
goto out_error;
@@ -8924,7 +9030,7 @@
sizeof(struct lpfc_queue *),
GFP_KERNEL);
if (!phba->sli4_hba.nvmet_mrq_data) {
- lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"3124 Fail allocate memory for "
"fast-path RQ set data array\n");
goto out_error;
@@ -8952,7 +9058,7 @@
phba->sli4_hba.eq_esize,
phba->sli4_hba.eq_ecount, cpu);
if (!qdesc) {
- lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"0497 Failed allocate EQ (%d)\n",
cpup->hdwq);
goto out_error;
@@ -9006,7 +9112,7 @@
phba->sli4_hba.cq_ecount,
cpu);
if (!qdesc) {
- lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"3142 Failed allocate NVME "
"CQ Set (%d)\n", idx);
goto out_error;
@@ -9028,7 +9134,7 @@
phba->sli4_hba.cq_esize,
phba->sli4_hba.cq_ecount, cpu);
if (!qdesc) {
- lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"0500 Failed allocate slow-path mailbox CQ\n");
goto out_error;
}
@@ -9040,7 +9146,7 @@
phba->sli4_hba.cq_esize,
phba->sli4_hba.cq_ecount, cpu);
if (!qdesc) {
- lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"0501 Failed allocate slow-path ELS CQ\n");
goto out_error;
}
@@ -9059,7 +9165,7 @@
phba->sli4_hba.mq_esize,
phba->sli4_hba.mq_ecount, cpu);
if (!qdesc) {
- lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"0505 Failed allocate slow-path MQ\n");
goto out_error;
}
@@ -9075,7 +9181,7 @@
phba->sli4_hba.wq_esize,
phba->sli4_hba.wq_ecount, cpu);
if (!qdesc) {
- lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"0504 Failed allocate slow-path ELS WQ\n");
goto out_error;
}
@@ -9089,7 +9195,7 @@
phba->sli4_hba.cq_esize,
phba->sli4_hba.cq_ecount, cpu);
if (!qdesc) {
- lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"6079 Failed allocate NVME LS CQ\n");
goto out_error;
}
@@ -9102,7 +9208,7 @@
phba->sli4_hba.wq_esize,
phba->sli4_hba.wq_ecount, cpu);
if (!qdesc) {
- lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"6080 Failed allocate NVME LS WQ\n");
goto out_error;
}
@@ -9120,7 +9226,7 @@
phba->sli4_hba.rq_esize,
phba->sli4_hba.rq_ecount, cpu);
if (!qdesc) {
- lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"0506 Failed allocate receive HRQ\n");
goto out_error;
}
@@ -9131,7 +9237,7 @@
phba->sli4_hba.rq_esize,
phba->sli4_hba.rq_ecount, cpu);
if (!qdesc) {
- lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"0507 Failed allocate receive DRQ\n");
goto out_error;
}
@@ -9149,7 +9255,7 @@
LPFC_NVMET_RQE_DEF_COUNT,
cpu);
if (!qdesc) {
- lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"3146 Failed allocate "
"receive HRQ\n");
goto out_error;
@@ -9162,7 +9268,7 @@
GFP_KERNEL,
cpu_to_node(cpu));
if (qdesc->rqbp == NULL) {
- lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"6131 Failed allocate "
"Header RQBP\n");
goto out_error;
@@ -9178,7 +9284,7 @@
LPFC_NVMET_RQE_DEF_COUNT,
cpu);
if (!qdesc) {
- lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"3156 Failed allocate "
"receive DRQ\n");
goto out_error;
@@ -9369,7 +9475,7 @@
int rc;
if (!eq || !cq || !wq) {
- lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"6085 Fast-path %s (%d) not allocated\n",
((eq) ? ((cq) ? "WQ" : "CQ") : "EQ"), qidx);
return -ENOMEM;
@@ -9379,9 +9485,9 @@
rc = lpfc_cq_create(phba, cq, eq,
(qtype == LPFC_MBOX) ? LPFC_MCQ : LPFC_WCQ, qtype);
if (rc) {
- lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
- "6086 Failed setup of CQ (%d), rc = 0x%x\n",
- qidx, (uint32_t)rc);
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
+ "6086 Failed setup of CQ (%d), rc = 0x%x\n",
+ qidx, (uint32_t)rc);
return rc;
}
@@ -9397,7 +9503,7 @@
/* create the wq */
rc = lpfc_wq_create(phba, wq, cq, qtype);
if (rc) {
- lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"4618 Fail setup fastpath WQ (%d), rc = 0x%x\n",
qidx, (uint32_t)rc);
/* no need to tear down cq - caller will do so */
@@ -9415,9 +9521,9 @@
} else {
rc = lpfc_mq_create(phba, wq, cq, LPFC_MBOX);
if (rc) {
- lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
- "0539 Failed setup of slow-path MQ: "
- "rc = 0x%x\n", rc);
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
+ "0539 Failed setup of slow-path MQ: "
+ "rc = 0x%x\n", rc);
/* no need to tear down cq - caller will do so */
return rc;
}
@@ -9490,7 +9596,7 @@
/* Check for dual-ULP support */
mboxq = (LPFC_MBOXQ_t *)mempool_alloc(phba->mbox_mem_pool, GFP_KERNEL);
if (!mboxq) {
- lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"3249 Unable to allocate memory for "
"QUERY_FW_CFG mailbox command\n");
return -ENOMEM;
@@ -9508,7 +9614,7 @@
shdr_status = bf_get(lpfc_mbox_hdr_status, &shdr->response);
shdr_add_status = bf_get(lpfc_mbox_hdr_add_status, &shdr->response);
if (shdr_status || shdr_add_status || rc) {
- lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"3250 QUERY_FW_CFG mailbox failed with status "
"x%x add_status x%x, mbx status x%x\n",
shdr_status, shdr_add_status, rc);
@@ -9539,7 +9645,7 @@
/* Set up HBA event queue */
if (!qp) {
- lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"3147 Fast-path EQs not allocated\n");
rc = -ENOMEM;
goto out_error;
@@ -9563,7 +9669,7 @@
rc = lpfc_eq_create(phba, qp[cpup->hdwq].hba_eq,
phba->cfg_fcp_imax);
if (rc) {
- lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"0523 Failed setup of fast-path"
" EQ (%d), rc = 0x%x\n",
cpup->eq, (uint32_t)rc);
@@ -9595,7 +9701,7 @@
qidx,
LPFC_IO);
if (rc) {
- lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"0535 Failed to setup fastpath "
"IO WQ/CQ (%d), rc = 0x%x\n",
qidx, (uint32_t)rc);
@@ -9610,7 +9716,7 @@
/* Set up slow-path MBOX CQ/MQ */
if (!phba->sli4_hba.mbx_cq || !phba->sli4_hba.mbx_wq) {
- lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"0528 %s not allocated\n",
phba->sli4_hba.mbx_cq ?
"Mailbox WQ" : "Mailbox CQ");
@@ -9623,14 +9729,14 @@
phba->sli4_hba.mbx_wq,
NULL, 0, LPFC_MBOX);
if (rc) {
- lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"0529 Failed setup of mailbox WQ/CQ: rc = 0x%x\n",
(uint32_t)rc);
goto out_destroy;
}
if (phba->nvmet_support) {
if (!phba->sli4_hba.nvmet_cqset) {
- lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"3165 Fast-path NVME CQ Set "
"array not allocated\n");
rc = -ENOMEM;
@@ -9642,7 +9748,7 @@
qp,
LPFC_WCQ, LPFC_NVMET);
if (rc) {
- lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"3164 Failed setup of NVME CQ "
"Set, rc = 0x%x\n",
(uint32_t)rc);
@@ -9654,7 +9760,7 @@
qp[0].hba_eq,
LPFC_WCQ, LPFC_NVMET);
if (rc) {
- lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"6089 Failed setup NVMET CQ: "
"rc = 0x%x\n", (uint32_t)rc);
goto out_destroy;
@@ -9671,7 +9777,7 @@
/* Set up slow-path ELS WQ/CQ */
if (!phba->sli4_hba.els_cq || !phba->sli4_hba.els_wq) {
- lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"0530 ELS %s not allocated\n",
phba->sli4_hba.els_cq ? "WQ" : "CQ");
rc = -ENOMEM;
@@ -9682,7 +9788,7 @@
phba->sli4_hba.els_wq,
NULL, 0, LPFC_ELS);
if (rc) {
- lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"0525 Failed setup of ELS WQ/CQ: rc = 0x%x\n",
(uint32_t)rc);
goto out_destroy;
@@ -9695,7 +9801,7 @@
if (phba->cfg_enable_fc4_type & LPFC_ENABLE_NVME) {
/* Set up NVME LS Complete Queue */
if (!phba->sli4_hba.nvmels_cq || !phba->sli4_hba.nvmels_wq) {
- lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"6091 LS %s not allocated\n",
phba->sli4_hba.nvmels_cq ? "WQ" : "CQ");
rc = -ENOMEM;
@@ -9706,7 +9812,7 @@
phba->sli4_hba.nvmels_wq,
NULL, 0, LPFC_NVME_LS);
if (rc) {
- lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"0526 Failed setup of NVVME LS WQ/CQ: "
"rc = 0x%x\n", (uint32_t)rc);
goto out_destroy;
@@ -9726,7 +9832,7 @@
if ((!phba->sli4_hba.nvmet_cqset) ||
(!phba->sli4_hba.nvmet_mrq_hdr) ||
(!phba->sli4_hba.nvmet_mrq_data)) {
- lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"6130 MRQ CQ Queues not "
"allocated\n");
rc = -ENOMEM;
@@ -9739,7 +9845,7 @@
phba->sli4_hba.nvmet_cqset,
LPFC_NVMET);
if (rc) {
- lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"6098 Failed setup of NVMET "
"MRQ: rc = 0x%x\n",
(uint32_t)rc);
@@ -9753,7 +9859,7 @@
phba->sli4_hba.nvmet_cqset[0],
LPFC_NVMET);
if (rc) {
- lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"6057 Failed setup of NVMET "
"Receive Queue: rc = 0x%x\n",
(uint32_t)rc);
@@ -9772,7 +9878,7 @@
}
if (!phba->sli4_hba.hdr_rq || !phba->sli4_hba.dat_rq) {
- lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"0540 Receive Queue not allocated\n");
rc = -ENOMEM;
goto out_destroy;
@@ -9781,7 +9887,7 @@
rc = lpfc_rq_create(phba, phba->sli4_hba.hdr_rq, phba->sli4_hba.dat_rq,
phba->sli4_hba.els_cq, LPFC_USOL);
if (rc) {
- lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"0541 Failed setup of Receive Queue: "
"rc = 0x%x\n", (uint32_t)rc);
goto out_destroy;
@@ -9809,7 +9915,7 @@
phba->sli4_hba.cq_lookup = kcalloc((phba->sli4_hba.cq_max + 1),
sizeof(struct lpfc_queue *), GFP_KERNEL);
if (!phba->sli4_hba.cq_lookup) {
- lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"0549 Failed setup of CQ Lookup table: "
"size 0x%x\n", phba->sli4_hba.cq_max);
rc = -ENOMEM;
@@ -10109,7 +10215,7 @@
mboxq = (LPFC_MBOXQ_t *) mempool_alloc(phba->mbox_mem_pool,
GFP_KERNEL);
if (!mboxq) {
- lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"0494 Unable to allocate memory for "
"issuing SLI_FUNCTION_RESET mailbox "
"command\n");
@@ -10129,7 +10235,7 @@
if (rc != MBX_TIMEOUT)
mempool_free(mboxq, phba->mbox_mem_pool);
if (shdr_status || shdr_add_status || rc) {
- lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"0495 SLI_FUNCTION_RESET mailbox "
"failed with status x%x add_status x%x,"
" mbx status x%x\n",
@@ -10161,7 +10267,7 @@
phba->sli4_hba.u.if_type2.ERR1regaddr);
phba->work_status[1] = readl(
phba->sli4_hba.u.if_type2.ERR2regaddr);
- lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"2890 Port not ready, port status reg "
"0x%x error 1=0x%x, error 2=0x%x\n",
reg_data.word0,
@@ -10203,7 +10309,7 @@
out:
/* Catch the not-ready port failure after a port reset. */
if (rc) {
- lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"3317 HBA not functional: IP Reset Failed "
"try: echo fw_reset > board_mode\n");
rc = -ENODEV;
@@ -10253,7 +10359,7 @@
/* There is no SLI3 failback for SLI4 devices. */
if (bf_get(lpfc_sli_intf_valid, &phba->sli4_hba.sli_intf) !=
LPFC_SLI_INTF_VALID) {
- lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"2894 SLI_INTF reg contents invalid "
"sli_intf reg 0x%x\n",
phba->sli4_hba.sli_intf.word0);
@@ -10528,7 +10634,7 @@
if (!pmb) {
rc = -ENOMEM;
- lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"0474 Unable to allocate memory for issuing "
"MBOX_CONFIG_MSI command\n");
goto mem_fail_out;
@@ -10611,6 +10717,7 @@
/**
* lpfc_sli_enable_intr - Enable device interrupt to SLI-3 device.
* @phba: pointer to lpfc hba data structure.
+ * @cfg_mode: Interrupt configuration mode (INTx, MSI or MSI-X).
*
* This routine is invoked to enable device interrupt and associate driver's
* interrupt handler(s) to interrupt vector(s) to device with SLI-3 interface
@@ -11008,7 +11115,7 @@
/* 1 to 1, the first LPFC_CPU_FIRST_IRQ cpus to a unique hdwq */
cpup->hdwq = idx;
idx++;
- lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+ lpfc_printf_log(phba, KERN_INFO, LOG_INIT,
"3333 Set Affinity: CPU %d (phys %d core %d): "
"hdwq %d eq %d flg x%x\n",
cpu, cpup->phys_id, cpup->core_id,
@@ -11086,7 +11193,7 @@
start_cpu = first_cpu;
cpup->hdwq = new_cpup->hdwq;
logit:
- lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+ lpfc_printf_log(phba, KERN_INFO, LOG_INIT,
"3335 Set Affinity: CPU %d (phys %d core %d): "
"hdwq %d eq %d flg x%x\n",
cpu, cpup->phys_id, cpup->core_id,
@@ -11128,7 +11235,7 @@
*
* @phba: pointer to lpfc hba data structure.
* @cpu: cpu going offline
- * @eqlist:
+ * @eqlist: eq list to append to
*/
static int
lpfc_cpuhp_get_eq(struct lpfc_hba *phba, unsigned int cpu,
@@ -11417,9 +11524,9 @@
char *name;
const struct cpumask *aff_mask = NULL;
unsigned int cpu = 0, cpu_cnt = 0, cpu_select = nr_cpu_ids;
+ struct lpfc_vector_map_info *cpup;
struct lpfc_hba_eq_hdl *eqhdl;
const struct cpumask *maskp;
- bool first;
unsigned int flags = PCI_IRQ_MSIX;
/* Set up MSI-X multi-message vectors */
@@ -11492,24 +11599,34 @@
} else {
maskp = pci_irq_get_affinity(phba->pcidev, index);
- first = true;
/* Loop through all CPUs associated with vector index */
for_each_cpu_and(cpu, maskp, cpu_present_mask) {
+ cpup = &phba->sli4_hba.cpu_map[cpu];
+
/* If this is the first CPU thats assigned to
* this vector, set LPFC_CPU_FIRST_IRQ.
+ *
+ * With certain platforms its possible that irq
+ * vectors are affinitized to all the cpu's.
+ * This can result in each cpu_map.eq to be set
+ * to the last vector, resulting in overwrite
+ * of all the previous cpu_map.eq. Ensure that
+ * each vector receives a place in cpu_map.
+ * Later call to lpfc_cpu_affinity_check will
+ * ensure we are nicely balanced out.
*/
+ if (cpup->eq != LPFC_VECTOR_MAP_EMPTY)
+ continue;
lpfc_assign_eq_map_info(phba, index,
- first ?
- LPFC_CPU_FIRST_IRQ : 0,
+ LPFC_CPU_FIRST_IRQ,
cpu);
- if (first)
- first = false;
+ break;
}
}
}
if (vectors != phba->cfg_irq_chann) {
- lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"3238 Reducing IO channels to match number of "
"MSI-X vectors, requested %d got %d\n",
phba->cfg_irq_chann, vectors);
@@ -11593,6 +11710,7 @@
/**
* lpfc_sli4_enable_intr - Enable device interrupt to SLI-4 device
* @phba: pointer to lpfc hba data structure.
+ * @cfg_mode: Interrupt configuration mode (INTx, MSI or MSI-X).
*
* This routine is invoked to enable device interrupt and associate driver's
* interrupt handler(s) to interrupt vector(s) to device with SLI-4
@@ -11782,17 +11900,17 @@
while (!els_xri_cmpl || !io_xri_cmpl || !nvmet_xri_cmpl) {
if (wait_time > LPFC_XRI_EXCH_BUSY_WAIT_TMO) {
if (!nvmet_xri_cmpl)
- lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"6424 NVMET XRI exchange busy "
"wait time: %d seconds.\n",
wait_time/1000);
if (!io_xri_cmpl)
- lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"6100 IO XRI exchange busy "
"wait time: %d seconds.\n",
wait_time/1000);
if (!els_xri_cmpl)
- lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"2878 ELS XRI exchange busy "
"wait time: %d seconds.\n",
wait_time/1000);
@@ -12287,14 +12405,14 @@
/* Configure and enable interrupt */
intr_mode = lpfc_sli_enable_intr(phba, cfg_mode);
if (intr_mode == LPFC_INTR_ERROR) {
- lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"0431 Failed to enable interrupt.\n");
error = -ENODEV;
goto out_free_sysfs_attr;
}
/* SLI-3 HBA setup */
if (lpfc_sli_hba_setup(phba)) {
- lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"1477 Failed to set up hba\n");
error = -ENODEV;
goto out_remove_device;
@@ -12552,7 +12670,7 @@
/* Configure and enable interrupt */
intr_mode = lpfc_sli_enable_intr(phba, phba->intr_mode);
if (intr_mode == LPFC_INTR_ERROR) {
- lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"0430 PM resume Failed to enable interrupt\n");
return -EIO;
} else
@@ -12578,7 +12696,7 @@
static void
lpfc_sli_prep_dev_for_recover(struct lpfc_hba *phba)
{
- lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"2723 PCI channel I/O abort preparing for recovery\n");
/*
@@ -12599,7 +12717,7 @@
static void
lpfc_sli_prep_dev_for_reset(struct lpfc_hba *phba)
{
- lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"2710 PCI channel disable preparing for reset\n");
/* Block any management I/Os to the device */
@@ -12630,7 +12748,7 @@
static void
lpfc_sli_prep_dev_for_perm_failure(struct lpfc_hba *phba)
{
- lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"2711 PCI channel permanent disable for failure\n");
/* Block all SCSI devices' I/Os on the host */
lpfc_scsi_dev_block(phba);
@@ -12681,7 +12799,7 @@
return PCI_ERS_RESULT_DISCONNECT;
default:
/* Unknown state, prepare and request slot reset */
- lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"0472 Unknown PCI error state: x%x\n", state);
lpfc_sli_prep_dev_for_reset(phba);
return PCI_ERS_RESULT_NEED_RESET;
@@ -12739,7 +12857,7 @@
/* Configure and enable interrupt */
intr_mode = lpfc_sli_enable_intr(phba, phba->intr_mode);
if (intr_mode == LPFC_INTR_ERROR) {
- lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"0427 Cannot re-enable interrupt after "
"slot reset.\n");
return PCI_ERS_RESULT_DISCONNECT;
@@ -12842,7 +12960,7 @@
magic_number != MAGIC_NUMBER_G6) ||
(phba->pcidev->device == PCI_DEVICE_ID_LANCER_G7_FC &&
magic_number != MAGIC_NUMBER_G7)) {
- lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"3030 This firmware version is not supported on"
" this HBA model. Device:%x Magic:%x Type:%x "
"ID:%x Size %d %zd\n",
@@ -12850,7 +12968,7 @@
fsize, fw->size);
rc = -EINVAL;
} else if (offset == ADD_STATUS_FW_DOWNLOAD_HW_DISABLED) {
- lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"3021 Firmware downloads have been prohibited "
"by a system configuration setting on "
"Device:%x Magic:%x Type:%x ID:%x Size %d "
@@ -12859,7 +12977,7 @@
fsize, fw->size);
rc = -EACCES;
} else {
- lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"3022 FW Download failed. Add Status x%x "
"Device:%x Magic:%x Type:%x ID:%x Size %d "
"%zd\n",
@@ -12874,7 +12992,6 @@
* lpfc_write_firmware - attempt to write a firmware image to the port
* @fw: pointer to firmware image returned from request_firmware.
* @context: pointer to firmware image returned from request_firmware.
- * @ret: return value this routine provides to the caller.
*
**/
static void
@@ -12904,7 +13021,7 @@
INIT_LIST_HEAD(&dma_buffer_list);
lpfc_decode_firmware_rev(phba, fwrev, 1);
if (strncmp(fwrev, image->revision, strnlen(image->revision, 16))) {
- lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"3023 Updating Firmware, Current Version:%s "
"New Version:%s\n",
fwrev, image->revision);
@@ -12954,7 +13071,7 @@
}
rc = offset;
} else
- lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"3029 Skipped Firmware update, Current "
"Version:%s New Version:%s\n",
fwrev, image->revision);
@@ -12969,16 +13086,17 @@
release_firmware(fw);
out:
if (rc < 0)
- lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"3062 Firmware update error, status %d.\n", rc);
else
- lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"3024 Firmware update success: size %d.\n", rc);
}
/**
* lpfc_sli4_request_firmware_update - Request linux generic firmware upgrade
* @phba: pointer to lpfc hba data structure.
+ * @fw_upgrade: which firmware to update.
*
* This routine is called to perform Linux generic firmware upgrade on device
* that supports such feature.
@@ -13101,7 +13219,7 @@
/* Configure and enable interrupt */
intr_mode = lpfc_sli4_enable_intr(phba, cfg_mode);
if (intr_mode == LPFC_INTR_ERROR) {
- lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"0426 Failed to enable interrupt.\n");
error = -ENODEV;
goto out_unset_driver_resource;
@@ -13136,7 +13254,7 @@
/* Set up SLI-4 HBA */
if (lpfc_sli4_hba_setup(phba)) {
- lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"1421 Failed to set up hba\n");
error = -ENODEV;
goto out_free_sysfs_attr;
@@ -13161,7 +13279,7 @@
*/
error = lpfc_nvme_create_localport(vport);
if (error) {
- lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"6004 NVME registration "
"failed, error x%x\n",
error);
@@ -13395,7 +13513,7 @@
/* Configure and enable interrupt */
intr_mode = lpfc_sli4_enable_intr(phba, phba->intr_mode);
if (intr_mode == LPFC_INTR_ERROR) {
- lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"0294 PM resume Failed to enable interrupt\n");
return -EIO;
} else
@@ -13421,7 +13539,7 @@
static void
lpfc_sli4_prep_dev_for_recover(struct lpfc_hba *phba)
{
- lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"2828 PCI channel I/O abort preparing for recovery\n");
/*
* There may be errored I/Os through HBA, abort all I/Os on txcmplq
@@ -13441,7 +13559,7 @@
static void
lpfc_sli4_prep_dev_for_reset(struct lpfc_hba *phba)
{
- lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"2826 PCI channel disable preparing for reset\n");
/* Block any management I/Os to the device */
@@ -13473,7 +13591,7 @@
static void
lpfc_sli4_prep_dev_for_perm_failure(struct lpfc_hba *phba)
{
- lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"2827 PCI channel permanent disable for failure\n");
/* Block all SCSI devices' I/Os on the host */
@@ -13523,7 +13641,7 @@
return PCI_ERS_RESULT_DISCONNECT;
default:
/* Unknown state, prepare and request slot reset */
- lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"2825 Unknown PCI error state: x%x\n", state);
lpfc_sli4_prep_dev_for_reset(phba);
return PCI_ERS_RESULT_NEED_RESET;
@@ -13581,7 +13699,7 @@
/* Configure and enable interrupt */
intr_mode = lpfc_sli4_enable_intr(phba, phba->intr_mode);
if (intr_mode == LPFC_INTR_ERROR) {
- lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"2824 Cannot re-enable interrupt after "
"slot reset.\n");
return PCI_ERS_RESULT_DISCONNECT;
@@ -13686,7 +13804,7 @@
lpfc_pci_remove_one_s4(pdev);
break;
default:
- lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"1424 Invalid PCI device group: 0x%x\n",
phba->pci_dev_grp);
break;
@@ -13723,7 +13841,7 @@
rc = lpfc_pci_suspend_one_s4(pdev, msg);
break;
default:
- lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"1425 Invalid PCI device group: 0x%x\n",
phba->pci_dev_grp);
break;
@@ -13759,7 +13877,7 @@
rc = lpfc_pci_resume_one_s4(pdev);
break;
default:
- lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"1426 Invalid PCI device group: 0x%x\n",
phba->pci_dev_grp);
break;
@@ -13797,7 +13915,7 @@
rc = lpfc_io_error_detected_s4(pdev, state);
break;
default:
- lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"1427 Invalid PCI device group: 0x%x\n",
phba->pci_dev_grp);
break;
@@ -13834,7 +13952,7 @@
rc = lpfc_io_slot_reset_s4(pdev);
break;
default:
- lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"1428 Invalid PCI device group: 0x%x\n",
phba->pci_dev_grp);
break;
@@ -13866,7 +13984,7 @@
lpfc_io_resume_s4(pdev);
break;
default:
- lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"1429 Invalid PCI device group: 0x%x\n",
phba->pci_dev_grp);
break;
@@ -13974,8 +14092,8 @@
{
int error = 0;
- printk(LPFC_MODULE_DESC "\n");
- printk(LPFC_COPYRIGHT "\n");
+ pr_info(LPFC_MODULE_DESC "\n");
+ pr_info(LPFC_COPYRIGHT "\n");
error = misc_register(&lpfc_mgmt_dev);
if (error)
@@ -14022,6 +14140,85 @@
return error;
}
+void lpfc_dmp_dbg(struct lpfc_hba *phba)
+{
+ unsigned int start_idx;
+ unsigned int dbg_cnt;
+ unsigned int temp_idx;
+ int i;
+ int j = 0;
+ unsigned long rem_nsec;
+
+ if (phba->cfg_log_verbose)
+ return;
+
+ if (atomic_cmpxchg(&phba->dbg_log_dmping, 0, 1) != 0)
+ return;
+
+ start_idx = (unsigned int)atomic_read(&phba->dbg_log_idx) % DBG_LOG_SZ;
+ dbg_cnt = (unsigned int)atomic_read(&phba->dbg_log_cnt);
+ temp_idx = start_idx;
+ if (dbg_cnt >= DBG_LOG_SZ) {
+ dbg_cnt = DBG_LOG_SZ;
+ temp_idx -= 1;
+ } else {
+ if ((start_idx + dbg_cnt) > (DBG_LOG_SZ - 1)) {
+ temp_idx = (start_idx + dbg_cnt) % DBG_LOG_SZ;
+ } else {
+ if (start_idx < dbg_cnt)
+ start_idx = DBG_LOG_SZ - (dbg_cnt - start_idx);
+ else
+ start_idx -= dbg_cnt;
+ }
+ }
+ dev_info(&phba->pcidev->dev, "start %d end %d cnt %d\n",
+ start_idx, temp_idx, dbg_cnt);
+
+ for (i = 0; i < dbg_cnt; i++) {
+ if ((start_idx + i) < DBG_LOG_SZ)
+ temp_idx = (start_idx + i) % DBG_LOG_SZ;
+ else
+ temp_idx = j++;
+ rem_nsec = do_div(phba->dbg_log[temp_idx].t_ns, NSEC_PER_SEC);
+ dev_info(&phba->pcidev->dev, "%d: [%5lu.%06lu] %s",
+ temp_idx,
+ (unsigned long)phba->dbg_log[temp_idx].t_ns,
+ rem_nsec / 1000,
+ phba->dbg_log[temp_idx].log);
+ }
+ atomic_set(&phba->dbg_log_cnt, 0);
+ atomic_set(&phba->dbg_log_dmping, 0);
+}
+
+__printf(2, 3)
+void lpfc_dbg_print(struct lpfc_hba *phba, const char *fmt, ...)
+{
+ unsigned int idx;
+ va_list args;
+ int dbg_dmping = atomic_read(&phba->dbg_log_dmping);
+ struct va_format vaf;
+
+
+ va_start(args, fmt);
+ if (unlikely(dbg_dmping)) {
+ vaf.fmt = fmt;
+ vaf.va = &args;
+ dev_info(&phba->pcidev->dev, "%pV", &vaf);
+ va_end(args);
+ return;
+ }
+ idx = (unsigned int)atomic_fetch_add(1, &phba->dbg_log_idx) %
+ DBG_LOG_SZ;
+
+ atomic_inc(&phba->dbg_log_cnt);
+
+ vscnprintf(phba->dbg_log[idx].log,
+ sizeof(phba->dbg_log[idx].log), fmt, args);
+ va_end(args);
+
+ phba->dbg_log[idx].t_ns = local_clock();
+}
+
/**
* lpfc_exit - lpfc module removal routine
*
diff --git a/drivers/scsi/lpfc/lpfc_logmsg.h b/drivers/scsi/lpfc/lpfc_logmsg.h
index 148d02a..5660a87 100644
--- a/drivers/scsi/lpfc/lpfc_logmsg.h
+++ b/drivers/scsi/lpfc/lpfc_logmsg.h
@@ -44,7 +44,11 @@
#define LOG_NVME_DISC 0x00200000 /* NVME Discovery/Connect events. */
#define LOG_NVME_ABTS 0x00400000 /* NVME ABTS events. */
#define LOG_NVME_IOERR 0x00800000 /* NVME IO Error events. */
-#define LOG_ALL_MSG 0xffffffff /* LOG all messages */
+#define LOG_TRACE_EVENT 0x80000000 /* Dmp the DBG log on this err */
+#define LOG_ALL_MSG 0x7fffffff /* LOG all messages */
+
+void lpfc_dmp_dbg(struct lpfc_hba *phba);
+void lpfc_dbg_print(struct lpfc_hba *phba, const char *fmt, ...);
/* generate message by verbose log setting or severity */
#define lpfc_vlog_msg(vport, level, mask, fmt, arg...) \
@@ -65,9 +69,15 @@
#define lpfc_printf_vlog(vport, level, mask, fmt, arg...) \
do { \
- { if (((mask) & (vport)->cfg_log_verbose) || (level[1] <= '3')) \
+ { if (((mask) & (vport)->cfg_log_verbose) || (level[1] <= '3')) { \
+ if ((mask) & LOG_TRACE_EVENT) \
+ lpfc_dmp_dbg((vport)->phba); \
dev_printk(level, &((vport)->phba->pcidev)->dev, "%d:(%d):" \
- fmt, (vport)->phba->brd_no, vport->vpi, ##arg); } \
+ fmt, (vport)->phba->brd_no, vport->vpi, ##arg); \
+ } else if (!(vport)->cfg_log_verbose) \
+ lpfc_dbg_print((vport)->phba, "%d:(%d):" fmt, \
+ (vport)->phba->brd_no, (vport)->vpi, ##arg); \
+ } \
} while (0)
#define lpfc_printf_log(phba, level, mask, fmt, arg...) \
@@ -75,8 +85,12 @@
{ uint32_t log_verbose = (phba)->pport ? \
(phba)->pport->cfg_log_verbose : \
(phba)->cfg_log_verbose; \
- if (((mask) & log_verbose) || (level[1] <= '3')) \
+ if (((mask) & log_verbose) || (level[1] <= '3')) { \
+ if ((mask) & LOG_TRACE_EVENT) \
+ lpfc_dmp_dbg(phba); \
dev_printk(level, &((phba)->pcidev)->dev, "%d:" \
- fmt, phba->brd_no, ##arg); \
+ fmt, phba->brd_no, ##arg); \
+ } else if (!(phba)->cfg_log_verbose)\
+ lpfc_dbg_print(phba, "%d:" fmt, phba->brd_no, ##arg); \
} \
} while (0)
diff --git a/drivers/scsi/lpfc/lpfc_mbox.c b/drivers/scsi/lpfc/lpfc_mbox.c
index e34e0f1..3414ffc 100644
--- a/drivers/scsi/lpfc/lpfc_mbox.c
+++ b/drivers/scsi/lpfc/lpfc_mbox.c
@@ -868,9 +868,7 @@
/**
* lpfc_reg_vpi - Prepare a mailbox command for registering vport identifier
- * @phba: pointer to lpfc hba data structure.
- * @vpi: virtual N_Port identifier.
- * @sid: Fibre Channel S_ID (N_Port_ID assigned to a virtual N_Port).
+ * @vport: pointer to a vport object.
* @pmb: pointer to the driver internal queue element for mailbox command.
*
* The registration vport identifier mailbox command is used to activate a
@@ -1199,7 +1197,7 @@
/**
* lpfc_config_ring - Prepare a mailbox command for configuring an IOCB ring
* @phba: pointer to lpfc hba data structure.
- * @ring:
+ * @ring: ring number/index
* @pmb: pointer to the driver internal queue element for mailbox command.
*
* The configure ring mailbox command is used to configure an IOCB ring. This
@@ -1613,7 +1611,7 @@
/**
* lpfc_mbox_tmo_val - Retrieve mailbox command timeout value
* @phba: pointer to lpfc hba data structure.
- * @cmd: mailbox command code.
+ * @mboxq: pointer to the driver internal queue element for mailbox command.
*
* This routine retrieves the proper timeout value according to the mailbox
* command code.
@@ -1700,6 +1698,7 @@
* lpfc_sli4_mbx_sge_get - Get a sge entry from non-embedded mailbox command
* @mbox: pointer to lpfc mbox command.
* @sgentry: sge entry index.
+ * @sge: pointer to lpfc mailbox sge to load into.
*
* This routine gets an entry from the non-embedded mailbox command at the sge
* index location.
@@ -1767,6 +1766,7 @@
* @subsystem: The sli4 config sub mailbox subsystem.
* @opcode: The sli4 config sub mailbox command opcode.
* @length: Length of the sli4 config mailbox command (including sub-header).
+ * @emb: True if embedded mbox command should be setup.
*
* This routine sets up the header fields of SLI4 specific mailbox command
* for sending IOCTL command.
@@ -2012,6 +2012,7 @@
/**
* lpfc_sli4_mbx_read_fcf_rec - Allocate and construct read fcf mbox cmd
* @phba: pointer to lpfc hba data structure.
+ * @mboxq: pointer to lpfc mbox command.
* @fcf_index: index to fcf table.
*
* This routine routine allocates and constructs non-embedded mailbox command
@@ -2068,6 +2069,7 @@
/**
* lpfc_request_features: Configure SLI4 REQUEST_FEATURES mailbox
+ * @phba: pointer to lpfc hba data structure.
* @mboxq: pointer to lpfc mbox command.
*
* This routine sets up the mailbox for an SLI4 REQUEST_FEATURES
diff --git a/drivers/scsi/lpfc/lpfc_mem.c b/drivers/scsi/lpfc/lpfc_mem.c
index 726f661..27ff67e 100644
--- a/drivers/scsi/lpfc/lpfc_mem.c
+++ b/drivers/scsi/lpfc/lpfc_mem.c
@@ -45,6 +45,7 @@
#define LPFC_MBUF_POOL_SIZE 64 /* max elements in MBUF safety pool */
#define LPFC_MEM_POOL_SIZE 64 /* max elem in non-DMA safety pool */
#define LPFC_DEVICE_DATA_POOL_SIZE 64 /* max elements in device data pool */
+#define LPFC_RRQ_POOL_SIZE 256 /* max elements in non-DMA pool */
int
lpfc_mem_alloc_active_rrq_pool_s4(struct lpfc_hba *phba) {
@@ -67,6 +68,7 @@
/**
* lpfc_mem_alloc - create and allocate all PCI and memory pools
* @phba: HBA to allocate pools for
+ * @align: alignment requirement for blocks; must be a power of two
*
* Description: Creates and allocates PCI pools lpfc_mbuf_pool,
* lpfc_hrb_pool. Creates and allocates kmalloc-backed mempools
@@ -121,7 +123,7 @@
if (phba->sli_rev == LPFC_SLI_REV4) {
phba->rrq_pool =
- mempool_create_kmalloc_pool(LPFC_MEM_POOL_SIZE,
+ mempool_create_kmalloc_pool(LPFC_RRQ_POOL_SIZE,
sizeof(struct lpfc_node_rrq));
if (!phba->rrq_pool)
goto fail_free_nlp_mem_pool;
diff --git a/drivers/scsi/lpfc/lpfc_nportdisc.c b/drivers/scsi/lpfc/lpfc_nportdisc.c
index d8501bd..e4c710f 100644
--- a/drivers/scsi/lpfc/lpfc_nportdisc.c
+++ b/drivers/scsi/lpfc/lpfc_nportdisc.c
@@ -152,7 +152,7 @@
memcpy(&ndlp->nlp_portname, &sp->portName, sizeof (struct lpfc_name));
return 1;
bad_service_param:
- lpfc_printf_vlog(vport, KERN_ERR, LOG_DISCOVERY,
+ lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
"0207 Device %x "
"(%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x) sent "
"invalid service parameters. Ignoring device.\n",
@@ -301,7 +301,7 @@
/* Check for CONFIG_LINK error */
if (mb->mbxStatus) {
- lpfc_printf_log(phba, KERN_ERR, LOG_DISCOVERY,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"4575 CONFIG_LINK fails pt2pt discovery: %x\n",
mb->mbxStatus);
mempool_free(login_mbox, phba->mbox_mem_pool);
@@ -316,7 +316,7 @@
rc = lpfc_els_rsp_acc(link_mbox->vport, ELS_CMD_PLOGI,
save_iocb, ndlp, login_mbox);
if (rc) {
- lpfc_printf_log(phba, KERN_ERR, LOG_DISCOVERY,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"4576 PLOGI ACC fails pt2pt discovery: %x\n",
rc);
mempool_free(login_mbox, phba->mbox_mem_pool);
@@ -361,7 +361,7 @@
lpfc_sli4_unreg_rpi_cmpl_clr(phba, pmb);
if (!piocb) {
- lpfc_printf_vlog(vport, KERN_ERR, LOG_DISCOVERY | LOG_ELS,
+ lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
"4578 PLOGI ACC fail\n");
if (mbox)
mempool_free(mbox, phba->mbox_mem_pool);
@@ -370,7 +370,7 @@
rc = lpfc_els_rsp_acc(vport, ELS_CMD_PLOGI, piocb, ndlp, mbox);
if (rc) {
- lpfc_printf_vlog(vport, KERN_ERR, LOG_DISCOVERY | LOG_ELS,
+ lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
"4579 PLOGI ACC fail %x\n", rc);
if (mbox)
mempool_free(mbox, phba->mbox_mem_pool);
@@ -405,7 +405,7 @@
lp = (uint32_t *) pcmd->virt;
sp = (struct serv_parm *) ((uint8_t *) lp + sizeof (uint32_t));
if (wwn_to_u64(sp->portName.u.wwn) == 0) {
- lpfc_printf_vlog(vport, KERN_ERR, LOG_ELS,
+ lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
"0140 PLOGI Reject: invalid nname\n");
stat.un.b.lsRjtRsnCode = LSRJT_UNABLE_TPC;
stat.un.b.lsRjtRsnCodeExp = LSEXP_INVALID_PNAME;
@@ -414,7 +414,7 @@
return 0;
}
if (wwn_to_u64(sp->nodeName.u.wwn) == 0) {
- lpfc_printf_vlog(vport, KERN_ERR, LOG_ELS,
+ lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
"0141 PLOGI Reject: invalid pname\n");
stat.un.b.lsRjtRsnCode = LSRJT_UNABLE_TPC;
stat.un.b.lsRjtRsnCodeExp = LSEXP_INVALID_NNAME;
@@ -481,7 +481,7 @@
}
if (nlp_portwwn != 0 &&
nlp_portwwn != wwn_to_u64(sp->portName.u.wwn))
- lpfc_printf_vlog(vport, KERN_ERR, LOG_ELS,
+ lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
"0143 PLOGI recv'd from DID: x%x "
"WWPN changed: old %llx new %llx\n",
ndlp->nlp_DID,
@@ -689,7 +689,7 @@
return 1;
out:
if (defer_acc)
- lpfc_printf_log(phba, KERN_ERR, LOG_DISCOVERY,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"4577 discovery failure: %p %p %p\n",
save_iocb, link_mbox, login_mbox);
kfree(save_iocb);
@@ -797,11 +797,17 @@
ndlp, NULL);
}
out:
- /* If we are authenticated, move to the proper state */
- if (ndlp->nlp_type & (NLP_FCP_TARGET | NLP_NVME_TARGET))
- lpfc_nlp_set_state(vport, ndlp, NLP_STE_MAPPED_NODE);
- else
- lpfc_nlp_set_state(vport, ndlp, NLP_STE_UNMAPPED_NODE);
+ /* If we are authenticated, move to the proper state.
+ * It is possible an ADISC arrived and the remote nport
+ * is already in MAPPED or UNMAPPED state. Catch this
+ * condition and don't set the nlp_state again because
+ * it causes an unnecessary transport unregister/register.
+ */
+ if (ndlp->nlp_type & (NLP_FCP_TARGET | NLP_NVME_TARGET)) {
+ if (ndlp->nlp_state != NLP_STE_MAPPED_NODE)
+ lpfc_nlp_set_state(vport, ndlp,
+ NLP_STE_MAPPED_NODE);
+ }
return 1;
}
@@ -1062,6 +1068,7 @@
* lpfc_release_rpi - Release a RPI by issuing unreg_login mailbox cmd.
* @phba : Pointer to lpfc_hba structure.
* @vport: Pointer to lpfc_vport structure.
+ * @ndlp: Pointer to lpfc_nodelist structure.
* @rpi : rpi to be release.
*
* This function will send a unreg_login mailbox command to the firmware
@@ -1091,8 +1098,8 @@
pmb = (LPFC_MBOXQ_t *) mempool_alloc(phba->mbox_mem_pool,
GFP_KERNEL);
if (!pmb)
- lpfc_printf_vlog(vport, KERN_ERR, LOG_MBOX,
- "2796 mailbox memory allocation failed \n");
+ lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
+ "2796 mailbox memory allocation failed \n");
else {
lpfc_unreg_login(phba, vport->vpi, rpi, pmb);
pmb->mbox_cmpl = lpfc_sli_def_mbox_cmpl;
@@ -1130,7 +1137,7 @@
rpi = pmb->u.mb.un.varWords[0];
lpfc_release_rpi(phba, vport, ndlp, rpi);
}
- lpfc_printf_vlog(vport, KERN_ERR, LOG_DISCOVERY,
+ lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
"0271 Illegal State Transition: node x%x "
"event x%x, state x%x Data: x%x x%x\n",
ndlp->nlp_DID, evt, ndlp->nlp_state, ndlp->nlp_rpi,
@@ -1148,11 +1155,11 @@
* to stop it.
*/
if (!(ndlp->nlp_flag & NLP_RCV_PLOGI)) {
- lpfc_printf_vlog(vport, KERN_ERR, LOG_DISCOVERY,
- "0272 Illegal State Transition: node x%x "
- "event x%x, state x%x Data: x%x x%x\n",
- ndlp->nlp_DID, evt, ndlp->nlp_state, ndlp->nlp_rpi,
- ndlp->nlp_flag);
+ lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
+ "0272 Illegal State Transition: node x%x "
+ "event x%x, state x%x Data: x%x x%x\n",
+ ndlp->nlp_DID, evt, ndlp->nlp_state,
+ ndlp->nlp_rpi, ndlp->nlp_flag);
}
return ndlp->nlp_state;
}
@@ -1372,7 +1379,7 @@
if ((ndlp->nlp_DID != FDMI_DID) &&
(wwn_to_u64(sp->portName.u.wwn) == 0 ||
wwn_to_u64(sp->nodeName.u.wwn) == 0)) {
- lpfc_printf_vlog(vport, KERN_ERR, LOG_ELS,
+ lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
"0142 PLOGI RSP: Invalid WWN.\n");
goto out;
}
@@ -1434,7 +1441,8 @@
} else {
mbox = mempool_alloc(phba->mbox_mem_pool, GFP_KERNEL);
if (!mbox) {
- lpfc_printf_vlog(vport, KERN_ERR, LOG_ELS,
+ lpfc_printf_vlog(vport, KERN_ERR,
+ LOG_TRACE_EVENT,
"0133 PLOGI: no memory "
"for config_link "
"Data: x%x x%x x%x x%x\n",
@@ -1459,7 +1467,7 @@
mbox = mempool_alloc(phba->mbox_mem_pool, GFP_KERNEL);
if (!mbox) {
- lpfc_printf_vlog(vport, KERN_ERR, LOG_ELS,
+ lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
"0018 PLOGI: no memory for reg_login "
"Data: x%x x%x x%x x%x\n",
ndlp->nlp_DID, ndlp->nlp_state,
@@ -1499,7 +1507,7 @@
kfree(mp);
mempool_free(mbox, phba->mbox_mem_pool);
- lpfc_printf_vlog(vport, KERN_ERR, LOG_ELS,
+ lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
"0134 PLOGI: cannot issue reg_login "
"Data: x%x x%x x%x x%x\n",
ndlp->nlp_DID, ndlp->nlp_state,
@@ -1507,7 +1515,7 @@
} else {
mempool_free(mbox, phba->mbox_mem_pool);
- lpfc_printf_vlog(vport, KERN_ERR, LOG_ELS,
+ lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
"0135 PLOGI: cannot format reg_login "
"Data: x%x x%x x%x x%x\n",
ndlp->nlp_DID, ndlp->nlp_state,
@@ -1518,7 +1526,7 @@
out:
if (ndlp->nlp_DID == NameServer_DID) {
lpfc_vport_set_state(vport, FC_VPORT_FAILED);
- lpfc_printf_vlog(vport, KERN_ERR, LOG_ELS,
+ lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
"0261 Cannot Register NameServer login\n");
}
@@ -1940,8 +1948,8 @@
if (mb->mbxStatus) {
/* RegLogin failed */
- lpfc_printf_vlog(vport, KERN_ERR, LOG_DISCOVERY,
- "0246 RegLogin failed Data: x%x x%x x%x x%x "
+ lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
+ "0246 RegLogin failed Data: x%x x%x x%x x%x "
"x%x\n",
did, mb->mbxStatus, vport->port_state,
mb->un.varRegLogin.vpi,
diff --git a/drivers/scsi/lpfc/lpfc_nvme.c b/drivers/scsi/lpfc/lpfc_nvme.c
index b16c087..e5be334 100644
--- a/drivers/scsi/lpfc/lpfc_nvme.c
+++ b/drivers/scsi/lpfc/lpfc_nvme.c
@@ -234,8 +234,10 @@
/**
* lpfc_nvme_create_queue -
+ * @pnvme_lport: Transport localport that LS is to be issued from
* @lpfc_pnvme: Pointer to the driver's nvme instance data
* @qidx: An cpu index used to affinitize IO queues and MSIX vectors.
+ * @qsize: Size of the queue in bytes
* @handle: An opaque driver handle used in follow-up calls.
*
* Driver registers this routine to preallocate and initialize any
@@ -292,7 +294,7 @@
/**
* lpfc_nvme_delete_queue -
- * @lpfc_pnvme: Pointer to the driver's nvme instance data
+ * @pnvme_lport: Transport localport that LS is to be issued from
* @qidx: An cpu index used to affinitize IO queues and MSIX vectors.
* @handle: An opaque driver handle from lpfc_nvme_create_queue
*
@@ -498,7 +500,7 @@
if (pnvme_lsreq->done)
pnvme_lsreq->done(pnvme_lsreq, status);
else
- lpfc_printf_vlog(vport, KERN_ERR, LOG_NVME_DISC,
+ lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
"6046 NVMEx cmpl without done call back? "
"Data %px DID %x Xri: %x status %x\n",
pnvme_lsreq, ndlp ? ndlp->nlp_DID : 0,
@@ -647,7 +649,7 @@
rc = lpfc_sli4_issue_wqe(phba, &phba->sli4_hba.hdwq[0], genwqe);
if (rc) {
- lpfc_printf_vlog(vport, KERN_ERR, LOG_NVME_DISC | LOG_ELS,
+ lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
"6045 Issue GEN REQ WQE to NPORT x%x "
"Data: x%x x%x rc x%x\n",
ndlp->nlp_DID, genwqe->iotag,
@@ -672,6 +674,7 @@
* @vport: The local port issuing the LS
* @ndlp: The remote port to send the LS to
* @pnvme_lsreq: Pointer to LS request structure from the transport
+ * @gen_req_cmp: Completion call-back
*
* Routine validates the ndlp, builds buffers and sends a GEN_REQUEST
* WQE to perform the LS operation.
@@ -693,8 +696,7 @@
uint16_t ntype, nstate;
if (!ndlp || !NLP_CHK_NODE_ACT(ndlp)) {
- lpfc_printf_vlog(vport, KERN_ERR,
- LOG_NVME_DISC | LOG_NODE | LOG_NVME_IOERR,
+ lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
"6051 NVMEx LS REQ: Bad NDLP x%px, Failing "
"LS Req\n",
ndlp);
@@ -705,8 +707,7 @@
nstate = ndlp->nlp_state;
if ((ntype & NLP_NVME_TARGET && nstate != NLP_STE_MAPPED_NODE) ||
(ntype & NLP_NVME_INITIATOR && nstate != NLP_STE_UNMAPPED_NODE)) {
- lpfc_printf_vlog(vport, KERN_ERR,
- LOG_NVME_DISC | LOG_NODE | LOG_NVME_IOERR,
+ lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
"6088 NVMEx LS REQ: Fail DID x%06x not "
"ready for IO. Type x%x, State x%x\n",
ndlp->nlp_DID, ntype, nstate);
@@ -727,9 +728,7 @@
bmp = kmalloc(sizeof(*bmp), GFP_KERNEL);
if (!bmp) {
-
- lpfc_printf_vlog(vport, KERN_ERR,
- LOG_NVME_DISC | LOG_NVME_IOERR,
+ lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
"6044 NVMEx LS REQ: Could not alloc LS buf "
"for DID %x\n",
ndlp->nlp_DID);
@@ -738,8 +737,7 @@
bmp->virt = lpfc_mbuf_alloc(vport->phba, MEM_PRI, &(bmp->phys));
if (!bmp->virt) {
- lpfc_printf_vlog(vport, KERN_ERR,
- LOG_NVME_DISC | LOG_NVME_IOERR,
+ lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
"6042 NVMEx LS REQ: Could not alloc mbuf "
"for DID %x\n",
ndlp->nlp_DID);
@@ -774,8 +772,7 @@
pnvme_lsreq, gen_req_cmp, ndlp, 2,
LPFC_NVME_LS_TIMEOUT, 0);
if (ret != WQE_SUCCESS) {
- lpfc_printf_vlog(vport, KERN_ERR,
- LOG_NVME_DISC | LOG_NVME_IOERR,
+ lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
"6052 NVMEx REQ: EXIT. issue ls wqe failed "
"lsreq x%px Status %x DID %x\n",
pnvme_lsreq, ret, ndlp->nlp_DID);
@@ -789,9 +786,9 @@
/**
* lpfc_nvme_ls_req - Issue an NVME Link Service request
- * @lpfc_nvme_lport: Transport localport that LS is to be issued from.
- * @lpfc_nvme_rport: Transport remoteport that LS is to be sent to.
- * @pnvme_lsreq - the transport nvme_ls_req structure for the LS
+ * @pnvme_lport: Transport localport that LS is to be issued from.
+ * @nvme_rport: Transport remoteport that LS is to be sent to.
+ * @pnvme_lsreq: the transport nvme_ls_req structure for the LS
*
* Driver registers this routine to handle any link service request
* from the nvme_fc transport to a remote nvme-aware port.
@@ -853,9 +850,7 @@
bool foundit = false;
if (!ndlp) {
- lpfc_printf_log(phba, KERN_ERR,
- LOG_NVME_DISC | LOG_NODE |
- LOG_NVME_IOERR | LOG_NVME_ABTS,
+ lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
"6049 NVMEx LS REQ Abort: Bad NDLP x%px DID "
"x%06x, Failing LS Req\n",
ndlp, ndlp ? ndlp->nlp_DID : 0);
@@ -931,9 +926,9 @@
/**
* lpfc_nvme_ls_abort - Abort a prior NVME LS request
- * @lpfc_nvme_lport: Transport localport that LS is to be issued from.
- * @lpfc_nvme_rport: Transport remoteport that LS is to be sent to.
- * @pnvme_lsreq - the transport nvme_ls_req structure for the LS
+ * @pnvme_lport: Transport localport that LS is to be issued from.
+ * @pnvme_rport: Transport remoteport that LS is to be sent to.
+ * @pnvme_lsreq: the transport nvme_ls_req structure for the LS
*
* Driver registers this routine to abort a NVME LS request that is
* in progress (from the transports perspective).
@@ -1063,11 +1058,8 @@
}
-/**
+/*
* lpfc_nvme_io_cmd_wqe_cmpl - Complete an NVME-over-FCP IO
- * @lpfc_pnvme: Pointer to the driver's nvme instance data
- * @lpfc_nvme_lport: Pointer to the driver's local port data
- * @lpfc_nvme_rport: Pointer to the rport getting the @lpfc_nvme_ereq
*
* Driver registers this routine as it io request handler. This
* routine issues an fcp WQE with data from the @lpfc_nvme_fcpreq
@@ -1099,8 +1091,7 @@
/* Sanity check on return of outstanding command */
if (!lpfc_ncmd) {
- lpfc_printf_vlog(vport, KERN_ERR,
- LOG_NODE | LOG_NVME_IOERR,
+ lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
"6071 Null lpfc_ncmd pointer. No "
"release, skip completion\n");
return;
@@ -1111,7 +1102,7 @@
if (!lpfc_ncmd->nvmeCmd) {
spin_unlock(&lpfc_ncmd->buf_lock);
- lpfc_printf_vlog(vport, KERN_ERR, LOG_NODE | LOG_NVME_IOERR,
+ lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
"6066 Missing cmpl ptrs: lpfc_ncmd x%px, "
"nvmeCmd x%px\n",
lpfc_ncmd, lpfc_ncmd->nvmeCmd);
@@ -1144,7 +1135,7 @@
*/
ndlp = lpfc_ncmd->ndlp;
if (!ndlp || !NLP_CHK_NODE_ACT(ndlp)) {
- lpfc_printf_vlog(vport, KERN_ERR, LOG_NVME_IOERR,
+ lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
"6062 Ignoring NVME cmpl. No ndlp\n");
goto out_err;
}
@@ -1215,7 +1206,7 @@
/* Sanity check */
if (nCmd->rcv_rsplen == LPFC_NVME_ERSP_LEN)
break;
- lpfc_printf_vlog(vport, KERN_ERR, LOG_NVME_IOERR,
+ lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
"6081 NVME Completion Protocol Error: "
"xri %x status x%x result x%x "
"placed x%x\n",
@@ -1300,11 +1291,10 @@
/**
* lpfc_nvme_prep_io_cmd - Issue an NVME-over-FCP IO
- * @lpfc_pnvme: Pointer to the driver's nvme instance data
- * @lpfc_nvme_lport: Pointer to the driver's local port data
- * @lpfc_nvme_rport: Pointer to the rport getting the @lpfc_nvme_ereq
- * @lpfc_nvme_fcreq: IO request from nvme fc to driver.
- * @hw_queue_handle: Driver-returned handle in lpfc_nvme_create_queue
+ * @vport: pointer to a host virtual N_Port data structure
+ * @lpfcn_cmd: Pointer to lpfc scsi command
+ * @pnode: pointer to a node-list data structure
+ * @cstat: pointer to the control status structure
*
* Driver registers this routine as it io request handler. This
* routine issues an fcp WQE with data from the @lpfc_nvme_fcpreq
@@ -1409,11 +1399,8 @@
/**
* lpfc_nvme_prep_io_dma - Issue an NVME-over-FCP IO
- * @lpfc_pnvme: Pointer to the driver's nvme instance data
- * @lpfc_nvme_lport: Pointer to the driver's local port data
- * @lpfc_nvme_rport: Pointer to the rport getting the @lpfc_nvme_ereq
- * @lpfc_nvme_fcreq: IO request from nvme fc to driver.
- * @hw_queue_handle: Driver-returned handle in lpfc_nvme_create_queue
+ * @vport: pointer to a host virtual N_Port data structure
+ * @lpfcn_cmd: Pointer to lpfc scsi command
*
* Driver registers this routine as it io request handler. This
* routine issues an fcp WQE with data from the @lpfc_nvme_fcpreq
@@ -1459,7 +1446,7 @@
first_data_sgl = sgl;
lpfc_ncmd->seg_cnt = nCmd->sg_cnt;
if (lpfc_ncmd->seg_cnt > lpfc_nvme_template.max_sgl_segments) {
- lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"6058 Too many sg segments from "
"NVME Transport. Max %d, "
"nvmeIO sg_cnt %d\n",
@@ -1482,7 +1469,7 @@
j = 2;
for (i = 0; i < nseg; i++) {
if (data_sg == NULL) {
- lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"6059 dptr err %d, nseg %d\n",
i, nseg);
lpfc_ncmd->seg_cnt = 0;
@@ -1583,7 +1570,7 @@
* and sg_cnt must zero.
*/
if (nCmd->payload_length != 0) {
- lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"6063 NVME DMA Prep Err: sg_cnt %d "
"payload_length x%x\n",
nCmd->sg_cnt, nCmd->payload_length);
@@ -1946,7 +1933,7 @@
/* driver queued commands are in process of being flushed */
if (phba->hba_flag & HBA_IOQ_FLUSH) {
spin_unlock_irqrestore(&phba->hbalock, flags);
- lpfc_printf_vlog(vport, KERN_ERR, LOG_NVME_ABTS,
+ lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
"6139 Driver in reset cleanup - flushing "
"NVME Req now. hba_flag x%x\n",
phba->hba_flag);
@@ -1956,13 +1943,13 @@
lpfc_nbuf = freqpriv->nvme_buf;
if (!lpfc_nbuf) {
spin_unlock_irqrestore(&phba->hbalock, flags);
- lpfc_printf_vlog(vport, KERN_ERR, LOG_NVME_ABTS,
+ lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
"6140 NVME IO req has no matching lpfc nvme "
"io buffer. Skipping abort req.\n");
return;
} else if (!lpfc_nbuf->nvmeCmd) {
spin_unlock_irqrestore(&phba->hbalock, flags);
- lpfc_printf_vlog(vport, KERN_ERR, LOG_NVME_ABTS,
+ lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
"6141 lpfc NVME IO req has no nvme_fcreq "
"io buffer. Skipping abort req.\n");
return;
@@ -1980,7 +1967,7 @@
* has not seen it yet.
*/
if (lpfc_nbuf->nvmeCmd != pnvme_fcreq) {
- lpfc_printf_vlog(vport, KERN_ERR, LOG_NVME_ABTS,
+ lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
"6143 NVME req mismatch: "
"lpfc_nbuf x%px nvmeCmd x%px, "
"pnvme_fcreq x%px. Skipping Abort xri x%x\n",
@@ -1991,7 +1978,7 @@
/* Don't abort IOs no longer on the pending queue. */
if (!(nvmereq_wqe->iocb_flag & LPFC_IO_ON_TXCMPLQ)) {
- lpfc_printf_vlog(vport, KERN_ERR, LOG_NVME_ABTS,
+ lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
"6142 NVME IO req x%px not queued - skipping "
"abort req xri x%x\n",
pnvme_fcreq, nvmereq_wqe->sli4_xritag);
@@ -2005,7 +1992,7 @@
/* Outstanding abort is in progress */
if (nvmereq_wqe->iocb_flag & LPFC_DRIVER_ABORTED) {
- lpfc_printf_vlog(vport, KERN_ERR, LOG_NVME_ABTS,
+ lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
"6144 Outstanding NVME I/O Abort Request "
"still pending on nvme_fcreq x%px, "
"lpfc_ncmd %px xri x%x\n",
@@ -2016,7 +2003,7 @@
abts_buf = __lpfc_sli_get_iocbq(phba);
if (!abts_buf) {
- lpfc_printf_vlog(vport, KERN_ERR, LOG_NVME_ABTS,
+ lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
"6136 No available abort wqes. Skipping "
"Abts req for nvme_fcreq x%px xri x%x\n",
pnvme_fcreq, nvmereq_wqe->sli4_xritag);
@@ -2037,7 +2024,7 @@
spin_unlock(&lpfc_nbuf->buf_lock);
spin_unlock_irqrestore(&phba->hbalock, flags);
if (ret_val) {
- lpfc_printf_vlog(vport, KERN_ERR, LOG_NVME_ABTS,
+ lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
"6137 Failed abts issue_wqe with status x%x "
"for nvme_fcreq x%px.\n",
ret_val, pnvme_fcreq);
@@ -2310,7 +2297,7 @@
if (pring->txcmplq_cnt)
pending += pring->txcmplq_cnt;
}
- lpfc_printf_vlog(vport, KERN_ERR, LOG_NVME_IOERR,
+ lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
"6176 Lport x%px Localport x%px wait "
"timed out. Pending %d. Renewing.\n",
lport, vport->localport, pending);
@@ -2432,6 +2419,7 @@
struct nvme_fc_remote_port *remote_port;
struct nvme_fc_port_info rpinfo;
struct lpfc_nodelist *prev_ndlp = NULL;
+ struct fc_rport *srport = ndlp->rport;
lpfc_printf_vlog(ndlp->vport, KERN_INFO, LOG_NVME_DISC,
"6006 Register NVME PORT. DID x%06x nlptype x%x\n",
@@ -2461,6 +2449,10 @@
rpinfo.port_name = wwn_to_u64(ndlp->nlp_portname.u.wwn);
rpinfo.node_name = wwn_to_u64(ndlp->nlp_nodename.u.wwn);
+ if (srport)
+ rpinfo.dev_loss_tmo = srport->dev_loss_tmo;
+ else
+ rpinfo.dev_loss_tmo = vport->cfg_devloss_tmo;
spin_lock_irq(&vport->phba->hbalock);
oldrport = lpfc_ndlp_get_nrport(ndlp);
@@ -2528,7 +2520,7 @@
ndlp, prev_ndlp);
} else {
lpfc_printf_vlog(vport, KERN_ERR,
- LOG_NVME_DISC | LOG_NODE,
+ LOG_TRACE_EVENT,
"6031 RemotePort Registration failed "
"err: %d, DID x%06x\n",
ret, ndlp->nlp_DID);
@@ -2574,7 +2566,7 @@
ndlp->nlp_state == NLP_STE_MAPPED_NODE) {
nvme_fc_rescan_remoteport(remoteport);
- lpfc_printf_vlog(vport, KERN_ERR, LOG_NVME_DISC,
+ lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
"6172 NVME rescanned DID x%06x "
"port_state x%x\n",
ndlp->nlp_DID, remoteport->port_state);
@@ -2657,7 +2649,7 @@
ret = nvme_fc_unregister_remoteport(remoteport);
if (ret != 0) {
lpfc_nlp_put(ndlp);
- lpfc_printf_vlog(vport, KERN_ERR, LOG_NVME_DISC,
+ lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
"6167 NVME unregister failed %d "
"port_state x%x\n",
ret, remoteport->port_state);
@@ -2667,7 +2659,7 @@
input_err:
#endif
- lpfc_printf_vlog(vport, KERN_ERR, LOG_NVME_DISC,
+ lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
"6168 State error: lport x%px, rport x%px FCID x%06x\n",
vport->localport, ndlp->rport, ndlp->nlp_DID);
}
@@ -2752,7 +2744,7 @@
* dump a message. Something is wrong.
*/
if ((wait_cnt % 1000) == 0) {
- lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"6178 NVME IO not empty, "
"cnt %d\n", wait_cnt);
}
diff --git a/drivers/scsi/lpfc/lpfc_nvmet.c b/drivers/scsi/lpfc/lpfc_nvmet.c
index 8876041..a4430ae 100644
--- a/drivers/scsi/lpfc/lpfc_nvmet.c
+++ b/drivers/scsi/lpfc/lpfc_nvmet.c
@@ -303,7 +303,7 @@
result = wcqe->parameter;
if (axchg->state != LPFC_NVME_STE_LS_RSP || axchg->entry_cnt != 2) {
- lpfc_printf_log(phba, KERN_ERR, LOG_NVME_DISC | LOG_NVME_IOERR,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"6410 NVMEx LS cmpl state mismatch IO x%x: "
"%d %d\n",
axchg->oxid, axchg->state, axchg->entry_cnt);
@@ -395,7 +395,7 @@
unsigned long iflag;
if (ctxp->state == LPFC_NVME_STE_FREE) {
- lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"6411 NVMET free, already free IO x%x: %d %d\n",
ctxp->oxid, ctxp->state, ctxp->entry_cnt);
}
@@ -474,7 +474,7 @@
if (!queue_work(phba->wq, &ctx_buf->defer_work)) {
atomic_inc(&tgtp->rcv_fcp_cmd_drop);
- lpfc_printf_log(phba, KERN_ERR, LOG_NVME,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"6181 Unable to queue deferred work "
"for oxid x%x. "
"FCP Drop IO [x%x x%x x%x]\n",
@@ -879,7 +879,7 @@
"6023 NVMEx LS rsp oxid x%x\n", axchg->oxid);
if (axchg->state != LPFC_NVME_STE_LS_RCV || axchg->entry_cnt != 1) {
- lpfc_printf_log(phba, KERN_ERR, LOG_NVME_DISC | LOG_NVME_IOERR,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"6412 NVMEx LS rsp state mismatch "
"oxid x%x: %d %d\n",
axchg->oxid, axchg->state, axchg->entry_cnt);
@@ -891,8 +891,7 @@
nvmewqeq = lpfc_nvmet_prep_ls_wqe(phba, axchg, ls_rsp->rspdma,
ls_rsp->rsplen);
if (nvmewqeq == NULL) {
- lpfc_printf_log(phba, KERN_ERR,
- LOG_NVME_DISC | LOG_NVME_IOERR | LOG_NVME_ABTS,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"6150 NVMEx LS Drop Rsp x%x: Prep\n",
axchg->oxid);
rc = -ENOMEM;
@@ -936,8 +935,7 @@
return 0;
}
- lpfc_printf_log(phba, KERN_ERR,
- LOG_NVME_DISC | LOG_NVME_IOERR | LOG_NVME_ABTS,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"6151 NVMEx LS RSP x%x: failed to transmit %d\n",
axchg->oxid, rc);
@@ -1058,7 +1056,7 @@
if ((ctxp->flag & LPFC_NVME_ABTS_RCV) ||
(ctxp->state == LPFC_NVME_STE_ABORT)) {
atomic_inc(&lpfc_nvmep->xmt_fcp_drop);
- lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"6102 IO oxid x%x aborted\n",
ctxp->oxid);
rc = -ENXIO;
@@ -1068,7 +1066,7 @@
nvmewqeq = lpfc_nvmet_prep_fcp_wqe(phba, ctxp);
if (nvmewqeq == NULL) {
atomic_inc(&lpfc_nvmep->xmt_fcp_drop);
- lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"6152 FCP Drop IO x%x: Prep\n",
ctxp->oxid);
rc = -ENXIO;
@@ -1116,7 +1114,7 @@
/* Give back resources */
atomic_inc(&lpfc_nvmep->xmt_fcp_drop);
- lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"6153 FCP Drop IO x%x: Issue: %d\n",
ctxp->oxid, rc);
@@ -1216,7 +1214,7 @@
ctxp->flag, ctxp->oxid);
else if (ctxp->state != LPFC_NVME_STE_DONE &&
ctxp->state != LPFC_NVME_STE_ABORT)
- lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"6413 NVMET release bad state %d %d oxid x%x\n",
ctxp->state, ctxp->entry_cnt, ctxp->oxid);
@@ -1395,7 +1393,7 @@
phba = tgtp->phba;
rc = lpfc_issue_els_rscn(phba->pport, 0);
- lpfc_printf_log(phba, KERN_ERR, LOG_NVME,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"6420 NVMET subsystem change: Notification %s\n",
(rc) ? "Failed" : "Sent");
}
@@ -1493,7 +1491,7 @@
phba->sli4_hba.num_possible_cpu * phba->cfg_nvmet_mrq,
sizeof(struct lpfc_nvmet_ctx_info), GFP_KERNEL);
if (!phba->sli4_hba.nvmet_ctx_info) {
- lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"6419 Failed allocate memory for "
"nvmet context lists\n");
return -ENOMEM;
@@ -1551,7 +1549,7 @@
for (i = 0; i < phba->sli4_hba.nvmet_xri_cnt; i++) {
ctx_buf = kzalloc(sizeof(*ctx_buf), GFP_KERNEL);
if (!ctx_buf) {
- lpfc_printf_log(phba, KERN_ERR, LOG_NVME,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"6404 Ran out of memory for NVMET\n");
return -ENOMEM;
}
@@ -1560,7 +1558,7 @@
GFP_KERNEL);
if (!ctx_buf->context) {
kfree(ctx_buf);
- lpfc_printf_log(phba, KERN_ERR, LOG_NVME,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"6405 Ran out of NVMET "
"context memory\n");
return -ENOMEM;
@@ -1572,7 +1570,7 @@
if (!ctx_buf->iocbq) {
kfree(ctx_buf->context);
kfree(ctx_buf);
- lpfc_printf_log(phba, KERN_ERR, LOG_NVME,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"6406 Ran out of NVMET iocb/WQEs\n");
return -ENOMEM;
}
@@ -1591,7 +1589,7 @@
lpfc_sli_release_iocbq(phba, ctx_buf->iocbq);
kfree(ctx_buf->context);
kfree(ctx_buf);
- lpfc_printf_log(phba, KERN_ERR, LOG_NVME,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"6407 Ran out of NVMET XRIs\n");
return -ENOMEM;
}
@@ -1670,7 +1668,7 @@
error = -ENOENT;
#endif
if (error) {
- lpfc_printf_log(phba, KERN_ERR, LOG_NVME_DISC,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"6025 Cannot register NVME targetport x%x: "
"portnm %llx nodenm %llx segs %d qs %d\n",
error,
@@ -2114,7 +2112,7 @@
nvmet_fc_unregister_targetport(phba->targetport);
if (!wait_for_completion_timeout(tgtp->tport_unreg_cmp,
msecs_to_jiffies(LPFC_NVMET_WAIT_TMO)))
- lpfc_printf_log(phba, KERN_ERR, LOG_NVME,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"6179 Unreg targetport x%px timeout "
"reached.\n", phba->targetport);
lpfc_nvmet_cleanup_io_context(phba);
@@ -2187,7 +2185,7 @@
unsigned long iflags;
if (!nvmebuf) {
- lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"6159 process_rcv_fcp_req, nvmebuf is NULL, "
"oxid: x%x flg: x%x state: x%x\n",
ctxp->oxid, ctxp->flag, ctxp->state);
@@ -2200,7 +2198,7 @@
}
if (ctxp->flag & LPFC_NVME_ABTS_RCV) {
- lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"6324 IO oxid x%x aborted\n",
ctxp->oxid);
return;
@@ -2264,7 +2262,7 @@
}
ctxp->flag &= ~LPFC_NVME_TNOTIFY;
atomic_inc(&tgtp->rcv_fcp_cmd_drop);
- lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"2582 FCP Drop IO x%x: err x%x: x%x x%x x%x\n",
ctxp->oxid, rc,
atomic_read(&tgtp->rcv_fcp_cmd_in),
@@ -2383,7 +2381,7 @@
ctx_buf = NULL;
if (!nvmebuf || !phba->targetport) {
- lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"6157 NVMET FCP Drop IO\n");
if (nvmebuf)
lpfc_rq_buf_free(phba, &nvmebuf->hbuf);
@@ -2456,7 +2454,7 @@
list_add_tail(&ctxp->list, &phba->sli4_hba.t_active_ctx_list);
spin_unlock_irqrestore(&phba->sli4_hba.t_active_list_lock, iflag);
if (ctxp->state != LPFC_NVME_STE_FREE) {
- lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"6414 NVMET Context corrupt %d %d oxid x%x\n",
ctxp->state, ctxp->entry_cnt, ctxp->oxid);
}
@@ -2498,7 +2496,7 @@
if (!queue_work(phba->wq, &ctx_buf->defer_work)) {
atomic_inc(&tgtp->rcv_fcp_cmd_drop);
- lpfc_printf_log(phba, KERN_ERR, LOG_NVME,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"6325 Unable to queue work for oxid x%x. "
"FCP Drop IO [x%x x%x x%x]\n",
ctxp->oxid,
@@ -2535,7 +2533,7 @@
uint8_t cqflag)
{
if (!nvmebuf) {
- lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"3167 NVMET FCP Drop IO\n");
return;
}
@@ -2581,7 +2579,7 @@
union lpfc_wqe128 *wqe;
if (!lpfc_is_link_up(phba)) {
- lpfc_printf_log(phba, KERN_ERR, LOG_NVME_DISC,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"6104 NVMET prep LS wqe: link err: "
"NPORT x%x oxid:x%x ste %d\n",
ctxp->sid, ctxp->oxid, ctxp->state);
@@ -2591,7 +2589,7 @@
/* Allocate buffer for command wqe */
nvmewqe = lpfc_sli_get_iocbq(phba);
if (nvmewqe == NULL) {
- lpfc_printf_log(phba, KERN_ERR, LOG_NVME_DISC,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"6105 NVMET prep LS wqe: No WQE: "
"NPORT x%x oxid x%x ste %d\n",
ctxp->sid, ctxp->oxid, ctxp->state);
@@ -2602,7 +2600,7 @@
if (!ndlp || !NLP_CHK_NODE_ACT(ndlp) ||
((ndlp->nlp_state != NLP_STE_UNMAPPED_NODE) &&
(ndlp->nlp_state != NLP_STE_MAPPED_NODE))) {
- lpfc_printf_log(phba, KERN_ERR, LOG_NVME_DISC,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"6106 NVMET prep LS wqe: No ndlp: "
"NPORT x%x oxid x%x ste %d\n",
ctxp->sid, ctxp->oxid, ctxp->state);
@@ -2711,7 +2709,7 @@
int xc = 1;
if (!lpfc_is_link_up(phba)) {
- lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"6107 NVMET prep FCP wqe: link err:"
"NPORT x%x oxid x%x ste %d\n",
ctxp->sid, ctxp->oxid, ctxp->state);
@@ -2722,7 +2720,7 @@
if (!ndlp || !NLP_CHK_NODE_ACT(ndlp) ||
((ndlp->nlp_state != NLP_STE_UNMAPPED_NODE) &&
(ndlp->nlp_state != NLP_STE_MAPPED_NODE))) {
- lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"6108 NVMET prep FCP wqe: no ndlp: "
"NPORT x%x oxid x%x ste %d\n",
ctxp->sid, ctxp->oxid, ctxp->state);
@@ -2730,7 +2728,7 @@
}
if (rsp->sg_cnt > lpfc_tgttemplate.max_sgl_segments) {
- lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"6109 NVMET prep FCP wqe: seg cnt err: "
"NPORT x%x oxid x%x ste %d cnt %d\n",
ctxp->sid, ctxp->oxid, ctxp->state,
@@ -2745,7 +2743,7 @@
/* Allocate buffer for command wqe */
nvmewqe = ctxp->ctxbuf->iocbq;
if (nvmewqe == NULL) {
- lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"6110 NVMET prep FCP wqe: No "
"WQE: NPORT x%x oxid x%x ste %d\n",
ctxp->sid, ctxp->oxid, ctxp->state);
@@ -2763,7 +2761,7 @@
(ctxp->state == LPFC_NVME_STE_DATA)) {
wqe = &nvmewqe->wqe;
} else {
- lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"6111 Wrong state NVMET FCP: %d cnt %d\n",
ctxp->state, ctxp->entry_cnt);
return NULL;
@@ -3136,7 +3134,7 @@
/* Sanity check */
if (ctxp->state != LPFC_NVME_STE_ABORT) {
- lpfc_printf_log(phba, KERN_ERR, LOG_NVME_ABTS,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"6112 ABTS Wrong state:%d oxid x%x\n",
ctxp->state, ctxp->oxid);
}
@@ -3210,7 +3208,7 @@
result, wcqe->word3);
if (!ctxp) {
- lpfc_printf_log(phba, KERN_ERR, LOG_NVME_ABTS,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"6415 NVMET LS Abort No ctx: WCQE: "
"%08x %08x %08x %08x\n",
wcqe->word0, wcqe->total_data_placed,
@@ -3221,7 +3219,7 @@
}
if (ctxp->state != LPFC_NVME_STE_LS_ABORT) {
- lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"6416 NVMET LS abort cmpl state mismatch: "
"oxid x%x: %d %d\n",
ctxp->oxid, ctxp->state, ctxp->entry_cnt);
@@ -3256,7 +3254,7 @@
(ndlp->nlp_state != NLP_STE_MAPPED_NODE))) {
if (tgtp)
atomic_inc(&tgtp->xmt_abort_rsp_error);
- lpfc_printf_log(phba, KERN_ERR, LOG_NVME_ABTS,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"6134 Drop ABTS - wrong NDLP state x%x.\n",
(ndlp) ? ndlp->nlp_state : NLP_STE_MAX_STATE);
@@ -3353,7 +3351,7 @@
((ndlp->nlp_state != NLP_STE_UNMAPPED_NODE) &&
(ndlp->nlp_state != NLP_STE_MAPPED_NODE))) {
atomic_inc(&tgtp->xmt_abort_rsp_error);
- lpfc_printf_log(phba, KERN_ERR, LOG_NVME_ABTS,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"6160 Drop ABORT - wrong NDLP state x%x.\n",
(ndlp) ? ndlp->nlp_state : NLP_STE_MAX_STATE);
@@ -3369,7 +3367,7 @@
spin_lock_irqsave(&ctxp->ctxlock, flags);
if (!ctxp->abort_wqeq) {
atomic_inc(&tgtp->xmt_abort_rsp_error);
- lpfc_printf_log(phba, KERN_ERR, LOG_NVME_ABTS,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"6161 ABORT failed: No wqeqs: "
"xri: x%x\n", ctxp->oxid);
/* No failure to an ABTS request. */
@@ -3396,7 +3394,7 @@
if (phba->hba_flag & HBA_IOQ_FLUSH) {
spin_unlock_irqrestore(&phba->hbalock, flags);
atomic_inc(&tgtp->xmt_abort_rsp_error);
- lpfc_printf_log(phba, KERN_ERR, LOG_NVME,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"6163 Driver in reset cleanup - flushing "
"NVME Req now. hba_flag x%x oxid x%x\n",
phba->hba_flag, ctxp->oxid);
@@ -3411,7 +3409,7 @@
if (abts_wqeq->iocb_flag & LPFC_DRIVER_ABORTED) {
spin_unlock_irqrestore(&phba->hbalock, flags);
atomic_inc(&tgtp->xmt_abort_rsp_error);
- lpfc_printf_log(phba, KERN_ERR, LOG_NVME,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"6164 Outstanding NVME I/O Abort Request "
"still pending on oxid x%x\n",
ctxp->oxid);
@@ -3449,7 +3447,7 @@
ctxp->flag &= ~LPFC_NVME_ABORT_OP;
spin_unlock_irqrestore(&ctxp->ctxlock, flags);
lpfc_sli_release_iocbq(phba, abts_wqeq);
- lpfc_printf_log(phba, KERN_ERR, LOG_NVME_ABTS,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"6166 Failed ABORT issue_wqe with status x%x "
"for oxid x%x.\n",
rc, ctxp->oxid);
@@ -3474,7 +3472,7 @@
}
if (ctxp->state == LPFC_NVME_STE_FREE) {
- lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"6417 NVMET ABORT ctx freed %d %d oxid x%x\n",
ctxp->state, ctxp->entry_cnt, ctxp->oxid);
rc = WQE_BUSY;
@@ -3512,7 +3510,7 @@
spin_unlock_irqrestore(&ctxp->ctxlock, flags);
atomic_inc(&tgtp->xmt_abort_rsp_error);
- lpfc_printf_log(phba, KERN_ERR, LOG_NVME_ABTS,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"6135 Failed to Issue ABTS for oxid x%x. Status x%x "
"(%x)\n",
ctxp->oxid, rc, released);
@@ -3544,7 +3542,7 @@
ctxp->state = LPFC_NVME_STE_LS_ABORT;
ctxp->entry_cnt++;
} else {
- lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"6418 NVMET LS abort state mismatch "
"IO x%x: %d %d\n",
ctxp->oxid, ctxp->state, ctxp->entry_cnt);
@@ -3558,7 +3556,7 @@
/* Issue ABTS for this WQE based on iotag */
ctxp->wqeq = lpfc_sli_get_iocbq(phba);
if (!ctxp->wqeq) {
- lpfc_printf_log(phba, KERN_ERR, LOG_NVME_ABTS,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"6068 Abort failed: No wqeqs: "
"xri: x%x\n", xri);
/* No failure to an ABTS request. */
@@ -3590,7 +3588,7 @@
abts_wqeq->context2 = NULL;
abts_wqeq->context3 = NULL;
lpfc_sli_release_iocbq(phba, abts_wqeq);
- lpfc_printf_log(phba, KERN_ERR, LOG_NVME_ABTS,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"6056 Failed to Issue ABTS. Status x%x\n", rc);
return 1;
}
diff --git a/drivers/scsi/lpfc/lpfc_scsi.c b/drivers/scsi/lpfc/lpfc_scsi.c
index ad62fb3..5e802c8 100644
--- a/drivers/scsi/lpfc/lpfc_scsi.c
+++ b/drivers/scsi/lpfc/lpfc_scsi.c
@@ -867,11 +867,11 @@
lpfc_cmd->seg_cnt = nseg;
if (lpfc_cmd->seg_cnt > phba->cfg_sg_seg_cnt) {
- lpfc_printf_log(phba, KERN_ERR, LOG_BG,
- "9064 BLKGRD: %s: Too many sg segments from "
- "dma_map_sg. Config %d, seg_cnt %d\n",
- __func__, phba->cfg_sg_seg_cnt,
- lpfc_cmd->seg_cnt);
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
+ "9064 BLKGRD: %s: Too many sg segments"
+ " from dma_map_sg. Config %d, seg_cnt"
+ " %d\n", __func__, phba->cfg_sg_seg_cnt,
+ lpfc_cmd->seg_cnt);
WARN_ON_ONCE(lpfc_cmd->seg_cnt > phba->cfg_sg_seg_cnt);
lpfc_cmd->seg_cnt = 0;
scsi_dma_unmap(scsi_cmnd);
@@ -1061,7 +1061,8 @@
* inserted in middle of the IO.
*/
- lpfc_printf_log(phba, KERN_ERR, LOG_BG,
+ lpfc_printf_log(phba, KERN_ERR,
+ LOG_TRACE_EVENT,
"9076 BLKGRD: Injecting reftag error: "
"write lba x%lx + x%x oldrefTag x%x\n",
(unsigned long)lba, blockoff,
@@ -1111,7 +1112,7 @@
}
rc = BG_ERR_TGT | BG_ERR_CHECK;
- lpfc_printf_log(phba, KERN_ERR, LOG_BG,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"9078 BLKGRD: Injecting reftag error: "
"write lba x%lx\n", (unsigned long)lba);
break;
@@ -1132,7 +1133,7 @@
}
rc = BG_ERR_INIT;
- lpfc_printf_log(phba, KERN_ERR, LOG_BG,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"9077 BLKGRD: Injecting reftag error: "
"write lba x%lx\n", (unsigned long)lba);
break;
@@ -1159,7 +1160,7 @@
}
rc = BG_ERR_INIT;
- lpfc_printf_log(phba, KERN_ERR, LOG_BG,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"9079 BLKGRD: Injecting reftag error: "
"read lba x%lx\n", (unsigned long)lba);
break;
@@ -1181,7 +1182,8 @@
* inserted in middle of the IO.
*/
- lpfc_printf_log(phba, KERN_ERR, LOG_BG,
+ lpfc_printf_log(phba, KERN_ERR,
+ LOG_TRACE_EVENT,
"9080 BLKGRD: Injecting apptag error: "
"write lba x%lx + x%x oldappTag x%x\n",
(unsigned long)lba, blockoff,
@@ -1230,7 +1232,7 @@
}
rc = BG_ERR_TGT | BG_ERR_CHECK;
- lpfc_printf_log(phba, KERN_ERR, LOG_BG,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"0813 BLKGRD: Injecting apptag error: "
"write lba x%lx\n", (unsigned long)lba);
break;
@@ -1251,7 +1253,7 @@
}
rc = BG_ERR_INIT;
- lpfc_printf_log(phba, KERN_ERR, LOG_BG,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"0812 BLKGRD: Injecting apptag error: "
"write lba x%lx\n", (unsigned long)lba);
break;
@@ -1278,7 +1280,7 @@
}
rc = BG_ERR_INIT;
- lpfc_printf_log(phba, KERN_ERR, LOG_BG,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"0814 BLKGRD: Injecting apptag error: "
"read lba x%lx\n", (unsigned long)lba);
break;
@@ -1313,7 +1315,7 @@
rc |= BG_ERR_TGT | BG_ERR_SWAP;
/* Signals the caller to swap CRC->CSUM */
- lpfc_printf_log(phba, KERN_ERR, LOG_BG,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"0817 BLKGRD: Injecting guard error: "
"write lba x%lx\n", (unsigned long)lba);
break;
@@ -1335,7 +1337,7 @@
rc = BG_ERR_INIT | BG_ERR_SWAP;
/* Signals the caller to swap CRC->CSUM */
- lpfc_printf_log(phba, KERN_ERR, LOG_BG,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"0816 BLKGRD: Injecting guard error: "
"write lba x%lx\n", (unsigned long)lba);
break;
@@ -1363,7 +1365,7 @@
rc = BG_ERR_INIT | BG_ERR_SWAP;
/* Signals the caller to swap CRC->CSUM */
- lpfc_printf_log(phba, KERN_ERR, LOG_BG,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"0818 BLKGRD: Injecting guard error: "
"read lba x%lx\n", (unsigned long)lba);
}
@@ -1413,7 +1415,7 @@
case SCSI_PROT_NORMAL:
default:
- lpfc_printf_log(phba, KERN_ERR, LOG_BG,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"9063 BLKGRD: Bad op/guard:%d/IP combination\n",
scsi_get_prot_op(sc));
ret = 1;
@@ -1442,7 +1444,7 @@
case SCSI_PROT_NORMAL:
default:
- lpfc_printf_log(phba, KERN_ERR, LOG_BG,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"9075 BLKGRD: Bad op/guard:%d/CRC combination\n",
scsi_get_prot_op(sc));
ret = 1;
@@ -1728,7 +1730,7 @@
sgde = scsi_sglist(sc);
if (!sgpe || !sgde) {
- lpfc_printf_log(phba, KERN_ERR, LOG_FCP,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"9020 Invalid s/g entry: data=x%px prot=x%px\n",
sgpe, sgde);
return 0;
@@ -1840,7 +1842,7 @@
return num_bde + 1;
if (!sgde) {
- lpfc_printf_log(phba, KERN_ERR, LOG_BG,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"9065 BLKGRD:%s Invalid data segment\n",
__func__);
return 0;
@@ -1903,8 +1905,8 @@
reftag += protgrp_blks;
} else {
/* if we're here, we have a bug */
- lpfc_printf_log(phba, KERN_ERR, LOG_BG,
- "9054 BLKGRD: bug in %s\n", __func__);
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
+ "9054 BLKGRD: bug in %s\n", __func__);
}
} while (!alldone);
@@ -2154,7 +2156,7 @@
sgde = scsi_sglist(sc);
if (!sgpe || !sgde) {
- lpfc_printf_log(phba, KERN_ERR, LOG_FCP,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"9082 Invalid s/g entry: data=x%px prot=x%px\n",
sgpe, sgde);
return 0;
@@ -2307,7 +2309,7 @@
return num_sge + 1;
if (!sgde) {
- lpfc_printf_log(phba, KERN_ERR, LOG_BG,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"9086 BLKGRD:%s Invalid data segment\n",
__func__);
return 0;
@@ -2412,8 +2414,8 @@
reftag += protgrp_blks;
} else {
/* if we're here, we have a bug */
- lpfc_printf_log(phba, KERN_ERR, LOG_BG,
- "9085 BLKGRD: bug in %s\n", __func__);
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
+ "9085 BLKGRD: bug in %s\n", __func__);
}
} while (!alldone);
@@ -2453,7 +2455,7 @@
break;
default:
if (phba)
- lpfc_printf_log(phba, KERN_ERR, LOG_FCP,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"9021 Unsupported protection op:%d\n",
op);
break;
@@ -2617,7 +2619,7 @@
scsi_dma_unmap(scsi_cmnd);
lpfc_cmd->seg_cnt = 0;
- lpfc_printf_log(phba, KERN_ERR, LOG_FCP,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"9022 Unexpected protection group %i\n",
prot_group_type);
return 2;
@@ -2661,7 +2663,7 @@
scsi_prot_sg_count(scsi_cmnd),
scsi_cmnd->sc_data_direction);
- lpfc_printf_log(phba, KERN_ERR, LOG_FCP,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"9023 Cannot setup S/G List for HBA"
"IO segs %d/%d BPL %d SCSI %d: %d %d\n",
lpfc_cmd->seg_cnt, lpfc_cmd->prot_seg_cnt,
@@ -3085,11 +3087,12 @@
lpfc_cmd->seg_cnt = nseg;
if (!phba->cfg_xpsgl &&
lpfc_cmd->seg_cnt > phba->cfg_sg_seg_cnt) {
- lpfc_printf_log(phba, KERN_ERR, LOG_BG, "9074 BLKGRD:"
- " %s: Too many sg segments from "
- "dma_map_sg. Config %d, seg_cnt %d\n",
- __func__, phba->cfg_sg_seg_cnt,
- lpfc_cmd->seg_cnt);
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
+ "9074 BLKGRD:"
+ " %s: Too many sg segments from "
+ "dma_map_sg. Config %d, seg_cnt %d\n",
+ __func__, phba->cfg_sg_seg_cnt,
+ lpfc_cmd->seg_cnt);
WARN_ON_ONCE(lpfc_cmd->seg_cnt > phba->cfg_sg_seg_cnt);
lpfc_cmd->seg_cnt = 0;
scsi_dma_unmap(scsi_cmnd);
@@ -3366,7 +3369,7 @@
scsi_dma_unmap(scsi_cmnd);
lpfc_cmd->seg_cnt = 0;
- lpfc_printf_log(phba, KERN_ERR, LOG_FCP,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"9083 Unexpected protection group %i\n",
prot_group_type);
return 2;
@@ -3422,7 +3425,7 @@
scsi_prot_sg_count(scsi_cmnd),
scsi_cmnd->sc_data_direction);
- lpfc_printf_log(phba, KERN_ERR, LOG_FCP,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"9084 Cannot setup S/G List for HBA"
"IO segs %d/%d SGL %d SCSI %d: %d %d\n",
lpfc_cmd->seg_cnt, lpfc_cmd->prot_seg_cnt,
@@ -3632,17 +3635,17 @@
if (resp_info & RSP_LEN_VALID) {
rsplen = be32_to_cpu(fcprsp->rspRspLen);
if (rsplen != 0 && rsplen != 4 && rsplen != 8) {
- lpfc_printf_vlog(vport, KERN_ERR, LOG_FCP,
- "2719 Invalid response length: "
- "tgt x%x lun x%llx cmnd x%x rsplen x%x\n",
- cmnd->device->id,
- cmnd->device->lun, cmnd->cmnd[0],
- rsplen);
+ lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
+ "2719 Invalid response length: "
+ "tgt x%x lun x%llx cmnd x%x rsplen "
+ "x%x\n", cmnd->device->id,
+ cmnd->device->lun, cmnd->cmnd[0],
+ rsplen);
host_status = DID_ERROR;
goto out;
}
if (fcprsp->rspInfo3 != RSP_NO_FAILURE) {
- lpfc_printf_vlog(vport, KERN_ERR, LOG_FCP,
+ lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
"2757 Protocol failure detected during "
"processing of FCP I/O op: "
"tgt x%x lun x%llx cmnd x%x rspInfo3 x%x\n",
@@ -3812,7 +3815,7 @@
/* Sanity check on return of outstanding command */
cmd = lpfc_cmd->pCmd;
if (!cmd || !phba) {
- lpfc_printf_vlog(vport, KERN_ERR, LOG_INIT,
+ lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
"2621 IO completion: Not an active IO\n");
spin_unlock(&lpfc_cmd->buf_lock);
return;
@@ -4277,7 +4280,7 @@
phba->lpfc_get_scsi_buf = lpfc_get_scsi_buf_s4;
break;
default:
- lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"1418 Invalid HBA PCI-device group: 0x%x\n",
dev_grp);
return -ENODEV;
@@ -4324,7 +4327,7 @@
* 0, successful
*/
int
-lpfc_check_pci_resettable(const struct lpfc_hba *phba)
+lpfc_check_pci_resettable(struct lpfc_hba *phba)
{
const struct pci_dev *pdev = phba->pcidev;
struct pci_dev *ptr = NULL;
@@ -4528,7 +4531,7 @@
if ((scsi_get_prot_op(cmnd) != SCSI_PROT_NORMAL) &&
(!(phba->sli3_options & LPFC_SLI3_BG_ENABLED))) {
- lpfc_printf_log(phba, KERN_ERR, LOG_BG,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"9058 BLKGRD: ERROR: rcvd protected cmd:%02x"
" op:%02x str=%s without registering for"
" BlockGuard - Rejecting command\n",
@@ -4887,7 +4890,7 @@
if (lpfc_cmd->pCmd == cmnd) {
ret = FAILED;
- lpfc_printf_vlog(vport, KERN_ERR, LOG_FCP,
+ lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
"0748 abort handler timed out waiting "
"for aborting I/O (xri:x%x) to complete: "
"ret %#x, ID %d, LUN %llu\n",
@@ -5080,7 +5083,7 @@
(iocbqrsp->iocb.ulpStatus != IOSTAT_SUCCESS)) {
if (status != IOCB_SUCCESS ||
iocbqrsp->iocb.ulpStatus != IOSTAT_FCP_RSP_ERROR)
- lpfc_printf_vlog(vport, KERN_ERR, LOG_FCP,
+ lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
"0727 TMF %s to TGT %d LUN %llu "
"failed (%d, %d) iocb_flag x%x\n",
lpfc_taskmgmt_name(task_mgmt_cmd),
@@ -5195,7 +5198,7 @@
cnt = lpfc_sli_sum_iocb(vport, tgt_id, lun_id, context);
}
if (cnt) {
- lpfc_printf_vlog(vport, KERN_ERR, LOG_FCP,
+ lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
"0724 I/O flush failure for context %s : cnt x%x\n",
((context == LPFC_CTX_LUN) ? "LUN" :
((context == LPFC_CTX_TGT) ? "TGT" :
@@ -5231,7 +5234,7 @@
rdata = lpfc_rport_data_from_scsi_device(cmnd->device);
if (!rdata || !rdata->pnode) {
- lpfc_printf_vlog(vport, KERN_ERR, LOG_FCP,
+ lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
"0798 Device Reset rdata failure: rdata x%px\n",
rdata);
return FAILED;
@@ -5243,7 +5246,7 @@
status = lpfc_chk_tgt_mapped(vport, cmnd);
if (status == FAILED) {
- lpfc_printf_vlog(vport, KERN_ERR, LOG_FCP,
+ lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
"0721 Device Reset rport failure: rdata x%px\n", rdata);
return FAILED;
}
@@ -5260,7 +5263,7 @@
status = lpfc_send_taskmgmt(vport, cmnd, tgt_id, lun_id,
FCP_LUN_RESET);
- lpfc_printf_vlog(vport, KERN_ERR, LOG_FCP,
+ lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
"0713 SCSI layer issued Device Reset (%d, %llu) "
"return x%x\n", tgt_id, lun_id, status);
@@ -5302,7 +5305,7 @@
rdata = lpfc_rport_data_from_scsi_device(cmnd->device);
if (!rdata || !rdata->pnode) {
- lpfc_printf_vlog(vport, KERN_ERR, LOG_FCP,
+ lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
"0799 Target Reset rdata failure: rdata x%px\n",
rdata);
return FAILED;
@@ -5314,7 +5317,7 @@
status = lpfc_chk_tgt_mapped(vport, cmnd);
if (status == FAILED) {
- lpfc_printf_vlog(vport, KERN_ERR, LOG_FCP,
+ lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
"0722 Target Reset rport failure: rdata x%px\n", rdata);
if (pnode) {
spin_lock_irq(shost->host_lock);
@@ -5339,7 +5342,7 @@
status = lpfc_send_taskmgmt(vport, cmnd, tgt_id, lun_id,
FCP_TARGET_RESET);
- lpfc_printf_vlog(vport, KERN_ERR, LOG_FCP,
+ lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
"0723 SCSI layer issued Target Reset (%d, %llu) "
"return x%x\n", tgt_id, lun_id, status);
@@ -5420,7 +5423,7 @@
i, 0, FCP_TARGET_RESET);
if (status != SUCCESS) {
- lpfc_printf_vlog(vport, KERN_ERR, LOG_FCP,
+ lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
"0700 Bus Reset on target %d failed\n",
i);
ret = FAILED;
@@ -5437,7 +5440,7 @@
if (status != SUCCESS)
ret = FAILED;
- lpfc_printf_vlog(vport, KERN_ERR, LOG_FCP,
+ lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
"0714 SCSI layer issued Bus Reset Data: x%x\n", ret);
return ret;
}
@@ -5466,7 +5469,7 @@
struct lpfc_hba *phba = vport->phba;
int rc, ret = SUCCESS;
- lpfc_printf_vlog(vport, KERN_ERR, LOG_FCP,
+ lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
"3172 SCSI layer issued Host Reset Data:\n");
lpfc_offline_prep(phba, LPFC_MBX_WAIT);
@@ -5483,7 +5486,7 @@
return ret;
error:
- lpfc_printf_vlog(vport, KERN_ERR, LOG_FCP,
+ lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
"3323 Failed host reset\n");
lpfc_unblock_mgmt_io(phba);
return FAILED;
@@ -5594,7 +5597,7 @@
}
num_allocated = lpfc_new_scsi_buf_s3(vport, num_to_alloc);
if (num_to_alloc != num_allocated) {
- lpfc_printf_vlog(vport, KERN_ERR, LOG_FCP,
+ lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
"0708 Allocation request of %d "
"command buffers did not succeed. "
"Allocated %d buffers.\n",
diff --git a/drivers/scsi/lpfc/lpfc_sli.c b/drivers/scsi/lpfc/lpfc_sli.c
index 25653ba..8582b51 100644
--- a/drivers/scsi/lpfc/lpfc_sli.c
+++ b/drivers/scsi/lpfc/lpfc_sli.c
@@ -35,6 +35,7 @@
#include <scsi/scsi_transport_fc.h>
#include <scsi/fc/fc_fs.h>
#include <linux/aer.h>
+#include <linux/crash_dump.h>
#ifdef CONFIG_X86
#include <asm/set_memory.h>
#endif
@@ -242,7 +243,7 @@
/**
* lpfc_sli4_mq_put - Put a Mailbox Queue Entry on an Mailbox Queue
* @q: The Mailbox Queue to operate on.
- * @wqe: The Mailbox Queue Entry to put on the Work queue.
+ * @mqe: The Mailbox Queue Entry to put on the Work queue.
*
* This routine will copy the contents of @mqe to the next available entry on
* the @q. This function will then ring the Work Queue Doorbell to signal the
@@ -656,10 +657,8 @@
writel(doorbell.word0, q->phba->sli4_hba.CQDBregaddr);
}
-/**
+/*
* lpfc_sli4_rq_put - Put a Receive Buffer Queue Entry on a Receive Queue
- * @q: The Header Receive Queue to operate on.
- * @wqe: The Receive Queue Entry to put on the Receive queue.
*
* This routine will copy the contents of @wqe to the next available entry on
* the @q. This function will then ring the Receive Queue Doorbell to signal the
@@ -722,9 +721,8 @@
return hq_put_index;
}
-/**
+/*
* lpfc_sli4_rq_release - Updates internal hba index for RQ
- * @q: The Header Receive Queue to operate on.
*
* This routine will update the HBA index of a queue to reflect consumption of
* one Receive Queue Entry by the HBA. When the HBA indicates that it has
@@ -1008,7 +1006,7 @@
* lpfc_test_rrq_active - Test RRQ bit in xri_bitmap.
* @phba: Pointer to HBA context object.
* @ndlp: Targets nodelist pointer for this exchange.
- * @xritag the xri in the bitmap to test.
+ * @xritag: the xri in the bitmap to test.
*
* This function returns:
* 0 = rrq not active for this xri
@@ -1079,7 +1077,7 @@
goto out;
spin_unlock_irqrestore(&phba->hbalock, iflags);
- rrq = mempool_alloc(phba->rrq_pool, GFP_KERNEL);
+ rrq = mempool_alloc(phba->rrq_pool, GFP_ATOMIC);
if (!rrq) {
lpfc_printf_log(phba, KERN_INFO, LOG_SLI,
"3155 Unable to allocate RRQ xri:0x%x rxid:0x%x"
@@ -1118,7 +1116,7 @@
/**
* __lpfc_sli_get_els_sglq - Allocates an iocb object from sgl pool
* @phba: Pointer to HBA context object.
- * @piocb: Pointer to the iocbq.
+ * @piocbq: Pointer to the iocbq.
*
* The driver calls this function with either the nvme ls ring lock
* or the fc els ring lock held depending on the iocb usage. This function
@@ -1194,7 +1192,7 @@
/**
* __lpfc_sli_get_nvmet_sglq - Allocates an iocb object from sgl pool
* @phba: Pointer to HBA context object.
- * @piocb: Pointer to the iocbq.
+ * @piocbq: Pointer to the iocbq.
*
* This function is called with the sgl_list lock held. This function
* gets a new driver sglq object from the sglq list. If the
@@ -1491,6 +1489,7 @@
case DSSCMD_IWRITE64_CX:
case DSSCMD_IREAD64_CR:
case DSSCMD_IREAD64_CX:
+ case CMD_SEND_FRAME:
type = LPFC_SOL_IOCB;
break;
case CMD_ABORT_XRI_CN:
@@ -1565,7 +1564,7 @@
lpfc_config_ring(phba, i, pmb);
rc = lpfc_sli_issue_mbox(phba, pmb, MBX_POLL);
if (rc != MBX_SUCCESS) {
- lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"0446 Adapter failed to init (%d), "
"mbxCmd x%x CFG_RING, mbxStatus x%x, "
"ring %d\n",
@@ -1674,7 +1673,7 @@
pring->sli.sli3.local_getidx = le32_to_cpu(pgp->cmdGetInx);
if (unlikely(pring->sli.sli3.local_getidx >= max_cmd_idx)) {
- lpfc_printf_log(phba, KERN_ERR, LOG_SLI,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"0315 Ring %d issue: portCmdGet %d "
"is bigger than cmd ring %d\n",
pring->ringno,
@@ -1960,8 +1959,7 @@
hbqp->local_hbqGetIdx = getidx;
if (unlikely(hbqp->local_hbqGetIdx >= hbqp->entry_count)) {
- lpfc_printf_log(phba, KERN_ERR,
- LOG_SLI | LOG_VPORT,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"1802 HBQ %d: local_hbqGetIdx "
"%u is > than hbqp->entry_count %u\n",
hbqno, hbqp->local_hbqGetIdx,
@@ -2229,10 +2227,8 @@
lpfc_hbq_defs[qno]->init_count);
}
-/**
+/*
* lpfc_sli_hbqbuf_get - Remove the first hbq off of an hbq list
- * @phba: Pointer to HBA context object.
- * @hbqno: HBQ number.
*
* This function removes the first hbq buffer on an hbq list and returns a
* pointer to that buffer. If it finds no buffers on the list it returns NULL.
@@ -2251,7 +2247,7 @@
/**
* lpfc_sli_rqbuf_get - Remove the first dma buffer off of an RQ list
* @phba: Pointer to HBA context object.
- * @hbqno: HBQ number.
+ * @hrq: HBQ number.
*
* This function removes the first RQ buffer on an RQ buffer list and returns a
* pointer to that buffer. If it finds no buffers on the list it returns NULL.
@@ -2300,7 +2296,7 @@
}
}
spin_unlock_irq(&phba->hbalock);
- lpfc_printf_log(phba, KERN_ERR, LOG_SLI | LOG_VPORT,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"1803 Bad hbq tag. Data: x%x x%x\n",
tag, phba->hbqs[tag >> 16].buffer_count);
return NULL;
@@ -2554,7 +2550,7 @@
/* Check security permission status on INIT_LINK mailbox command */
if ((pmb->u.mb.mbxCommand == MBX_INIT_LINK) &&
(pmb->u.mb.mbxStatus == MBXERR_SEC_NO_PERMISSION))
- lpfc_printf_log(phba, KERN_ERR, LOG_MBOX | LOG_SLI,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"2860 SLI authentication is required "
"for INIT_LINK but has not done yet\n");
@@ -2690,7 +2686,7 @@
if (lpfc_sli_chk_mbx_command(pmbox->mbxCommand) ==
MBX_SHUTDOWN) {
/* Unknown mailbox command compl */
- lpfc_printf_log(phba, KERN_ERR, LOG_MBOX | LOG_SLI,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"(%d):0323 Unknown Mailbox command "
"x%x (x%x/x%x) Cmpl\n",
pmb->vport ? pmb->vport->vpi :
@@ -2848,7 +2844,7 @@
}
if (unlikely(failwhy)) {
- lpfc_printf_log(phba, KERN_ERR, LOG_NVME_DISC | LOG_NVME_IOERR,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"6154 Drop NVME LS: SID %06X OXID x%X: %s\n",
sid, oxid, failwhy);
goto out_fail;
@@ -2888,7 +2884,7 @@
if (!ret)
return;
- lpfc_printf_log(phba, KERN_ERR, LOG_NVME_DISC | LOG_NVME_IOERR,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"6155 Drop NVME LS from DID %06X: SID %06X OXID x%X "
"NVMe%s handler failed %d\n",
did, sid, oxid,
@@ -3171,7 +3167,7 @@
}
spin_unlock_irqrestore(temp_lock, iflag);
- lpfc_printf_log(phba, KERN_ERR, LOG_SLI,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"0317 iotag x%x is out of "
"range: max iotag x%x wd0 x%x\n",
iotag, phba->sli.last_iotag,
@@ -3218,7 +3214,7 @@
}
spin_unlock_irqrestore(temp_lock, iflag);
- lpfc_printf_log(phba, KERN_ERR, LOG_SLI,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"0372 iotag x%x lookup error: max iotag (x%x) "
"iocb_flag x%x\n",
iotag, phba->sli.last_iotag,
@@ -3394,7 +3390,7 @@
* Ring <ringno> handler: portRspPut <portRspPut> is bigger than
* rsp ring <portRspMax>
*/
- lpfc_printf_log(phba, KERN_ERR, LOG_SLI,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"0312 Ring %d handler: portRspPut %d "
"is bigger than rsp ring %d\n",
pring->ringno, le32_to_cpu(pgp->rspPutInx),
@@ -3416,7 +3412,7 @@
/**
* lpfc_poll_eratt - Error attention polling timer timeout handler
- * @ptr: Pointer to address of HBA context object.
+ * @t: Context to fetch pointer to address of HBA context object from.
*
* This function is invoked by the Error Attention polling timer when the
* timer times out. It will check the SLI Error Attention register for
@@ -3613,7 +3609,7 @@
phba->brd_no, adaptermsg);
} else {
/* Unknown IOCB command */
- lpfc_printf_log(phba, KERN_ERR, LOG_SLI,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"0334 Unknown IOCB command "
"Data: x%x, x%x x%x x%x x%x\n",
type, irsp->ulpCommand,
@@ -3811,7 +3807,7 @@
phba->brd_no, adaptermsg);
} else {
/* Unknown IOCB command */
- lpfc_printf_log(phba, KERN_ERR, LOG_SLI,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"0335 Unknown IOCB "
"command Data: x%x "
"x%x x%x x%x\n",
@@ -3891,7 +3887,7 @@
* Ring <ringno> handler: portRspPut <portRspPut> is bigger than
* rsp ring <portRspMax>
*/
- lpfc_printf_log(phba, KERN_ERR, LOG_SLI,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"0303 Ring %d handler: portRspPut %d "
"is bigger than rsp ring %d\n",
pring->ringno, portRspPut, portRspMax);
@@ -4102,7 +4098,6 @@
/**
* lpfc_sli_abort_fcp_rings - Abort all iocbs in all FCP rings
* @phba: Pointer to HBA context object.
- * @pring: Pointer to driver SLI ring object.
*
* This function aborts all iocbs in FCP rings and frees all the iocb
* objects in txq. This function issues an abort iocb for all the iocb commands
@@ -4263,7 +4258,7 @@
/* Check to see if any errors occurred during init */
if ((status & HS_FFERM) || (i >= 20)) {
- lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"2751 Adapter failed to restart, "
"status reg x%x, FW Data: A8 x%x AC x%x\n",
status,
@@ -4485,7 +4480,7 @@
if (retval != MBX_SUCCESS) {
if (retval != MBX_BUSY)
mempool_free(pmb, phba->mbox_mem_pool);
- lpfc_printf_log(phba, KERN_ERR, LOG_SLI,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"2752 KILL_BOARD command failed retval %d\n",
retval);
spin_lock_irq(&phba->hbalock);
@@ -4837,7 +4832,7 @@
if (i++ >= 200) {
/* Adapter failed to init, timeout, status reg
<status> */
- lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"0436 Adapter failed to init, "
"timeout, status reg x%x, "
"FW Data: A8 x%x AC x%x\n", status,
@@ -4852,7 +4847,7 @@
/* ERROR: During chipset initialization */
/* Adapter failed to init, chipset, status reg
<status> */
- lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"0437 Adapter failed to init, "
"chipset, status reg x%x, "
"FW Data: A8 x%x AC x%x\n", status,
@@ -4883,7 +4878,7 @@
if (status & HS_FFERM) {
/* ERROR: During chipset initialization */
/* Adapter failed to init, chipset, status reg <status> */
- lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"0438 Adapter failed to init, chipset, "
"status reg x%x, "
"FW Data: A8 x%x AC x%x\n", status,
@@ -5106,7 +5101,7 @@
LPFC_SLI3_CRP_ENABLED |
LPFC_SLI3_DSS_ENABLED);
if (rc != MBX_SUCCESS) {
- lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"0442 Adapter failed to init, mbxCmd x%x "
"CONFIG_PORT, mbxStatus x%x Data: x%x\n",
pmb->u.mb.mbxCommand, pmb->u.mb.mbxStatus, 0);
@@ -5156,7 +5151,7 @@
if (pmb->u.mb.un.varCfgPort.gbg == 0) {
phba->cfg_enable_bg = 0;
phba->sli3_options &= ~LPFC_SLI3_BG_ENABLED;
- lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"0443 Adapter did not grant "
"BlockGuard\n");
}
@@ -5195,7 +5190,7 @@
switch (phba->cfg_sli_mode) {
case 2:
if (phba->cfg_enable_npiv) {
- lpfc_printf_log(phba, KERN_ERR, LOG_INIT | LOG_VPORT,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"1824 NPIV enabled: Override sli_mode "
"parameter (%d) to auto (0).\n",
phba->cfg_sli_mode);
@@ -5207,7 +5202,7 @@
case 3:
break;
default:
- lpfc_printf_log(phba, KERN_ERR, LOG_INIT | LOG_VPORT,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"1819 Unrecognized sli_mode parameter: %d.\n",
phba->cfg_sli_mode);
@@ -5218,7 +5213,7 @@
rc = lpfc_sli_config_port(phba, mode);
if (rc && phba->cfg_sli_mode == 3)
- lpfc_printf_log(phba, KERN_ERR, LOG_INIT | LOG_VPORT,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"1820 Unable to select SLI-3. "
"Not supported by adapter.\n");
if (rc && mode != 2)
@@ -5312,7 +5307,7 @@
lpfc_sli_hba_setup_error:
phba->link_state = LPFC_HBA_ERROR;
- lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"0445 Firmware initialization failed\n");
return rc;
}
@@ -5320,7 +5315,7 @@
/**
* lpfc_sli4_read_fcoe_params - Read fcoe params from conf region
* @phba: Pointer to HBA context object.
- * @mboxq: mailbox pointer.
+ *
* This function issue a dump mailbox command to read config region
* 23 and parse the records in the region and populate driver
* data structure.
@@ -5509,7 +5504,7 @@
LPFC_SLI4_MBX_NEMBED);
if (alloclen < reqlen) {
- lpfc_printf_log(phba, KERN_ERR, LOG_SLI,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"3084 Allocated DMA memory size (%d) is "
"less than the requested DMA memory size "
"(%d)\n", alloclen, reqlen);
@@ -5769,7 +5764,7 @@
rsrc_info = &mbox->u.mqe.un.rsrc_extent_info;
if (bf_get(lpfc_mbox_hdr_status,
&rsrc_info->header.cfg_shdr.response)) {
- lpfc_printf_log(phba, KERN_ERR, LOG_MBOX | LOG_INIT,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"2930 Failed to get resource extents "
"Status 0x%x Add'l Status 0x%x\n",
bf_get(lpfc_mbox_hdr_status,
@@ -5858,10 +5853,10 @@
/**
* lpfc_sli4_cfg_post_extnts -
* @phba: Pointer to HBA context object.
- * @extnt_cnt - number of available extents.
- * @type - the extent type (rpi, xri, vfi, vpi).
- * @emb - buffer to hold either MBX_EMBED or MBX_NEMBED operation.
- * @mbox - pointer to the caller's allocated mailbox structure.
+ * @extnt_cnt: number of available extents.
+ * @type: the extent type (rpi, xri, vfi, vpi).
+ * @emb: buffer to hold either MBX_EMBED or MBX_NEMBED operation.
+ * @mbox: pointer to the caller's allocated mailbox structure.
*
* This function executes the extents allocation request. It also
* takes care of the amount of memory needed to allocate or get the
@@ -5907,7 +5902,7 @@
LPFC_MBOX_OPCODE_ALLOC_RSRC_EXTENT,
req_len, *emb);
if (alloc_len < req_len) {
- lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"2982 Allocated DMA memory size (x%x) is "
"less than the requested DMA memory "
"size (x%x)\n", alloc_len, req_len);
@@ -5963,7 +5958,7 @@
return -EIO;
if ((rsrc_cnt == 0) || (rsrc_size == 0)) {
- lpfc_printf_log(phba, KERN_ERR, LOG_MBOX | LOG_INIT,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"3009 No available Resource Extents "
"for resource type 0x%x: Count: 0x%x, "
"Size 0x%x\n", type, rsrc_cnt,
@@ -6214,7 +6209,7 @@
dealloc_rsrc = &mbox->u.mqe.un.dealloc_rsrc_extents;
if (bf_get(lpfc_mbox_hdr_status,
&dealloc_rsrc->header.cfg_shdr.response)) {
- lpfc_printf_log(phba, KERN_ERR, LOG_MBOX | LOG_INIT,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"2919 Failed to release resource extents "
"for type %d - Status 0x%x Add'l Status 0x%x. "
"Resource memory not released.\n",
@@ -6408,7 +6403,7 @@
&ras_fwlog->lwpd.phys,
GFP_KERNEL);
if (!ras_fwlog->lwpd.virt) {
- lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"6185 LWPD Memory Alloc Failed\n");
return -ENOMEM;
@@ -6449,7 +6444,7 @@
/**
* lpfc_sli4_ras_mbox_cmpl: Completion handler for RAS MBX command
* @phba: pointer to lpfc hba data structure.
- * @pmboxq: pointer to the driver internal queue element for mailbox command.
+ * @pmb: pointer to the driver internal queue element for mailbox command.
*
* Completion handler for driver's RAS MBX command to the device.
**/
@@ -6469,7 +6464,7 @@
shdr_add_status = bf_get(lpfc_mbox_hdr_add_status, &shdr->response);
if (mb->mbxStatus != MBX_SUCCESS || shdr_status) {
- lpfc_printf_log(phba, KERN_ERR, LOG_MBOX,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"6188 FW LOG mailbox "
"completed with status x%x add_status x%x,"
" mbx status x%x\n",
@@ -6537,7 +6532,7 @@
/* Setup Mailbox command */
mbox = mempool_alloc(phba->mbox_mem_pool, GFP_KERNEL);
if (!mbox) {
- lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"6190 RAS MBX Alloc Failed");
rc = -ENOMEM;
goto mem_free;
@@ -6585,7 +6580,7 @@
rc = lpfc_sli_issue_mbox(phba, mbox, MBX_NOWAIT);
if (rc == MBX_NOT_FINISHED) {
- lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"6191 FW-Log Mailbox failed. "
"status %d mbxStatus : x%x", rc,
bf_get(lpfc_mqe_status, &mbox->u.mqe));
@@ -6721,7 +6716,7 @@
/* RPIs. */
count = phba->sli4_hba.max_cfg_param.max_rpi;
if (count <= 0) {
- lpfc_printf_log(phba, KERN_ERR, LOG_SLI,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"3279 Invalid provisioning of "
"rpi:%d\n", count);
rc = -EINVAL;
@@ -6749,7 +6744,7 @@
/* VPIs. */
count = phba->sli4_hba.max_cfg_param.max_vpi;
if (count <= 0) {
- lpfc_printf_log(phba, KERN_ERR, LOG_SLI,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"3280 Invalid provisioning of "
"vpi:%d\n", count);
rc = -EINVAL;
@@ -6776,7 +6771,7 @@
/* XRIs. */
count = phba->sli4_hba.max_cfg_param.max_xri;
if (count <= 0) {
- lpfc_printf_log(phba, KERN_ERR, LOG_SLI,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"3281 Invalid provisioning of "
"xri:%d\n", count);
rc = -EINVAL;
@@ -6805,7 +6800,7 @@
/* VFIs. */
count = phba->sli4_hba.max_cfg_param.max_vfi;
if (count <= 0) {
- lpfc_printf_log(phba, KERN_ERR, LOG_SLI,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"3282 Invalid provisioning of "
"vfi:%d\n", count);
rc = -EINVAL;
@@ -6899,7 +6894,7 @@
* lpfc_sli4_get_allocated_extnts - Get the port's allocated extents.
* @phba: Pointer to HBA context object.
* @type: The resource extent type.
- * @extnt_count: buffer to hold port extent count response
+ * @extnt_cnt: buffer to hold port extent count response
* @extnt_size: buffer to hold port extent size response.
*
* This function calls the port to read the host allocated extents
@@ -6983,7 +6978,7 @@
LPFC_MBOX_OPCODE_GET_ALLOC_RSRC_EXTENT,
req_len, emb);
if (alloc_len < req_len) {
- lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"2983 Allocated DMA memory size (x%x) is "
"less than the requested DMA memory "
"size (x%x)\n", alloc_len, req_len);
@@ -7026,7 +7021,7 @@
}
if (bf_get(lpfc_mbox_hdr_status, &shdr->response)) {
- lpfc_printf_log(phba, KERN_ERR, LOG_MBOX | LOG_INIT,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"2984 Failed to read allocated resources "
"for type %d - Status 0x%x Add'l Status 0x%x.\n",
type,
@@ -7043,7 +7038,6 @@
/**
* lpfc_sli4_repost_sgl_list - Repost the buffers sgl pages as block
* @phba: pointer to lpfc hba data structure.
- * @pring: Pointer to driver SLI ring object.
* @sgl_list: linked link of sgl buffers to post
* @cnt: number of linked list buffers
*
@@ -7181,7 +7175,7 @@
spin_unlock(&phba->sli4_hba.sgl_list_lock);
spin_unlock_irq(&phba->hbalock);
} else {
- lpfc_printf_log(phba, KERN_ERR, LOG_SLI,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"3161 Failure to post sgl to port.\n");
return -EIO;
}
@@ -7278,7 +7272,7 @@
drqe.address_hi = putPaddrHigh(rqb_buffer->dbuf.phys);
rc = lpfc_sli4_rq_put(hrq, drq, &hrqe, &drqe);
if (rc < 0) {
- lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"6421 Cannot post to HRQ %d: %x %x %x "
"DRQ %x %x\n",
hrq->queue_id,
@@ -7299,6 +7293,68 @@
}
/**
+ * lpfc_init_idle_stat_hb - Initialize idle_stat tracking
+ * @phba: pointer to lpfc hba data structure.
+ *
+ * This routine initializes the per-cq idle_stat to dynamically dictate
+ * polling decisions.
+ *
+ * Return codes:
+ * None
+ **/
+static void lpfc_init_idle_stat_hb(struct lpfc_hba *phba)
+{
+ int i;
+ struct lpfc_sli4_hdw_queue *hdwq;
+ struct lpfc_queue *cq;
+ struct lpfc_idle_stat *idle_stat;
+ u64 wall;
+
+ for_each_present_cpu(i) {
+ hdwq = &phba->sli4_hba.hdwq[phba->sli4_hba.cpu_map[i].hdwq];
+ cq = hdwq->io_cq;
+
+ /* Skip if we've already handled this cq's primary CPU */
+ if (cq->chann != i)
+ continue;
+
+ idle_stat = &phba->sli4_hba.idle_stat[i];
+
+ idle_stat->prev_idle = get_cpu_idle_time(i, &wall, 1);
+ idle_stat->prev_wall = wall;
+
+ if (phba->nvmet_support)
+ cq->poll_mode = LPFC_QUEUE_WORK;
+ else
+ cq->poll_mode = LPFC_IRQ_POLL;
+ }
+
+ if (!phba->nvmet_support)
+ schedule_delayed_work(&phba->idle_stat_delay_work,
+ msecs_to_jiffies(LPFC_IDLE_STAT_DELAY));
+}
+
+static void lpfc_sli4_dip(struct lpfc_hba *phba)
+{
+ uint32_t if_type;
+
+ if_type = bf_get(lpfc_sli_intf_if_type, &phba->sli4_hba.sli_intf);
+ if (if_type == LPFC_SLI_INTF_IF_TYPE_2 ||
+ if_type == LPFC_SLI_INTF_IF_TYPE_6) {
+ struct lpfc_register reg_data;
+
+ if (lpfc_readl(phba->sli4_hba.u.if_type2.STATUSregaddr,
+ ®_data.word0))
+ return;
+
+ if (bf_get(lpfc_sliport_status_dip, ®_data))
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
+ "2904 Firmware Dump Image Present"
+ " on Adapter");
+ }
+}
+
+/**
* lpfc_sli4_hba_setup - SLI4 device initialization PCI function
* @phba: Pointer to HBA context object.
*
@@ -7336,6 +7392,8 @@
spin_unlock_irq(&phba->hbalock);
}
+ lpfc_sli4_dip(phba);
+
/*
* Allocate a single mailbox container for initializing the
* port.
@@ -7376,7 +7434,7 @@
phba->hba_flag &= ~HBA_IOQ_FLUSH;
if (phba->sli_rev != LPFC_SLI_REV4) {
- lpfc_printf_log(phba, KERN_ERR, LOG_MBOX | LOG_SLI,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"0376 READ_REV Error. SLI Level %d "
"FCoE enabled %d\n",
phba->sli_rev, phba->hba_flag & HBA_FCOE_MODE);
@@ -7418,7 +7476,7 @@
*/
rc = lpfc_parse_vpd(phba, vpd, vpd_size);
if (unlikely(!rc)) {
- lpfc_printf_log(phba, KERN_ERR, LOG_MBOX | LOG_SLI,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"0377 Error %d parsing vpd. "
"Using defaults.\n", rc);
rc = 0;
@@ -7557,7 +7615,7 @@
rc = lpfc_sli_issue_mbox(phba, mboxq, MBX_POLL);
dd = bf_get(lpfc_mbx_set_feature_dd, &mboxq->u.mqe.un.set_feature);
if ((rc == MBX_SUCCESS) && (dd == LPFC_ENABLE_DUAL_DUMP))
- lpfc_printf_log(phba, KERN_ERR, LOG_SLI | LOG_INIT,
+ lpfc_printf_log(phba, KERN_ERR, LOG_SLI,
"6448 Dual Dump is enabled\n");
else
lpfc_printf_log(phba, KERN_INFO, LOG_SLI | LOG_INIT,
@@ -7575,7 +7633,7 @@
*/
rc = lpfc_sli4_alloc_resource_identifiers(phba);
if (rc) {
- lpfc_printf_log(phba, KERN_ERR, LOG_MBOX | LOG_SLI,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"2920 Failed to alloc Resource IDs "
"rc = x%x\n", rc);
goto out_free_mbox;
@@ -7614,7 +7672,7 @@
kfree(mp);
mboxq->ctx_buf = NULL;
if (unlikely(rc)) {
- lpfc_printf_log(phba, KERN_ERR, LOG_MBOX | LOG_SLI,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"0382 READ_SPARAM command failed "
"status %d, mbxStatus x%x\n",
rc, bf_get(lpfc_mqe_status, mqe));
@@ -7632,7 +7690,7 @@
/* Create all the SLI4 queues */
rc = lpfc_sli4_queue_create(phba);
if (rc) {
- lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"3089 Failed to allocate queues\n");
rc = -ENODEV;
goto out_free_mbox;
@@ -7640,7 +7698,7 @@
/* Set up all the queues to the device */
rc = lpfc_sli4_queue_setup(phba);
if (unlikely(rc)) {
- lpfc_printf_log(phba, KERN_ERR, LOG_MBOX | LOG_SLI,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"0381 Error %d during queue setup.\n ", rc);
goto out_stop_timers;
}
@@ -7651,7 +7709,7 @@
/* update host els xri-sgl sizes and mappings */
rc = lpfc_sli4_els_sgl_update(phba);
if (unlikely(rc)) {
- lpfc_printf_log(phba, KERN_ERR, LOG_MBOX | LOG_SLI,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"1400 Failed to update xri-sgl size and "
"mapping: %d\n", rc);
goto out_destroy_queue;
@@ -7661,7 +7719,7 @@
rc = lpfc_sli4_repost_sgl_list(phba, &phba->sli4_hba.lpfc_els_sgl_list,
phba->sli4_hba.els_xri_cnt);
if (unlikely(rc < 0)) {
- lpfc_printf_log(phba, KERN_ERR, LOG_MBOX | LOG_SLI,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"0582 Error %d during els sgl post "
"operation\n", rc);
rc = -ENODEV;
@@ -7673,7 +7731,7 @@
/* update host nvmet xri-sgl sizes and mappings */
rc = lpfc_sli4_nvmet_sgl_update(phba);
if (unlikely(rc)) {
- lpfc_printf_log(phba, KERN_ERR, LOG_MBOX | LOG_SLI,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"6308 Failed to update nvmet-sgl size "
"and mapping: %d\n", rc);
goto out_destroy_queue;
@@ -7685,7 +7743,7 @@
&phba->sli4_hba.lpfc_nvmet_sgl_list,
phba->sli4_hba.nvmet_xri_cnt);
if (unlikely(rc < 0)) {
- lpfc_printf_log(phba, KERN_ERR, LOG_MBOX | LOG_SLI,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"3117 Error %d during nvmet "
"sgl post\n", rc);
rc = -ENODEV;
@@ -7702,7 +7760,7 @@
/* update host common xri-sgl sizes and mappings */
rc = lpfc_sli4_io_sgl_update(phba);
if (unlikely(rc)) {
- lpfc_printf_log(phba, KERN_ERR, LOG_MBOX | LOG_SLI,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"6082 Failed to update nvme-sgl size "
"and mapping: %d\n", rc);
goto out_destroy_queue;
@@ -7711,7 +7769,7 @@
/* register the allocated common sgl pool to the port */
rc = lpfc_sli4_repost_io_sgl_list(phba);
if (unlikely(rc)) {
- lpfc_printf_log(phba, KERN_ERR, LOG_MBOX | LOG_SLI,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"6116 Error %d during nvme sgl post "
"operation\n", rc);
/* Some NVME buffers were moved to abort nvme list */
@@ -7732,7 +7790,7 @@
cnt);
rc = lpfc_init_iocb_list(phba, cnt);
if (rc) {
- lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"1413 Failed to init iocb list.\n");
goto out_destroy_queue;
}
@@ -7761,7 +7819,7 @@
/* Post the rpi header region to the device. */
rc = lpfc_sli4_post_all_rpi_hdrs(phba);
if (unlikely(rc)) {
- lpfc_printf_log(phba, KERN_ERR, LOG_MBOX | LOG_SLI,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"0393 Error %d during rpi post operation\n",
rc);
rc = -ENODEV;
@@ -7853,6 +7911,9 @@
queue_delayed_work(phba->wq, &phba->eq_delay_work,
msecs_to_jiffies(LPFC_EQ_DELAY_MSECS));
+ /* start per phba idle_stat_delay heartbeat */
+ lpfc_init_idle_stat_hb(phba);
+
/* Start error attention (ERATT) polling timer */
mod_timer(&phba->eratt_poll,
jiffies + msecs_to_jiffies(1000 * phba->eratt_poll_interval));
@@ -7902,12 +7963,12 @@
if (!(phba->hba_flag & HBA_FCOE_MODE) &&
(phba->hba_flag & LINK_DISABLED)) {
- lpfc_printf_log(phba, KERN_ERR, LOG_INIT | LOG_SLI,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"3103 Adapter Link is disabled.\n");
lpfc_down_link(phba, mboxq);
rc = lpfc_sli_issue_mbox(phba, mboxq, MBX_POLL);
if (rc != MBX_SUCCESS) {
- lpfc_printf_log(phba, KERN_ERR, LOG_INIT | LOG_SLI,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"3104 Adapter failed to issue "
"DOWN_LINK mbox cmd, rc:x%x\n", rc);
goto out_io_buff_free;
@@ -7940,7 +8001,7 @@
/**
* lpfc_mbox_timeout - Timeout call back function for mbox timer
- * @ptr: context object - pointer to hba structure.
+ * @t: Context to fetch pointer to hba structure from.
*
* This is the callback function for mailbox timer. The mailbox
* timer is armed when a new mailbox command is issued and the timer
@@ -8114,7 +8175,7 @@
}
/* Mbox cmd <mbxCommand> timeout */
- lpfc_printf_log(phba, KERN_ERR, LOG_MBOX | LOG_SLI,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"0310 Mailbox command x%x timeout Data: x%x x%x x%px\n",
mb->mbxCommand,
phba->pport->port_state,
@@ -8136,7 +8197,7 @@
lpfc_sli_abort_fcp_rings(phba);
- lpfc_printf_log(phba, KERN_ERR, LOG_MBOX | LOG_SLI,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"0345 Resetting board due to mailbox timeout\n");
/* Reset the HBA device */
@@ -8234,7 +8295,7 @@
spin_unlock_irqrestore(&phba->hbalock, drvr_flag);
/* Mbox command <mbxCommand> cannot issue */
- lpfc_printf_log(phba, KERN_ERR, LOG_MBOX | LOG_SLI,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"(%d):0311 Mailbox command x%x cannot "
"issue Data: x%x x%x\n",
pmbox->vport ? pmbox->vport->vpi : 0,
@@ -8246,7 +8307,7 @@
if (lpfc_readl(phba->HCregaddr, &hc_copy) ||
!(hc_copy & HC_MBINT_ENA)) {
spin_unlock_irqrestore(&phba->hbalock, drvr_flag);
- lpfc_printf_log(phba, KERN_ERR, LOG_MBOX | LOG_SLI,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"(%d):2528 Mailbox command x%x cannot "
"issue Data: x%x x%x\n",
pmbox->vport ? pmbox->vport->vpi : 0,
@@ -8265,7 +8326,7 @@
spin_unlock_irqrestore(&phba->hbalock, drvr_flag);
/* Mbox command <mbxCommand> cannot issue */
- lpfc_printf_log(phba, KERN_ERR, LOG_MBOX | LOG_SLI,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"(%d):2529 Mailbox command x%x "
"cannot issue Data: x%x x%x\n",
pmbox->vport ? pmbox->vport->vpi : 0,
@@ -8277,7 +8338,7 @@
if (!(psli->sli_flag & LPFC_SLI_ACTIVE)) {
spin_unlock_irqrestore(&phba->hbalock, drvr_flag);
/* Mbox command <mbxCommand> cannot issue */
- lpfc_printf_log(phba, KERN_ERR, LOG_MBOX | LOG_SLI,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"(%d):2530 Mailbox command x%x "
"cannot issue Data: x%x x%x\n",
pmbox->vport ? pmbox->vport->vpi : 0,
@@ -8330,7 +8391,7 @@
psli->sli_flag &= ~LPFC_SLI_MBOX_ACTIVE;
spin_unlock_irqrestore(&phba->hbalock, drvr_flag);
/* Mbox command <mbxCommand> cannot issue */
- lpfc_printf_log(phba, KERN_ERR, LOG_MBOX | LOG_SLI,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"(%d):2531 Mailbox command x%x "
"cannot issue Data: x%x x%x\n",
pmbox->vport ? pmbox->vport->vpi : 0,
@@ -8721,7 +8782,7 @@
spin_lock_irqsave(&phba->hbalock, iflag);
if (psli->sli_flag & LPFC_SLI_MBOX_ACTIVE) {
spin_unlock_irqrestore(&phba->hbalock, iflag);
- lpfc_printf_log(phba, KERN_ERR, LOG_MBOX | LOG_SLI,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"(%d):2532 Mailbox command x%x (x%x/x%x) "
"cannot issue Data: x%x x%x\n",
mboxq->vport ? mboxq->vport->vpi : 0,
@@ -8820,7 +8881,7 @@
/**
* lpfc_sli_issue_mbox_s4 - Issue an SLI4 mailbox command to firmware
* @phba: Pointer to HBA context object.
- * @pmbox: Pointer to mailbox object.
+ * @mboxq: Pointer to mailbox object.
* @flag: Flag indicating how the mailbox need to be processed.
*
* This function is called by discovery code and HBA management code to submit
@@ -8842,7 +8903,7 @@
rc = lpfc_mbox_dev_check(phba);
if (unlikely(rc)) {
- lpfc_printf_log(phba, KERN_ERR, LOG_MBOX | LOG_SLI,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"(%d):2544 Mailbox command x%x (x%x/x%x) "
"cannot issue Data: x%x x%x\n",
mboxq->vport ? mboxq->vport->vpi : 0,
@@ -8919,7 +8980,7 @@
/* Now, interrupt mode asynchronous mailbox command */
rc = lpfc_mbox_cmd_check(phba, mboxq);
if (rc) {
- lpfc_printf_log(phba, KERN_ERR, LOG_MBOX | LOG_SLI,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"(%d):2543 Mailbox command x%x (x%x/x%x) "
"cannot issue Data: x%x x%x\n",
mboxq->vport ? mboxq->vport->vpi : 0,
@@ -8987,7 +9048,7 @@
}
if (unlikely(phba->sli.mbox_active)) {
spin_unlock_irqrestore(&phba->hbalock, iflags);
- lpfc_printf_log(phba, KERN_ERR, LOG_MBOX | LOG_SLI,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"0384 There is pending active mailbox cmd\n");
return MBX_NOT_FINISHED;
}
@@ -9048,7 +9109,7 @@
/* Post the mailbox command to the port */
rc = lpfc_sli4_mq_put(phba->sli4_hba.mbx_wq, mqe);
if (rc != MBX_SUCCESS) {
- lpfc_printf_log(phba, KERN_ERR, LOG_MBOX | LOG_SLI,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"(%d):2533 Mailbox command x%x (x%x/x%x) "
"cannot issue Data: x%x x%x\n",
mboxq->vport ? mboxq->vport->vpi : 0,
@@ -9124,7 +9185,7 @@
phba->lpfc_sli_brdready = lpfc_sli_brdready_s4;
break;
default:
- lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"1420 Invalid HBA PCI-device group: 0x%x\n",
dev_grp);
return -ENODEV;
@@ -9225,8 +9286,7 @@
if (piocb->iocb_cmpl && (!piocb->vport) &&
(piocb->iocb.ulpCommand != CMD_ABORT_XRI_CN) &&
(piocb->iocb.ulpCommand != CMD_CLOSE_XRI_CN)) {
- lpfc_printf_log(phba, KERN_ERR,
- LOG_SLI | LOG_VPORT,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"1807 IOCB x%x failed. No vport\n",
piocb->iocb.ulpCommand);
dump_stack();
@@ -9327,7 +9387,7 @@
/**
* lpfc_sli4_bpl2sgl - Convert the bpl/bde to a sgl.
* @phba: Pointer to HBA context object.
- * @piocb: Pointer to command iocb.
+ * @piocbq: Pointer to command iocb.
* @sglq: Pointer to the scatter gather queue object.
*
* This routine converts the bpl or bde that is in the IOCB
@@ -9435,7 +9495,7 @@
/**
* lpfc_sli_iocb2wqe - Convert the IOCB to a work queue entry.
* @phba: Pointer to HBA context object.
- * @piocb: Pointer to command iocb.
+ * @iocbq: Pointer to command iocb.
* @wqe: Pointer to the work queue entry.
*
* This routine converts the iocb command to its Work Queue Entry
@@ -9521,7 +9581,7 @@
else
ndlp = (struct lpfc_nodelist *)iocbq->context1;
if (!iocbq->iocb.ulpLe) {
- lpfc_printf_log(phba, KERN_ERR, LOG_SLI,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"2007 Only Limited Edition cmd Format"
" supported 0x%x\n",
iocbq->iocb.ulpCommand);
@@ -9829,7 +9889,7 @@
/* word6 context tag copied in memcpy */
if (iocbq->iocb.ulpCt_h || iocbq->iocb.ulpCt_l) {
ct = ((iocbq->iocb.ulpCt_h << 1) | iocbq->iocb.ulpCt_l);
- lpfc_printf_log(phba, KERN_ERR, LOG_SLI,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"2015 Invalid CT %x command 0x%x\n",
ct, iocbq->iocb.ulpCommand);
return IOCB_ERROR;
@@ -10008,7 +10068,7 @@
case CMD_FCP_TRSP64_CX: /* Target mode rcv */
case CMD_FCP_AUTO_TRSP_CX: /* Auto target rsp */
default:
- lpfc_printf_log(phba, KERN_ERR, LOG_SLI,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"2014 Invalid command 0x%x\n",
iocbq->iocb.ulpCommand);
return IOCB_ERROR;
@@ -10129,7 +10189,7 @@
return 0;
}
-/**
+/*
* __lpfc_sli_issue_iocb - Wrapper func of lockless version for issuing iocb
*
* This routine wraps the actual lockless version for issusing IOCB function
@@ -10170,7 +10230,7 @@
phba->__lpfc_sli_release_iocbq = __lpfc_sli_release_iocbq_s4;
break;
default:
- lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"1419 Invalid HBA PCI-device group: 0x%x\n",
dev_grp);
return -ENODEV;
@@ -10218,7 +10278,7 @@
/**
* lpfc_sli_issue_iocb - Wrapper function for __lpfc_sli_issue_iocb
* @phba: Pointer to HBA context object.
- * @pring: Pointer to driver SLI ring object.
+ * @ring_number: Ring number
* @piocb: Pointer to command iocb.
* @flag: Flag indicating if this command can be put into txq.
*
@@ -10430,13 +10490,13 @@
temp_event_data.event_type = FC_REG_TEMPERATURE_EVENT;
if (evt_code == ASYNC_TEMP_WARN) {
temp_event_data.event_code = LPFC_THRESHOLD_TEMP;
- lpfc_printf_log(phba, KERN_ERR, LOG_TEMP,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"0347 Adapter is very hot, please take "
"corrective action. temperature : %d Celsius\n",
(uint32_t) icmd->ulpContext);
} else {
temp_event_data.event_code = LPFC_NORMAL_TEMP;
- lpfc_printf_log(phba, KERN_ERR, LOG_TEMP,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"0340 Adapter temperature is OK now. "
"temperature : %d Celsius\n",
(uint32_t) icmd->ulpContext);
@@ -10453,7 +10513,7 @@
break;
default:
iocb_w = (uint32_t *) icmd;
- lpfc_printf_log(phba, KERN_ERR, LOG_SLI,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"0346 Ring %d handler: unexpected ASYNC_STATUS"
" evt_code 0x%x\n"
"W0 0x%08x W1 0x%08x W2 0x%08x W3 0x%08x\n"
@@ -11127,7 +11187,7 @@
}
spin_unlock_irq(&phba->hbalock);
- lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"0402 Cannot find virtual addr for buffer tag on "
"ring %d Data x%lx x%px x%px x%x\n",
pring->ringno, (unsigned long) tag,
@@ -11171,7 +11231,7 @@
}
spin_unlock_irq(&phba->hbalock);
- lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"0410 Cannot find virtual addr for mapped buf on "
"ring %d Data x%llx x%px x%px x%x\n",
pring->ringno, (unsigned long long)phys,
@@ -11719,7 +11779,7 @@
* @pring: Pointer to driver SLI ring object.
* @tgt_id: SCSI ID of the target.
* @lun_id: LUN ID of the scsi device.
- * @taskmgmt_cmd: LPFC_CTX_LUN/LPFC_CTX_TGT/LPFC_CTX_HOST.
+ * @cmd: LPFC_CTX_LUN/LPFC_CTX_TGT/LPFC_CTX_HOST.
*
* This function sends an abort command for every SCSI command
* associated with the given virtual port pending on the ring
@@ -11959,7 +12019,7 @@
/**
* lpfc_sli_issue_iocb_wait - Synchronous function to issue iocb commands
* @phba: Pointer to HBA context object..
- * @pring: Pointer to sli ring.
+ * @ring_number: Ring number
* @piocb: Pointer to command iocb.
* @prspiocbq: Pointer to response iocb.
* @timeout: Timeout in number of seconds.
@@ -12065,12 +12125,12 @@
* completed. Not that it completed successfully.
* */
} else if (timeleft == 0) {
- lpfc_printf_log(phba, KERN_ERR, LOG_SLI,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"0338 IOCB wait timeout error - no "
"wake response Data x%x\n", timeout);
retval = IOCB_TIMEDOUT;
} else {
- lpfc_printf_log(phba, KERN_ERR, LOG_SLI,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"0330 IOCB wake NOT set, "
"Data x%x x%lx\n",
timeout, (timeleft / jiffies));
@@ -12179,6 +12239,7 @@
/**
* lpfc_sli_mbox_sys_shutdown - shutdown mailbox command sub-system
* @phba: Pointer to HBA context.
+ * @mbx_action: Mailbox shutdown options.
*
* This function is called to shutdown the driver's mailbox sub-system.
* It first marks the mailbox sub-system is in a block state to prevent
@@ -12332,7 +12393,7 @@
}
if ((~phba->sli4_hba.ue_mask_lo & uerr_sta_lo) ||
(~phba->sli4_hba.ue_mask_hi & uerr_sta_hi)) {
- lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"1423 HBA Unrecoverable error: "
"uerr_lo_reg=0x%x, uerr_hi_reg=0x%x, "
"ue_mask_lo_reg=0x%x, "
@@ -12363,7 +12424,7 @@
readl(phba->sli4_hba.u.if_type2.ERR1regaddr);
phba->work_status[1] =
readl(phba->sli4_hba.u.if_type2.ERR2regaddr);
- lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"2885 Port Status Event: "
"port status reg 0x%x, "
"port smphr reg 0x%x, "
@@ -12379,7 +12440,7 @@
break;
case LPFC_SLI_INTF_IF_TYPE_1:
default:
- lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"2886 HBA Error Attention on unsupported "
"if type %d.", if_type);
return 1;
@@ -12443,7 +12504,7 @@
ha_copy = lpfc_sli4_eratt_read(phba);
break;
default:
- lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"0299 Invalid SLI revision (%d)\n",
phba->sli_rev);
ha_copy = 0;
@@ -12676,8 +12737,7 @@
* Stray Mailbox Interrupt, mbxCommand <cmd>
* mbxStatus <status>
*/
- lpfc_printf_log(phba, KERN_ERR, LOG_MBOX |
- LOG_SLI,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"(%d):0304 Stray Mailbox "
"Interrupt mbxCommand x%x "
"mbxStatus x%x\n",
@@ -12737,7 +12797,7 @@
if (rc != MBX_BUSY)
lpfc_printf_log(phba,
KERN_ERR,
- LOG_MBOX | LOG_SLI,
+ LOG_TRACE_EVENT,
"0350 rc should have"
"been MBX_BUSY\n");
if (rc != MBX_NOT_FINISHED)
@@ -12766,8 +12826,9 @@
MBX_NOWAIT);
} while (rc == MBX_NOT_FINISHED);
if (rc != MBX_SUCCESS)
- lpfc_printf_log(phba, KERN_ERR, LOG_MBOX |
- LOG_SLI, "0349 rc should be "
+ lpfc_printf_log(phba, KERN_ERR,
+ LOG_TRACE_EVENT,
+ "0349 rc should be "
"MBX_SUCCESS\n");
}
@@ -13139,7 +13200,7 @@
/**
* lpfc_sli4_els_wcqe_to_rspiocbq - Get response iocbq from els wcqe
* @phba: Pointer to HBA context object.
- * @wcqe: Pointer to work-queue completion queue entry.
+ * @irspiocbq: Pointer to work-queue completion queue entry.
*
* This routine handles an ELS work-queue completion event and construct
* a pseudo response ELS IODBQ from the SLI4 ELS WCQE for the common
@@ -13194,7 +13255,7 @@
/* Allocate a new internal CQ_EVENT entry */
cq_event = lpfc_sli4_cq_event_alloc(phba);
if (!cq_event) {
- lpfc_printf_log(phba, KERN_ERR, LOG_SLI,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"0602 Failed to alloc CQ_EVENT entry\n");
return NULL;
}
@@ -13207,7 +13268,7 @@
/**
* lpfc_sli4_sp_handle_async_event - Handle an asynchronous event
* @phba: Pointer to HBA context object.
- * @cqe: Pointer to mailbox completion queue entry.
+ * @mcqe: Pointer to mailbox completion queue entry.
*
* This routine process a mailbox completion queue entry with asynchronous
* event.
@@ -13240,7 +13301,7 @@
/**
* lpfc_sli4_sp_handle_mbox_event - Handle a mailbox completion event
* @phba: Pointer to HBA context object.
- * @cqe: Pointer to mailbox completion queue entry.
+ * @mcqe: Pointer to mailbox completion queue entry.
*
* This routine process a mailbox completion queue entry with mailbox
* completion event.
@@ -13269,7 +13330,7 @@
spin_lock_irqsave(&phba->hbalock, iflags);
pmb = phba->sli.mbox_active;
if (unlikely(!pmb)) {
- lpfc_printf_log(phba, KERN_ERR, LOG_MBOX,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"1832 No pending MBOX command to handle\n");
spin_unlock_irqrestore(&phba->hbalock, iflags);
goto out_no_mqe_complete;
@@ -13318,8 +13379,9 @@
pmb->vport = vport;
rc = lpfc_sli_issue_mbox(phba, pmb, MBX_NOWAIT);
if (rc != MBX_BUSY)
- lpfc_printf_log(phba, KERN_ERR, LOG_MBOX |
- LOG_SLI, "0385 rc should "
+ lpfc_printf_log(phba, KERN_ERR,
+ LOG_TRACE_EVENT,
+ "0385 rc should "
"have been MBX_BUSY\n");
if (rc != MBX_NOT_FINISHED)
goto send_current_mbox;
@@ -13360,6 +13422,7 @@
/**
* lpfc_sli4_sp_handle_mcqe - Process a mailbox completion queue entry
* @phba: Pointer to HBA context object.
+ * @cq: Pointer to associated CQ
* @cqe: Pointer to mailbox completion queue entry.
*
* This routine process a mailbox completion queue entry, it invokes the
@@ -13426,7 +13489,7 @@
txq_cnt++;
if (!list_empty(&pring->txcmplq))
txcmplq_cnt++;
- lpfc_printf_log(phba, KERN_ERR, LOG_SLI,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"0387 NO IOCBQ data: txq_cnt=%d iocb_cnt=%d "
"els_txcmplq_cnt=%d\n",
txq_cnt, phba->iocb_cnt,
@@ -13517,7 +13580,7 @@
workposted = true;
break;
default:
- lpfc_printf_log(phba, KERN_ERR, LOG_SLI,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"0603 Invalid CQ subtype %d: "
"%08x %08x %08x %08x\n",
cq->subtype, wcqe->word0, wcqe->parameter,
@@ -13565,7 +13628,7 @@
status = bf_get(lpfc_rcqe_status, rcqe);
switch (status) {
case FC_STATUS_RQ_BUF_LEN_EXCEEDED:
- lpfc_printf_log(phba, KERN_ERR, LOG_SLI,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"2537 Receive Frame Truncated!!\n");
/* fall through */
case FC_STATUS_RQ_SUCCESS:
@@ -13602,7 +13665,7 @@
case FC_STATUS_INSUFF_BUF_FRM_DISC:
if (phba->nvmet_support) {
tgtp = phba->targetport->private;
- lpfc_printf_log(phba, KERN_ERR, LOG_SLI | LOG_NVME,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"6402 RQE Error x%x, posted %d err_cnt "
"%d: %x %x %x\n",
status, hrq->RQ_buf_posted,
@@ -13674,7 +13737,7 @@
(struct lpfc_rcqe *)&cqevt);
break;
default:
- lpfc_printf_log(phba, KERN_ERR, LOG_SLI,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"0388 Not a valid WCQE code: x%x\n",
bf_get(lpfc_cqe_code, &cqevt));
break;
@@ -13686,6 +13749,7 @@
* lpfc_sli4_sp_handle_eqe - Process a slow-path event queue entry
* @phba: Pointer to HBA context object.
* @eqe: Pointer to fast-path event queue entry.
+ * @speq: Pointer to slow-path event queue.
*
* This routine process a event queue entry from the slow-path event queue.
* It will check the MajorCode and MinorCode to determine this is for a
@@ -13701,6 +13765,7 @@
{
struct lpfc_queue *cq = NULL, *childq;
uint16_t cqid;
+ int ret = 0;
/* Get the reference to the corresponding CQ */
cqid = bf_get_le32(lpfc_eqe_resource_id, eqe);
@@ -13713,7 +13778,7 @@
}
if (unlikely(!cq)) {
if (phba->sli.sli_flag & LPFC_SLI_ACTIVE)
- lpfc_printf_log(phba, KERN_ERR, LOG_SLI,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"0365 Slow-path CQ identifier "
"(%d) does not exist\n", cqid);
return;
@@ -13722,9 +13787,14 @@
/* Save EQ associated with this CQ */
cq->assoc_qp = speq;
- if (!queue_work_on(cq->chann, phba->wq, &cq->spwork))
- lpfc_printf_log(phba, KERN_ERR, LOG_SLI,
- "0390 Cannot schedule soft IRQ "
+ if (is_kdump_kernel())
+ ret = queue_work(phba->wq, &cq->spwork);
+ else
+ ret = queue_work_on(cq->chann, phba->wq, &cq->spwork);
+
+ if (!ret)
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
+ "0390 Cannot schedule queue work "
"for CQ eqcqid=%d, cqid=%d on CPU %d\n",
cqid, cq->queue_id, raw_smp_processor_id());
}
@@ -13735,6 +13805,7 @@
* @cq: Pointer to CQ to be processed
* @handler: Routine to process each cqe
* @delay: Pointer to usdelay to set in case of rescheduling of the handler
+ * @poll_mode: Polling mode we were called from
*
* This routine processes completion queue entries in a CQ. While a valid
* queue element is found, the handler is called. During processing checks
@@ -13752,7 +13823,8 @@
static bool
__lpfc_sli4_process_cq(struct lpfc_hba *phba, struct lpfc_queue *cq,
bool (*handler)(struct lpfc_hba *, struct lpfc_queue *,
- struct lpfc_cqe *), unsigned long *delay)
+ struct lpfc_cqe *), unsigned long *delay,
+ enum lpfc_poll_mode poll_mode)
{
struct lpfc_cqe *cqe;
bool workposted = false;
@@ -13793,6 +13865,10 @@
arm = false;
}
+ /* Note: complete the irq_poll softirq before rearming CQ */
+ if (poll_mode == LPFC_IRQ_POLL)
+ irq_poll_complete(&cq->iop);
+
/* Track the max number of CQEs processed in 1 EQ */
if (count > cq->CQ_max_cqe)
cq->CQ_max_cqe = count;
@@ -13835,36 +13911,42 @@
struct lpfc_hba *phba = cq->phba;
unsigned long delay;
bool workposted = false;
+ int ret = 0;
/* Process and rearm the CQ */
switch (cq->type) {
case LPFC_MCQ:
workposted |= __lpfc_sli4_process_cq(phba, cq,
lpfc_sli4_sp_handle_mcqe,
- &delay);
+ &delay, LPFC_QUEUE_WORK);
break;
case LPFC_WCQ:
if (cq->subtype == LPFC_IO)
workposted |= __lpfc_sli4_process_cq(phba, cq,
lpfc_sli4_fp_handle_cqe,
- &delay);
+ &delay, LPFC_QUEUE_WORK);
else
workposted |= __lpfc_sli4_process_cq(phba, cq,
lpfc_sli4_sp_handle_cqe,
- &delay);
+ &delay, LPFC_QUEUE_WORK);
break;
default:
- lpfc_printf_log(phba, KERN_ERR, LOG_SLI,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"0370 Invalid completion queue type (%d)\n",
cq->type);
return;
}
if (delay) {
- if (!queue_delayed_work_on(cq->chann, phba->wq,
- &cq->sched_spwork, delay))
- lpfc_printf_log(phba, KERN_ERR, LOG_SLI,
- "0394 Cannot schedule soft IRQ "
+ if (is_kdump_kernel())
+ ret = queue_delayed_work(phba->wq, &cq->sched_spwork,
+ delay);
+ else
+ ret = queue_delayed_work_on(cq->chann, phba->wq,
+ &cq->sched_spwork, delay);
+ if (!ret)
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
+ "0394 Cannot schedule queue work "
"for cqid=%d on CPU %d\n",
cq->queue_id, cq->chann);
}
@@ -13933,9 +14015,9 @@
IOERR_NO_RESOURCES))
phba->lpfc_rampdown_queue_depth(phba);
- /* Log the error status */
+ /* Log the cmpl status */
lpfc_printf_log(phba, KERN_INFO, LOG_SLI,
- "0373 FCP CQE error: status=x%x: "
+ "0373 FCP CQE cmpl: status=x%x: "
"CQE: %08x %08x %08x %08x\n",
bf_get(lpfc_wcqe_c_status, wcqe),
wcqe->word0, wcqe->total_data_placed,
@@ -14029,6 +14111,7 @@
/**
* lpfc_sli4_nvmet_handle_rcqe - Process a receive-queue completion queue entry
* @phba: Pointer to HBA context object.
+ * @cq: Pointer to completion queue.
* @rcqe: Pointer to receive-queue completion queue entry.
*
* This routine process a receive-queue completion queue entry.
@@ -14073,7 +14156,7 @@
status = bf_get(lpfc_rcqe_status, rcqe);
switch (status) {
case FC_STATUS_RQ_BUF_LEN_EXCEEDED:
- lpfc_printf_log(phba, KERN_ERR, LOG_SLI,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"6126 Receive Frame Truncated!!\n");
/* fall through */
case FC_STATUS_RQ_SUCCESS:
@@ -14113,7 +14196,7 @@
case FC_STATUS_INSUFF_BUF_FRM_DISC:
if (phba->nvmet_support) {
tgtp = phba->targetport->private;
- lpfc_printf_log(phba, KERN_ERR, LOG_SLI | LOG_NVME,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"6401 RQE Error x%x, posted %d err_cnt "
"%d: %x %x %x\n",
status, hrq->RQ_buf_posted,
@@ -14137,7 +14220,7 @@
* lpfc_sli4_fp_handle_cqe - Process fast-path work queue completion entry
* @phba: adapter with cq
* @cq: Pointer to the completion queue.
- * @eqe: Pointer to fast-path completion queue entry.
+ * @cqe: Pointer to fast-path completion queue entry.
*
* This routine process a fast-path work queue completion entry from fast-path
* event queue for FCP command response completion.
@@ -14187,7 +14270,7 @@
}
break;
default:
- lpfc_printf_log(phba, KERN_ERR, LOG_SLI,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"0144 Not a valid CQE code: x%x\n",
bf_get(lpfc_wcqe_c_code, &wcqe));
break;
@@ -14196,8 +14279,47 @@
}
/**
+ * lpfc_sli4_sched_cq_work - Schedules cq work
+ * @phba: Pointer to HBA context object.
+ * @cq: Pointer to CQ
+ * @cqid: CQ ID
+ *
+ * This routine checks the poll mode of the CQ corresponding to
+ * cq->chann, then either schedules a softirq or queue_work to complete
+ * cq work.
+ *
+ * queue_work path is taken if in NVMET mode, or if poll_mode is in
+ * LPFC_QUEUE_WORK mode. Otherwise, softirq path is taken.
+ *
+ **/
+static void lpfc_sli4_sched_cq_work(struct lpfc_hba *phba,
+ struct lpfc_queue *cq, uint16_t cqid)
+{
+ int ret = 0;
+
+ switch (cq->poll_mode) {
+ case LPFC_IRQ_POLL:
+ irq_poll_sched(&cq->iop);
+ break;
+ case LPFC_QUEUE_WORK:
+ default:
+ if (is_kdump_kernel())
+ ret = queue_work(phba->wq, &cq->irqwork);
+ else
+ ret = queue_work_on(cq->chann, phba->wq, &cq->irqwork);
+ if (!ret)
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
+ "0383 Cannot schedule queue work "
+ "for CQ eqcqid=%d, cqid=%d on CPU %d\n",
+ cqid, cq->queue_id,
+ raw_smp_processor_id());
+ }
+}
+
+/**
* lpfc_sli4_hba_handle_eqe - Process a fast-path event queue entry
* @phba: Pointer to HBA context object.
+ * @eq: Pointer to the queue structure.
* @eqe: Pointer to fast-path event queue entry.
*
* This routine process a event queue entry from the fast-path event queue.
@@ -14216,7 +14338,7 @@
uint16_t cqid, id;
if (unlikely(bf_get_le32(lpfc_eqe_major_code, eqe) != 0)) {
- lpfc_printf_log(phba, KERN_ERR, LOG_SLI,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"0366 Not a valid completion "
"event: majorcode=x%x, minorcode=x%x\n",
bf_get_le32(lpfc_eqe_major_code, eqe),
@@ -14259,7 +14381,7 @@
process_cq:
if (unlikely(cqid != cq->queue_id)) {
- lpfc_printf_log(phba, KERN_ERR, LOG_SLI,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"0368 Miss-matched fast-path completion "
"queue identifier: eqcqid=%d, fcpcqid=%d\n",
cqid, cq->queue_id);
@@ -14273,16 +14395,13 @@
else
cq->isr_timestamp = 0;
#endif
- if (!queue_work_on(cq->chann, phba->wq, &cq->irqwork))
- lpfc_printf_log(phba, KERN_ERR, LOG_SLI,
- "0363 Cannot schedule soft IRQ "
- "for CQ eqcqid=%d, cqid=%d on CPU %d\n",
- cqid, cq->queue_id, raw_smp_processor_id());
+ lpfc_sli4_sched_cq_work(phba, cq, cqid);
}
/**
* __lpfc_sli4_hba_process_cq - Process a fast-path event queue entry
* @cq: Pointer to CQ to be processed
+ * @poll_mode: Enum lpfc_poll_state to determine poll mode
*
* This routine calls the cq processing routine with the handler for
* fast path CQEs.
@@ -14296,23 +14415,30 @@
* the delay indicates when to reschedule it.
**/
static void
-__lpfc_sli4_hba_process_cq(struct lpfc_queue *cq)
+__lpfc_sli4_hba_process_cq(struct lpfc_queue *cq,
+ enum lpfc_poll_mode poll_mode)
{
struct lpfc_hba *phba = cq->phba;
unsigned long delay;
bool workposted = false;
+ int ret = 0;
/* process and rearm the CQ */
workposted |= __lpfc_sli4_process_cq(phba, cq, lpfc_sli4_fp_handle_cqe,
- &delay);
+ &delay, poll_mode);
if (delay) {
- if (!queue_delayed_work_on(cq->chann, phba->wq,
- &cq->sched_irqwork, delay))
- lpfc_printf_log(phba, KERN_ERR, LOG_SLI,
- "0367 Cannot schedule soft IRQ "
- "for cqid=%d on CPU %d\n",
- cq->queue_id, cq->chann);
+ if (is_kdump_kernel())
+ ret = queue_delayed_work(phba->wq, &cq->sched_irqwork,
+ delay);
+ else
+ ret = queue_delayed_work_on(cq->chann, phba->wq,
+ &cq->sched_irqwork, delay);
+ if (!ret)
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
+ "0367 Cannot schedule queue work "
+ "for cqid=%d on CPU %d\n",
+ cq->queue_id, cq->chann);
}
/* wake up worker thread if there are works to be done */
@@ -14332,7 +14458,7 @@
{
struct lpfc_queue *cq = container_of(work, struct lpfc_queue, irqwork);
- __lpfc_sli4_hba_process_cq(cq);
+ __lpfc_sli4_hba_process_cq(cq, LPFC_QUEUE_WORK);
}
/**
@@ -14347,7 +14473,7 @@
struct lpfc_queue *cq = container_of(to_delayed_work(work),
struct lpfc_queue, sched_irqwork);
- __lpfc_sli4_hba_process_cq(cq);
+ __lpfc_sli4_hba_process_cq(cq, LPFC_QUEUE_WORK);
}
/**
@@ -14668,7 +14794,7 @@
* @phba: The HBA that this queue is being created on.
* @page_size: The size of a queue page
* @entry_size: The size of each queue entry for this queue.
- * @entry count: The number of entries that this queue will handle.
+ * @entry_count: The number of entries that this queue will handle.
* @cpu: The cpu that will primarily utilize this queue.
*
* This function allocates a queue structure and the DMAable memory used for
@@ -14840,7 +14966,7 @@
mbox = mempool_alloc(phba->mbox_mem_pool, GFP_KERNEL);
if (!mbox) {
- lpfc_printf_log(phba, KERN_ERR, LOG_INIT | LOG_FCP | LOG_NVME,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"6428 Failed allocating mailbox cmd buffer."
" EQ delay was not set.\n");
return;
@@ -14882,7 +15008,7 @@
shdr_status = bf_get(lpfc_mbox_hdr_status, &shdr->response);
shdr_add_status = bf_get(lpfc_mbox_hdr_add_status, &shdr->response);
if (shdr_status || shdr_add_status || rc) {
- lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"2512 MODIFY_EQ_DELAY mailbox failed with "
"status x%x add_status x%x, mbx status x%x\n",
shdr_status, shdr_add_status, rc);
@@ -14959,7 +15085,7 @@
dmult);
switch (eq->entry_count) {
default:
- lpfc_printf_log(phba, KERN_ERR, LOG_SLI,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"0360 Unsupported EQ count. (%d)\n",
eq->entry_count);
if (eq->entry_count < 256) {
@@ -15003,7 +15129,7 @@
shdr_status = bf_get(lpfc_mbox_hdr_status, &shdr->response);
shdr_add_status = bf_get(lpfc_mbox_hdr_add_status, &shdr->response);
if (shdr_status || shdr_add_status || rc) {
- lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"2500 EQ_CREATE mailbox failed with "
"status x%x add_status x%x, mbx status x%x\n",
shdr_status, shdr_add_status, rc);
@@ -15022,11 +15148,22 @@
return status;
}
+static int lpfc_cq_poll_hdler(struct irq_poll *iop, int budget)
+{
+ struct lpfc_queue *cq = container_of(iop, struct lpfc_queue, iop);
+
+ __lpfc_sli4_hba_process_cq(cq, LPFC_IRQ_POLL);
+
+ return 1;
+}
+
/**
* lpfc_cq_create - Create a Completion Queue on the HBA
* @phba: HBA structure that indicates port to create a queue on.
* @cq: The queue structure to use to create the completion queue.
* @eq: The event queue to bind this completion queue to.
+ * @type: Type of queue (EQ, GCQ, MCQ, WCQ, etc).
+ * @subtype: Functional purpose of the queue (MBOX, IO, ELS, NVMET, etc).
*
* This function creates a completion queue, as detailed in @wq, on a port,
* described by @phba by sending a CQ_CREATE mailbox command to the HBA.
@@ -15099,7 +15236,7 @@
}
/* fall through */
default:
- lpfc_printf_log(phba, KERN_ERR, LOG_SLI,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"0361 Unsupported CQ count: "
"entry cnt %d sz %d pg cnt %d\n",
cq->entry_count, cq->entry_size,
@@ -15135,7 +15272,7 @@
shdr_status = bf_get(lpfc_mbox_hdr_status, &shdr->response);
shdr_add_status = bf_get(lpfc_mbox_hdr_add_status, &shdr->response);
if (shdr_status || shdr_add_status || rc) {
- lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"2501 CQ_CREATE mailbox failed with "
"status x%x add_status x%x, mbx status x%x\n",
shdr_status, shdr_add_status, rc);
@@ -15161,6 +15298,8 @@
if (cq->queue_id > phba->sli4_hba.cq_max)
phba->sli4_hba.cq_max = cq->queue_id;
+
+ irq_poll_init(&cq->iop, LPFC_IRQ_POLL_WEIGHT, lpfc_cq_poll_hdler);
out:
mempool_free(mbox, phba->mbox_mem_pool);
return status;
@@ -15171,6 +15310,8 @@
* @phba: HBA structure that indicates port to create a queue on.
* @cqp: The queue structure array to use to create the completion queues.
* @hdwq: The hardware queue array with the EQ to bind completion queues to.
+ * @type: Type of queue (EQ, GCQ, MCQ, WCQ, etc).
+ * @subtype: Functional purpose of the queue (MBOX, IO, ELS, NVMET, etc).
*
* This function creates a set of completion queue, s to support MRQ
* as detailed in @cqp, on a port,
@@ -15220,7 +15361,7 @@
LPFC_MBOX_OPCODE_FCOE_CQ_CREATE_SET, length,
LPFC_SLI4_MBX_NEMBED);
if (alloclen < length) {
- lpfc_printf_log(phba, KERN_ERR, LOG_SLI,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"3098 Allocated DMA memory size (%d) is "
"less than the requested DMA memory size "
"(%d)\n", alloclen, length);
@@ -15274,7 +15415,7 @@
}
/* fall through */
default:
- lpfc_printf_log(phba, KERN_ERR, LOG_SLI,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"3118 Bad CQ count. (%d)\n",
cq->entry_count);
if (cq->entry_count < 256) {
@@ -15392,7 +15533,7 @@
shdr_status = bf_get(lpfc_mbox_hdr_status, &shdr->response);
shdr_add_status = bf_get(lpfc_mbox_hdr_add_status, &shdr->response);
if (shdr_status || shdr_add_status || rc) {
- lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"3119 CQ_CREATE_SET mailbox failed with "
"status x%x add_status x%x, mbx status x%x\n",
shdr_status, shdr_add_status, rc);
@@ -15550,7 +15691,7 @@
cq->queue_id);
switch (mq->entry_count) {
default:
- lpfc_printf_log(phba, KERN_ERR, LOG_SLI,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"0362 Unsupported MQ count. (%d)\n",
mq->entry_count);
if (mq->entry_count < 16) {
@@ -15606,7 +15747,7 @@
shdr_status = bf_get(lpfc_mbox_hdr_status, &shdr->response);
shdr_add_status = bf_get(lpfc_mbox_hdr_add_status, &shdr->response);
if (shdr_status || shdr_add_status || rc) {
- lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"2502 MQ_CREATE mailbox failed with "
"status x%x add_status x%x, mbx status x%x\n",
shdr_status, shdr_add_status, rc);
@@ -15669,8 +15810,10 @@
uint16_t pci_barset;
uint8_t dpp_barset;
uint32_t dpp_offset;
- unsigned long pg_addr;
uint8_t wq_create_version;
+#ifdef CONFIG_X86
+ unsigned long pg_addr;
+#endif
/* sanity check on queue memory */
if (!wq || !cq)
@@ -15755,7 +15898,7 @@
shdr_status = bf_get(lpfc_mbox_hdr_status, &shdr->response);
shdr_add_status = bf_get(lpfc_mbox_hdr_add_status, &shdr->response);
if (shdr_status || shdr_add_status || rc) {
- lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"2503 WQ_CREATE mailbox failed with "
"status x%x add_status x%x, mbx status x%x\n",
shdr_status, shdr_add_status, rc);
@@ -15782,7 +15925,7 @@
&wq_create->u.response);
if ((wq->db_format != LPFC_DB_LIST_FORMAT) &&
(wq->db_format != LPFC_DB_RING_FORMAT)) {
- lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"3265 WQ[%d] doorbell format "
"not supported: x%x\n",
wq->queue_id, wq->db_format);
@@ -15794,7 +15937,7 @@
bar_memmap_p = lpfc_dual_chute_pci_bar_map(phba,
pci_barset);
if (!bar_memmap_p) {
- lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"3263 WQ[%d] failed to memmap "
"pci barset:x%x\n",
wq->queue_id, pci_barset);
@@ -15804,7 +15947,7 @@
db_offset = wq_create->u.response.doorbell_offset;
if ((db_offset != LPFC_ULP0_WQ_DOORBELL) &&
(db_offset != LPFC_ULP1_WQ_DOORBELL)) {
- lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"3252 WQ[%d] doorbell offset "
"not supported: x%x\n",
wq->queue_id, db_offset);
@@ -15828,7 +15971,7 @@
bar_memmap_p = lpfc_dual_chute_pci_bar_map(phba,
pci_barset);
if (!bar_memmap_p) {
- lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"3267 WQ[%d] failed to memmap "
"pci barset:x%x\n",
wq->queue_id, pci_barset);
@@ -15844,7 +15987,7 @@
bar_memmap_p = lpfc_dual_chute_pci_bar_map(phba,
dpp_barset);
if (!bar_memmap_p) {
- lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"3268 WQ[%d] failed to memmap "
"pci barset:x%x\n",
wq->queue_id, dpp_barset);
@@ -15860,9 +16003,9 @@
wq->queue_id, pci_barset, db_offset,
wq->dpp_id, dpp_barset, dpp_offset);
+#ifdef CONFIG_X86
/* Enable combined writes for DPP aperture */
pg_addr = (unsigned long)(wq->dpp_regaddr) & PAGE_MASK;
-#ifdef CONFIG_X86
rc = set_memory_wc(pg_addr, 1);
if (rc) {
lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
@@ -15902,6 +16045,7 @@
* @hrq: The queue structure to use to create the header receive queue.
* @drq: The queue structure to use to create the data receive queue.
* @cq: The completion queue to bind this work queue to.
+ * @subtype: The subtype of the work queue indicating its functionality.
*
* This function creates a receive buffer queue pair , as detailed in @hrq and
* @drq, on a port, described by @phba by sending a RQ_CREATE mailbox command
@@ -15968,7 +16112,7 @@
} else {
switch (hrq->entry_count) {
default:
- lpfc_printf_log(phba, KERN_ERR, LOG_SLI,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"2535 Unsupported RQ count. (%d)\n",
hrq->entry_count);
if (hrq->entry_count < 512) {
@@ -16019,7 +16163,7 @@
shdr_status = bf_get(lpfc_mbox_hdr_status, &shdr->response);
shdr_add_status = bf_get(lpfc_mbox_hdr_add_status, &shdr->response);
if (shdr_status || shdr_add_status || rc) {
- lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"2504 RQ_CREATE mailbox failed with "
"status x%x add_status x%x, mbx status x%x\n",
shdr_status, shdr_add_status, rc);
@@ -16037,7 +16181,7 @@
&rq_create->u.response);
if ((hrq->db_format != LPFC_DB_LIST_FORMAT) &&
(hrq->db_format != LPFC_DB_RING_FORMAT)) {
- lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"3262 RQ [%d] doorbell format not "
"supported: x%x\n", hrq->queue_id,
hrq->db_format);
@@ -16049,7 +16193,7 @@
&rq_create->u.response);
bar_memmap_p = lpfc_dual_chute_pci_bar_map(phba, pci_barset);
if (!bar_memmap_p) {
- lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"3269 RQ[%d] failed to memmap pci "
"barset:x%x\n", hrq->queue_id,
pci_barset);
@@ -16060,7 +16204,7 @@
db_offset = rq_create->u.response.doorbell_offset;
if ((db_offset != LPFC_ULP0_RQ_DOORBELL) &&
(db_offset != LPFC_ULP1_RQ_DOORBELL)) {
- lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"3270 RQ[%d] doorbell offset not "
"supported: x%x\n", hrq->queue_id,
db_offset);
@@ -16105,7 +16249,7 @@
} else {
switch (drq->entry_count) {
default:
- lpfc_printf_log(phba, KERN_ERR, LOG_SLI,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"2536 Unsupported RQ count. (%d)\n",
drq->entry_count);
if (drq->entry_count < 512) {
@@ -16191,6 +16335,7 @@
* @hrqp: The queue structure array to use to create the header receive queues.
* @drqp: The queue structure array to use to create the data receive queues.
* @cqp: The completion queue array to bind these receive queues to.
+ * @subtype: Functional purpose of the queue (MBOX, IO, ELS, NVMET, etc).
*
* This function creates a receive buffer queue pair , as detailed in @hrq and
* @drq, on a port, described by @phba by sending a RQ_CREATE mailbox command
@@ -16242,7 +16387,7 @@
LPFC_MBOX_OPCODE_FCOE_RQ_CREATE, length,
LPFC_SLI4_MBX_NEMBED);
if (alloclen < length) {
- lpfc_printf_log(phba, KERN_ERR, LOG_SLI,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"3099 Allocated DMA memory size (%d) is "
"less than the requested DMA memory size "
"(%d)\n", alloclen, length);
@@ -16352,7 +16497,7 @@
shdr_status = bf_get(lpfc_mbox_hdr_status, &shdr->response);
shdr_add_status = bf_get(lpfc_mbox_hdr_add_status, &shdr->response);
if (shdr_status || shdr_add_status || rc) {
- lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"3120 RQ_CREATE mailbox failed with "
"status x%x add_status x%x, mbx status x%x\n",
shdr_status, shdr_add_status, rc);
@@ -16380,6 +16525,7 @@
/**
* lpfc_eq_destroy - Destroy an event Queue on the HBA
+ * @phba: HBA structure that indicates port to destroy a queue on.
* @eq: The queue structure associated with the queue to destroy.
*
* This function destroys a queue, as detailed in @eq by sending an mailbox
@@ -16422,7 +16568,7 @@
shdr_status = bf_get(lpfc_mbox_hdr_status, &shdr->response);
shdr_add_status = bf_get(lpfc_mbox_hdr_add_status, &shdr->response);
if (shdr_status || shdr_add_status || rc) {
- lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"2505 EQ_DESTROY mailbox failed with "
"status x%x add_status x%x, mbx status x%x\n",
shdr_status, shdr_add_status, rc);
@@ -16437,6 +16583,7 @@
/**
* lpfc_cq_destroy - Destroy a Completion Queue on the HBA
+ * @phba: HBA structure that indicates port to destroy a queue on.
* @cq: The queue structure associated with the queue to destroy.
*
* This function destroys a queue, as detailed in @cq by sending an mailbox
@@ -16477,7 +16624,7 @@
shdr_status = bf_get(lpfc_mbox_hdr_status, &shdr->response);
shdr_add_status = bf_get(lpfc_mbox_hdr_add_status, &shdr->response);
if (shdr_status || shdr_add_status || rc) {
- lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"2506 CQ_DESTROY mailbox failed with "
"status x%x add_status x%x, mbx status x%x\n",
shdr_status, shdr_add_status, rc);
@@ -16491,7 +16638,8 @@
/**
* lpfc_mq_destroy - Destroy a Mailbox Queue on the HBA
- * @qm: The queue structure associated with the queue to destroy.
+ * @phba: HBA structure that indicates port to destroy a queue on.
+ * @mq: The queue structure associated with the queue to destroy.
*
* This function destroys a queue, as detailed in @mq by sending an mailbox
* command, specific to the type of queue, to the HBA.
@@ -16531,7 +16679,7 @@
shdr_status = bf_get(lpfc_mbox_hdr_status, &shdr->response);
shdr_add_status = bf_get(lpfc_mbox_hdr_add_status, &shdr->response);
if (shdr_status || shdr_add_status || rc) {
- lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"2507 MQ_DESTROY mailbox failed with "
"status x%x add_status x%x, mbx status x%x\n",
shdr_status, shdr_add_status, rc);
@@ -16545,6 +16693,7 @@
/**
* lpfc_wq_destroy - Destroy a Work Queue on the HBA
+ * @phba: HBA structure that indicates port to destroy a queue on.
* @wq: The queue structure associated with the queue to destroy.
*
* This function destroys a queue, as detailed in @wq by sending an mailbox
@@ -16584,7 +16733,7 @@
shdr_status = bf_get(lpfc_mbox_hdr_status, &shdr->response);
shdr_add_status = bf_get(lpfc_mbox_hdr_add_status, &shdr->response);
if (shdr_status || shdr_add_status || rc) {
- lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"2508 WQ_DESTROY mailbox failed with "
"status x%x add_status x%x, mbx status x%x\n",
shdr_status, shdr_add_status, rc);
@@ -16600,7 +16749,9 @@
/**
* lpfc_rq_destroy - Destroy a Receive Queue on the HBA
- * @rq: The queue structure associated with the queue to destroy.
+ * @phba: HBA structure that indicates port to destroy a queue on.
+ * @hrq: The queue structure associated with the queue to destroy.
+ * @drq: The queue structure associated with the queue to destroy.
*
* This function destroys a queue, as detailed in @rq by sending an mailbox
* command, specific to the type of queue, to the HBA.
@@ -16641,7 +16792,7 @@
shdr_status = bf_get(lpfc_mbox_hdr_status, &shdr->response);
shdr_add_status = bf_get(lpfc_mbox_hdr_add_status, &shdr->response);
if (shdr_status || shdr_add_status || rc) {
- lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"2509 RQ_DESTROY mailbox failed with "
"status x%x add_status x%x, mbx status x%x\n",
shdr_status, shdr_add_status, rc);
@@ -16657,7 +16808,7 @@
shdr_status = bf_get(lpfc_mbox_hdr_status, &shdr->response);
shdr_add_status = bf_get(lpfc_mbox_hdr_add_status, &shdr->response);
if (shdr_status || shdr_add_status || rc) {
- lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"2510 RQ_DESTROY mailbox failed with "
"status x%x add_status x%x, mbx status x%x\n",
shdr_status, shdr_add_status, rc);
@@ -16705,7 +16856,7 @@
union lpfc_sli4_cfg_shdr *shdr;
if (xritag == NO_XRI) {
- lpfc_printf_log(phba, KERN_ERR, LOG_SLI,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"0364 Invalid param:\n");
return -EINVAL;
}
@@ -16746,7 +16897,7 @@
if (rc != MBX_TIMEOUT)
mempool_free(mbox, phba->mbox_mem_pool);
if (shdr_status || shdr_add_status || rc) {
- lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"2511 POST_SGL mailbox failed with "
"status x%x add_status x%x, mbx status x%x\n",
shdr_status, shdr_add_status, rc);
@@ -16793,6 +16944,7 @@
/**
* lpfc_sli4_free_xri - Release an xri for reuse.
* @phba: pointer to lpfc hba data structure.
+ * @xri: xri to release.
*
* This routine is invoked to release an xri to the pool of
* available rpis maintained by the driver.
@@ -16808,6 +16960,7 @@
/**
* lpfc_sli4_free_xri - Release an xri for reuse.
* @phba: pointer to lpfc hba data structure.
+ * @xri: xri to release.
*
* This routine is invoked to release an xri to the pool of
* available rpis maintained by the driver.
@@ -16850,7 +17003,7 @@
* lpfc_sli4_post_sgl_list - post a block of ELS sgls to the port.
* @phba: pointer to lpfc hba data structure.
* @post_sgl_list: pointer to els sgl entry list.
- * @count: number of els sgl entries on the list.
+ * @post_cnt: number of els sgl entries on the list.
*
* This routine is invoked to post a block of driver's sgl pages to the
* HBA using non-embedded mailbox command. No Lock is held. This routine
@@ -16877,7 +17030,7 @@
reqlen = post_cnt * sizeof(struct sgl_page_pairs) +
sizeof(union lpfc_sli4_cfg_shdr) + sizeof(uint32_t);
if (reqlen > SLI4_PAGE_SIZE) {
- lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"2559 Block sgl registration required DMA "
"size (%d) great than a page\n", reqlen);
return -ENOMEM;
@@ -16893,7 +17046,7 @@
LPFC_SLI4_MBX_NEMBED);
if (alloclen < reqlen) {
- lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"0285 Allocated DMA memory size (%d) is "
"less than the requested DMA memory "
"size (%d)\n", alloclen, reqlen);
@@ -16941,7 +17094,7 @@
if (rc != MBX_TIMEOUT)
lpfc_sli4_mbox_cmd_free(phba, mbox);
if (shdr_status || shdr_add_status || rc) {
- lpfc_printf_log(phba, KERN_ERR, LOG_SLI,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"2513 POST_SGL_BLOCK mailbox command failed "
"status x%x add_status x%x mbx status x%x\n",
shdr_status, shdr_add_status, rc);
@@ -16989,7 +17142,7 @@
}
mbox = mempool_alloc(phba->mbox_mem_pool, GFP_KERNEL);
if (!mbox) {
- lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"6119 Failed to allocate mbox cmd memory\n");
return -ENOMEM;
}
@@ -17000,7 +17153,7 @@
reqlen, LPFC_SLI4_MBX_NEMBED);
if (alloclen < reqlen) {
- lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"6120 Allocated DMA memory size (%d) is "
"less than the requested DMA memory "
"size (%d)\n", alloclen, reqlen);
@@ -17054,7 +17207,7 @@
if (rc != MBX_TIMEOUT)
lpfc_sli4_mbox_cmd_free(phba, mbox);
if (shdr_status || shdr_add_status || rc) {
- lpfc_printf_log(phba, KERN_ERR, LOG_SLI,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"6125 POST_SGL_BLOCK mailbox command failed "
"status x%x add_status x%x mbx status x%x\n",
shdr_status, shdr_add_status, rc);
@@ -17067,6 +17220,7 @@
* lpfc_sli4_post_io_sgl_list - Post blocks of nvme buffer sgls from a list
* @phba: pointer to lpfc hba data structure.
* @post_nblist: pointer to the nvme buffer list.
+ * @sb_count: number of nvme buffers.
*
* This routine walks a list of nvme buffers that was passed in. It attempts
* to construct blocks of nvme buffer sgls which contains contiguous xris and
@@ -17300,6 +17454,7 @@
* @phba: Pointer to the HBA structure to search for the vport on
* @fc_hdr: A pointer to the FC Header data (In Big Endian Format)
* @fcfi: The FC Fabric ID that the frame came from
+ * @did: Destination ID to match against
*
* This function searches the @phba for a vport that matches the content of the
* @fc_hdr passed in and the @fcfi. This function uses the @fc_hdr to fetch the
@@ -17437,6 +17592,7 @@
/**
* lpfc_fc_frame_add - Adds a frame to the vport's list of received sequences
+ * @vport: pointer to a vitural port
* @dmabuf: pointer to a dmabuf that describes the hdr and data of the FC frame
*
* This function searches through the existing incomplete sequences that have
@@ -17638,7 +17794,7 @@
/* Failure means BLS ABORT RSP did not get delivered to remote node*/
if (rsp_iocbq && rsp_iocbq->iocb.ulpStatus)
- lpfc_printf_log(phba, KERN_ERR, LOG_SLI,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"3154 BLS ABORT RSP failed, data: x%x/x%x\n",
rsp_iocbq->iocb.ulpStatus,
rsp_iocbq->iocb.un.ulpWord[4]);
@@ -17667,8 +17823,9 @@
/**
* lpfc_sli4_seq_abort_rsp - bls rsp to sequence abort
- * @phba: Pointer to HBA context object.
+ * @vport: pointer to a vitural port.
* @fc_hdr: pointer to a FC frame header.
+ * @aborted: was the partially assembled receive sequence successfully aborted
*
* This function sends a basic response to a previous unsol sequence abort
* event after aborting the sequence handling.
@@ -17800,7 +17957,7 @@
rc = lpfc_sli_issue_iocb(phba, LPFC_ELS_RING, ctiocb, 0);
if (rc == IOCB_ERROR) {
- lpfc_printf_vlog(vport, KERN_ERR, LOG_ELS,
+ lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
"2925 Failed to issue CT ABTS RSP x%x on "
"xri x%x, Data x%x\n",
icmd->un.xseq64.w5.hcsw.Rctl, oxid,
@@ -17820,7 +17977,7 @@
* receive sequence is only partially assembed by the driver, it shall abort
* the partially assembled frames for the sequence. Otherwise, if the
* unsolicited receive sequence has been completely assembled and passed to
- * the Upper Layer Protocol (UPL), it then mark the per oxid status for the
+ * the Upper Layer Protocol (ULP), it then mark the per oxid status for the
* unsolicited sequence has been aborted. After that, it will issue a basic
* accept to accept the abort.
**/
@@ -17907,7 +18064,7 @@
/**
* lpfc_prep_seq - Prep sequence for ULP processing
* @vport: Pointer to the vport on which this sequence was received
- * @dmabuf: pointer to a dmabuf that describes the FC sequence
+ * @seq_dmabuf: pointer to a dmabuf that describes the FC sequence
*
* This function takes a sequence, described by a list of frames, and creates
* a list of iocbq structures to describe the sequence. This iocbq list will be
@@ -18050,7 +18207,7 @@
fc_hdr = (struct fc_frame_header *)seq_dmabuf->hbuf.virt;
iocbq = lpfc_prep_seq(vport, seq_dmabuf);
if (!iocbq) {
- lpfc_printf_log(phba, KERN_ERR, LOG_SLI,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"2707 Ring %d handler: Failed to allocate "
"iocb Rctl x%x Type x%x received\n",
LPFC_ELS_RING,
@@ -18061,7 +18218,7 @@
phba->sli4_hba.els_wq->pring,
iocbq, fc_hdr->fh_r_ctl,
fc_hdr->fh_type))
- lpfc_printf_log(phba, KERN_ERR, LOG_SLI,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"2540 Ring %d handler: unexpected Rctl "
"x%x Type x%x received\n",
LPFC_ELS_RING,
@@ -18180,6 +18337,7 @@
/**
* lpfc_sli4_handle_received_buffer - Handle received buffers from firmware
* @phba: Pointer to HBA context object.
+ * @dmabuf: Pointer to a dmabuf that describes the FC sequence.
*
* This function is called with no lock held. This function processes all
* the received buffers and gives it to upper layers when a received buffer
@@ -18326,7 +18484,7 @@
rc = lpfc_sli4_post_rpi_hdr(phba, rpi_page);
if (rc != MBX_SUCCESS) {
- lpfc_printf_log(phba, KERN_ERR, LOG_SLI,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"2008 Error %d posting all rpi "
"headers\n", rc);
rc = -EIO;
@@ -18372,7 +18530,7 @@
/* The port is notified of the header region via a mailbox command. */
mboxq = (LPFC_MBOXQ_t *) mempool_alloc(phba->mbox_mem_pool, GFP_KERNEL);
if (!mboxq) {
- lpfc_printf_log(phba, KERN_ERR, LOG_SLI,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"2001 Unable to allocate memory for issuing "
"SLI_CONFIG_SPECIAL mailbox command\n");
return -ENOMEM;
@@ -18402,7 +18560,7 @@
if (rc != MBX_TIMEOUT)
mempool_free(mboxq, phba->mbox_mem_pool);
if (shdr_status || shdr_add_status || rc) {
- lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"2514 POST_RPI_HDR mailbox failed with "
"status x%x add_status x%x, mbx status x%x\n",
shdr_status, shdr_add_status, rc);
@@ -18492,7 +18650,7 @@
if (rpi_remaining < LPFC_RPI_LOW_WATER_MARK) {
rpi_hdr = lpfc_sli4_create_rpi_hdr(phba);
if (!rpi_hdr) {
- lpfc_printf_log(phba, KERN_ERR, LOG_SLI,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"2002 Error Could not grow rpi "
"count\n");
} else {
@@ -18508,6 +18666,7 @@
/**
* lpfc_sli4_free_rpi - Release an rpi for reuse.
* @phba: pointer to lpfc hba data structure.
+ * @rpi: rpi to free
*
* This routine is invoked to release an rpi to the pool of
* available rpis maintained by the driver.
@@ -18536,6 +18695,7 @@
/**
* lpfc_sli4_free_rpi - Release an rpi for reuse.
* @phba: pointer to lpfc hba data structure.
+ * @rpi: rpi to free
*
* This routine is invoked to release an rpi to the pool of
* available rpis maintained by the driver.
@@ -18565,7 +18725,9 @@
/**
* lpfc_sli4_resume_rpi - Remove the rpi bitmask region
- * @phba: pointer to lpfc hba data structure.
+ * @ndlp: pointer to lpfc nodelist data structure.
+ * @cmpl: completion call-back.
+ * @arg: data to load as MBox 'caller buffer information'
*
* This routine is invoked to remove the memory region that
* provided rpi via a bitmask.
@@ -18594,7 +18756,7 @@
mboxq->vport = ndlp->vport;
rc = lpfc_sli_issue_mbox(phba, mboxq, MBX_NOWAIT);
if (rc == MBX_NOT_FINISHED) {
- lpfc_printf_log(phba, KERN_ERR, LOG_SLI,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"2010 Resume RPI Mailbox failed "
"status %d, mbxStatus x%x\n", rc,
bf_get(lpfc_mqe_status, &mboxq->u.mqe));
@@ -18629,7 +18791,7 @@
mbox_tmo = lpfc_mbox_tmo_val(phba, mboxq);
rc = lpfc_sli_issue_mbox_wait(phba, mboxq, mbox_tmo);
if (rc != MBX_SUCCESS) {
- lpfc_printf_vlog(vport, KERN_ERR, LOG_SLI,
+ lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
"2022 INIT VPI Mailbox failed "
"status %d, mbxStatus x%x\n", rc,
bf_get(lpfc_mqe_status, &mboxq->u.mqe));
@@ -18665,7 +18827,7 @@
if ((shdr_status || shdr_add_status) &&
(shdr_status != STATUS_FCF_IN_USE))
- lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"2558 ADD_FCF_RECORD mailbox failed with "
"status x%x add_status x%x\n",
shdr_status, shdr_add_status);
@@ -18695,7 +18857,7 @@
mboxq = mempool_alloc(phba->mbox_mem_pool, GFP_KERNEL);
if (!mboxq) {
- lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"2009 Failed to allocate mbox for ADD_FCF cmd\n");
return -ENOMEM;
}
@@ -18708,7 +18870,7 @@
LPFC_MBOX_OPCODE_FCOE_ADD_FCF,
req_len, LPFC_SLI4_MBX_NEMBED);
if (alloc_len < req_len) {
- lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"2523 Allocated DMA memory size (x%x) is "
"less than the requested DMA memory "
"size (x%x)\n", alloc_len, req_len);
@@ -18741,7 +18903,7 @@
mboxq->mbox_cmpl = lpfc_mbx_cmpl_add_fcf_record;
rc = lpfc_sli_issue_mbox(phba, mboxq, MBX_NOWAIT);
if (rc == MBX_NOT_FINISHED) {
- lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"2515 ADD_FCF_RECORD mailbox failed with "
"status 0x%x\n", rc);
lpfc_sli4_mbox_cmd_free(phba, mboxq);
@@ -18814,7 +18976,7 @@
phba->fcoe_cvl_eventtag_attn = phba->fcoe_cvl_eventtag;
mboxq = mempool_alloc(phba->mbox_mem_pool, GFP_KERNEL);
if (!mboxq) {
- lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"2000 Failed to allocate mbox for "
"READ_FCF cmd\n");
error = -ENOMEM;
@@ -18949,7 +19111,7 @@
/**
* lpfc_check_next_fcf_pri_level
- * phba pointer to the lpfc_hba struct for this port.
+ * @phba: pointer to the lpfc_hba struct for this port.
* This routine is called from the lpfc_sli4_fcf_rr_next_index_get
* routine when the rr_bmask is empty. The FCF indecies are put into the
* rr_bmask based on their priority level. Starting from the highest priority
@@ -19114,6 +19276,7 @@
/**
* lpfc_sli4_fcf_rr_index_set - Set bmask with eligible fcf record index
* @phba: pointer to lpfc hba data structure.
+ * @fcf_index: index into the FCF table to 'set'
*
* This routine sets the FCF record index in to the eligible bmask for
* roundrobin failover search. It checks to make sure that the index
@@ -19146,6 +19309,7 @@
/**
* lpfc_sli4_fcf_rr_index_clear - Clear bmask from eligible fcf record index
* @phba: pointer to lpfc hba data structure.
+ * @fcf_index: index into the FCF table to 'clear'
*
* This routine clears the FCF record index from the eligible bmask for
* roundrobin failover search. It checks to make sure that the index
@@ -19183,6 +19347,7 @@
/**
* lpfc_mbx_cmpl_redisc_fcf_table - completion routine for rediscover FCF table
* @phba: pointer to lpfc hba data structure.
+ * @mbox: An allocated pointer to type LPFC_MBOXQ_t
*
* This routine is the completion routine for the rediscover FCF table mailbox
* command. If the mailbox command returned failure, it will try to stop the
@@ -19257,7 +19422,7 @@
mbox = mempool_alloc(phba->mbox_mem_pool, GFP_KERNEL);
if (!mbox) {
- lpfc_printf_log(phba, KERN_ERR, LOG_SLI,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"2745 Failed to allocate mbox for "
"requesting FCF rediscover.\n");
return -ENOMEM;
@@ -19325,14 +19490,14 @@
LPFC_MBOXQ_t *pmb = NULL;
MAILBOX_t *mb;
uint32_t offset = 0;
- int rc;
+ int i, rc;
if (!rgn23_data)
return 0;
pmb = mempool_alloc(phba->mbox_mem_pool, GFP_KERNEL);
if (!pmb) {
- lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"2600 failed to allocate mailbox memory\n");
return 0;
}
@@ -19355,14 +19520,14 @@
*/
if (mb->un.varDmp.word_cnt == 0)
break;
- if (mb->un.varDmp.word_cnt > DMP_RGN23_SIZE - offset)
- mb->un.varDmp.word_cnt = DMP_RGN23_SIZE - offset;
+ i = mb->un.varDmp.word_cnt * sizeof(uint32_t);
+ if (offset + i > DMP_RGN23_SIZE)
+ i = DMP_RGN23_SIZE - offset;
lpfc_sli_pcimem_bcopy(((uint8_t *)mb) + DMP_RSP_OFFSET,
- rgn23_data + offset,
- mb->un.varDmp.word_cnt);
- offset += mb->un.varDmp.word_cnt;
- } while (mb->un.varDmp.word_cnt && offset < DMP_RGN23_SIZE);
+ rgn23_data + offset, i);
+ offset += i;
+ } while (offset < DMP_RGN23_SIZE);
mempool_free(pmb, phba->mbox_mem_pool);
return offset;
@@ -19391,7 +19556,7 @@
mboxq = mempool_alloc(phba->mbox_mem_pool, GFP_KERNEL);
if (!mboxq) {
- lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"3105 failed to allocate mailbox memory\n");
return 0;
}
@@ -19455,7 +19620,7 @@
/* Check the region signature first */
if (memcmp(&rgn23_data[offset], LPFC_REGION23_SIGNATURE, 4)) {
- lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"2619 Config region 23 has bad signature\n");
goto out;
}
@@ -19463,7 +19628,7 @@
/* Check the data structure version */
if (rgn23_data[offset] != LPFC_REGION23_VERSION) {
- lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"2620 Config region 23 has bad version\n");
goto out;
}
@@ -19640,7 +19805,7 @@
if (rc != MBX_TIMEOUT)
mempool_free(mbox, phba->mbox_mem_pool);
if (shdr_status || shdr_add_status || rc) {
- lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"3025 Write Object mailbox failed with "
"status x%x add_status x%x, mbx status x%x\n",
shdr_status, shdr_add_status, rc);
@@ -19820,7 +19985,7 @@
piocbq = lpfc_sli_ringtx_get(phba, pring);
if (!piocbq) {
spin_unlock_irqrestore(&pring->ring_lock, iflags);
- lpfc_printf_log(phba, KERN_ERR, LOG_SLI,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"2823 txq empty and txq_cnt is %d\n ",
txq_cnt);
break;
@@ -19849,7 +20014,7 @@
if (fail_msg) {
/* Failed means we can't issue and need to cancel */
- lpfc_printf_log(phba, KERN_ERR, LOG_SLI,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"2822 IOCB failed %s iotag 0x%x "
"xri 0x%x\n",
fail_msg,
@@ -19869,7 +20034,7 @@
/**
* lpfc_wqe_bpl2sgl - Convert the bpl/bde to a sgl.
* @phba: Pointer to HBA context object.
- * @pwqe: Pointer to command WQE.
+ * @pwqeq: Pointer to command WQE.
* @sglq: Pointer to the scatter gather queue object.
*
* This routine converts the bpl or bde that is in the WQE
@@ -19994,7 +20159,7 @@
/**
* lpfc_sli4_issue_wqe - Issue an SLI4 Work Queue Entry (WQE)
* @phba: Pointer to HBA context object.
- * @ring_number: Base sli ring number
+ * @qp: Pointer to HDW queue.
* @pwqe: Pointer to command WQE.
**/
int
@@ -20276,6 +20441,7 @@
/**
* _lpfc_move_xri_pbl_to_pvt - Move some XRIs from public to private pool
* @phba: pointer to lpfc hba data structure
+ * @qp: pointer to HDW queue
* @pbl_pool: specified public free XRI pool
* @pvt_pool: specified private free XRI pool
* @count: number of XRIs to move
@@ -20405,7 +20571,7 @@
/**
* lpfc_keep_pvt_pool_above_lowwm - Keep pvt_pool above low watermark
* @phba: pointer to lpfc hba data structure.
- * @qp: belong to which HWQ.
+ * @hwqid: belong to which HWQ.
*
* This routine get a batch of XRIs from pbl_pool if pvt_pool is less than
* low watermark.
@@ -20527,6 +20693,7 @@
/**
* lpfc_get_io_buf_from_private_pool - Get one free IO buf from private pool
* @phba: pointer to lpfc hba data structure.
+ * @qp: pointer to HDW queue
* @pvt_pool: pointer to private pool data structure.
* @ndlp: pointer to lpfc nodelist data structure.
*
diff --git a/drivers/scsi/lpfc/lpfc_sli4.h b/drivers/scsi/lpfc/lpfc_sli4.h
index 4decb53..a966cde 100644
--- a/drivers/scsi/lpfc/lpfc_sli4.h
+++ b/drivers/scsi/lpfc/lpfc_sli4.h
@@ -20,6 +20,9 @@
* included with this package. *
*******************************************************************/
+#include <linux/irq_poll.h>
+#include <linux/cpufreq.h>
+
#if defined(CONFIG_DEBUG_FS) && !defined(CONFIG_SCSI_LPFC_DEBUG_FS)
#define CONFIG_SCSI_LPFC_DEBUG_FS
#endif
@@ -135,6 +138,16 @@
struct rqb_dmabuf *);
};
+enum lpfc_poll_mode {
+ LPFC_QUEUE_WORK,
+ LPFC_IRQ_POLL
+};
+
+struct lpfc_idle_stat {
+ u64 prev_idle;
+ u64 prev_wall;
+};
+
struct lpfc_queue {
struct list_head list;
struct list_head wq_list;
@@ -265,6 +278,10 @@
struct lpfc_queue *assoc_qp;
struct list_head _poll_list;
void **q_pgs; /* array to index entries per page */
+
+#define LPFC_IRQ_POLL_WEIGHT 256
+ struct irq_poll iop;
+ enum lpfc_poll_mode poll_mode;
};
struct lpfc_sli4_link {
@@ -926,6 +943,7 @@
#ifdef CONFIG_SCSI_LPFC_DEBUG_FS
struct lpfc_hdwq_stat __percpu *c_stat;
#endif
+ struct lpfc_idle_stat *idle_stat;
uint32_t conf_trunk;
#define lpfc_conf_trunk_port0_WORD conf_trunk
#define lpfc_conf_trunk_port0_SHIFT 0
diff --git a/drivers/scsi/lpfc/lpfc_version.h b/drivers/scsi/lpfc/lpfc_version.h
index ab0bc26..1987c66 100644
--- a/drivers/scsi/lpfc/lpfc_version.h
+++ b/drivers/scsi/lpfc/lpfc_version.h
@@ -20,7 +20,7 @@
* included with this package. *
*******************************************************************/
-#define LPFC_DRIVER_VERSION "12.8.0.1"
+#define LPFC_DRIVER_VERSION "12.8.0.2"
#define LPFC_DRIVER_NAME "lpfc"
/* Used for SLI 2/3 */
diff --git a/drivers/scsi/lpfc/lpfc_vport.c b/drivers/scsi/lpfc/lpfc_vport.c
index b766463..aa4e451 100644
--- a/drivers/scsi/lpfc/lpfc_vport.c
+++ b/drivers/scsi/lpfc/lpfc_vport.c
@@ -145,7 +145,7 @@
rc = lpfc_sli_issue_mbox_wait(phba, pmb, phba->fc_ratov * 2);
if (rc != MBX_SUCCESS) {
if (signal_pending(current)) {
- lpfc_printf_vlog(vport, KERN_ERR, LOG_INIT | LOG_VPORT,
+ lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
"1830 Signal aborted mbxCmd x%x\n",
mb->mbxCommand);
lpfc_mbuf_free(phba, mp->virt, mp->phys);
@@ -154,7 +154,7 @@
mempool_free(pmb, phba->mbox_mem_pool);
return -EINTR;
} else {
- lpfc_printf_vlog(vport, KERN_ERR, LOG_INIT | LOG_VPORT,
+ lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
"1818 VPort failed init, mbxCmd x%x "
"READ_SPARM mbxStatus x%x, rc = x%x\n",
mb->mbxCommand, mb->mbxStatus, rc);
@@ -190,7 +190,7 @@
((wwn->u.wwn[0] & 0xf) != 0 || (wwn->u.wwn[1] & 0xf) != 0)))
return 1;
- lpfc_printf_log(phba, KERN_ERR, LOG_VPORT,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"1822 Invalid %s: %02x:%02x:%02x:%02x:"
"%02x:%02x:%02x:%02x\n",
name_type,
@@ -284,11 +284,11 @@
}
if (time_after(jiffies, wait_time_max))
- lpfc_printf_vlog(vport, KERN_ERR, LOG_VPORT,
- "1835 Vport discovery quiesce failed:"
- " state x%x fc_flags x%x wait msecs x%x\n",
- vport->port_state, vport->fc_flag,
- jiffies_to_msecs(jiffies - start_time));
+ lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
+ "1835 Vport discovery quiesce failed:"
+ " state x%x fc_flags x%x wait msecs x%x\n",
+ vport->port_state, vport->fc_flag,
+ jiffies_to_msecs(jiffies - start_time));
}
int
@@ -305,7 +305,7 @@
int status;
if ((phba->sli_rev < 3) || !(phba->cfg_enable_npiv)) {
- lpfc_printf_log(phba, KERN_ERR, LOG_VPORT,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"1808 Create VPORT failed: "
"NPIV is not enabled: SLImode:%d\n",
phba->sli_rev);
@@ -315,7 +315,7 @@
/* NPIV is not supported if HBA has NVME Target enabled */
if (phba->nvmet_support) {
- lpfc_printf_log(phba, KERN_ERR, LOG_VPORT,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"3189 Create VPORT failed: "
"NPIV is not supported on NVME Target\n");
rc = VPORT_INVAL;
@@ -324,7 +324,7 @@
vpi = lpfc_alloc_vpi(phba);
if (vpi == 0) {
- lpfc_printf_log(phba, KERN_ERR, LOG_VPORT,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"1809 Create VPORT failed: "
"Max VPORTs (%d) exceeded\n",
phba->max_vpi);
@@ -334,7 +334,7 @@
/* Assign an unused board number */
if ((instance = lpfc_get_instance()) < 0) {
- lpfc_printf_log(phba, KERN_ERR, LOG_VPORT,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"1810 Create VPORT failed: Cannot get "
"instance number\n");
lpfc_free_vpi(phba, vpi);
@@ -344,7 +344,7 @@
vport = lpfc_create_port(phba, instance, &fc_vport->dev);
if (!vport) {
- lpfc_printf_log(phba, KERN_ERR, LOG_VPORT,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"1811 Create VPORT failed: vpi x%x\n", vpi);
lpfc_free_vpi(phba, vpi);
rc = VPORT_NORESOURCES;
@@ -356,11 +356,11 @@
if ((status = lpfc_vport_sparm(phba, vport))) {
if (status == -EINTR) {
- lpfc_printf_vlog(vport, KERN_ERR, LOG_VPORT,
+ lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
"1831 Create VPORT Interrupted.\n");
rc = VPORT_ERROR;
} else {
- lpfc_printf_vlog(vport, KERN_ERR, LOG_VPORT,
+ lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
"1813 Create VPORT failed. "
"Cannot get sparam\n");
rc = VPORT_NORESOURCES;
@@ -378,7 +378,7 @@
if (!lpfc_valid_wwn_format(phba, &vport->fc_sparam.nodeName, "WWNN") ||
!lpfc_valid_wwn_format(phba, &vport->fc_sparam.portName, "WWPN")) {
- lpfc_printf_vlog(vport, KERN_ERR, LOG_VPORT,
+ lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
"1821 Create VPORT failed. "
"Invalid WWN format\n");
lpfc_free_vpi(phba, vpi);
@@ -388,7 +388,7 @@
}
if (!lpfc_unique_wwpn(phba, vport)) {
- lpfc_printf_vlog(vport, KERN_ERR, LOG_VPORT,
+ lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
"1823 Create VPORT failed. "
"Duplicate WWN on HBA\n");
lpfc_free_vpi(phba, vpi);
@@ -426,7 +426,7 @@
(pport->fc_flag & FC_VFI_REGISTERED)) {
rc = lpfc_sli4_init_vpi(vport);
if (rc) {
- lpfc_printf_log(phba, KERN_ERR, LOG_VPORT,
+ lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
"1838 Failed to INIT_VPI on vpi %d "
"status %d\n", vpi, rc);
rc = VPORT_NORESOURCES;
@@ -469,7 +469,7 @@
lpfc_initial_fdisc(vport);
} else {
lpfc_vport_set_state(vport, FC_VPORT_NO_FABRIC_SUPP);
- lpfc_printf_vlog(vport, KERN_ERR, LOG_ELS,
+ lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
"0262 No NPIV Fabric support\n");
}
} else {
@@ -478,8 +478,8 @@
rc = VPORT_OK;
out:
- lpfc_printf_vlog(vport, KERN_ERR, LOG_VPORT,
- "1825 Vport Created.\n");
+ lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
+ "1825 Vport Created.\n");
lpfc_host_attrib_init(lpfc_shost_from_vport(vport));
error_out:
return rc;
@@ -534,7 +534,7 @@
}
lpfc_vport_set_state(vport, FC_VPORT_DISABLED);
- lpfc_printf_vlog(vport, KERN_ERR, LOG_VPORT,
+ lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
"1826 Vport Disabled.\n");
return VPORT_OK;
}
@@ -575,7 +575,7 @@
lpfc_initial_fdisc(vport);
} else {
lpfc_vport_set_state(vport, FC_VPORT_NO_FABRIC_SUPP);
- lpfc_printf_vlog(vport, KERN_ERR, LOG_ELS,
+ lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
"0264 No NPIV Fabric support\n");
}
} else {
@@ -583,7 +583,7 @@
}
out:
- lpfc_printf_vlog(vport, KERN_ERR, LOG_VPORT,
+ lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
"1827 Vport Enabled.\n");
return VPORT_OK;
}
@@ -609,7 +609,7 @@
bool ns_ndlp_referenced = false;
if (vport->port_type == LPFC_PHYSICAL_PORT) {
- lpfc_printf_vlog(vport, KERN_ERR, LOG_VPORT,
+ lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
"1812 vport_delete failed: Cannot delete "
"physical host\n");
return VPORT_ERROR;
@@ -618,7 +618,7 @@
/* If the vport is a static vport fail the deletion. */
if ((vport->vport_flag & STATIC_VPORT) &&
!(phba->pport->load_flag & FC_UNLOADING)) {
- lpfc_printf_vlog(vport, KERN_ERR, LOG_VPORT,
+ lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
"1837 vport_delete failed: Cannot delete "
"static vport.\n");
return VPORT_ERROR;
@@ -642,27 +642,16 @@
vport->port_state < LPFC_VPORT_READY)
return -EAGAIN;
}
+
/*
- * This is a bit of a mess. We want to ensure the shost doesn't get
- * torn down until we're done with the embedded lpfc_vport structure.
- *
- * Beyond holding a reference for this function, we also need a
- * reference for outstanding I/O requests we schedule during delete
- * processing. But once we scsi_remove_host() we can no longer obtain
- * a reference through scsi_host_get().
- *
- * So we take two references here. We release one reference at the
- * bottom of the function -- after delinking the vport. And we
- * release the other at the completion of the unreg_vpi that get's
- * initiated after we've disposed of all other resources associated
- * with the port.
+ * Take early refcount for outstanding I/O requests we schedule during
+ * delete processing for unreg_vpi. Always keep this before
+ * scsi_remove_host() as we can no longer obtain a reference through
+ * scsi_host_get() after scsi_host_remove as shost is set to SHOST_DEL.
*/
if (!scsi_host_get(shost))
return VPORT_INVAL;
- if (!scsi_host_get(shost)) {
- scsi_host_put(shost);
- return VPORT_INVAL;
- }
+
lpfc_free_sysfs_attr(vport);
lpfc_debugfs_terminate(vport);
@@ -809,15 +798,16 @@
if (!(vport->vpi_state & LPFC_VPI_REGISTERED) ||
lpfc_mbx_unreg_vpi(vport))
scsi_host_put(shost);
- } else
+ } else {
scsi_host_put(shost);
+ }
lpfc_free_vpi(phba, vport->vpi);
vport->work_port_events = 0;
spin_lock_irq(&phba->port_list_lock);
list_del_init(&vport->listentry);
spin_unlock_irq(&phba->port_list_lock);
- lpfc_printf_vlog(vport, KERN_ERR, LOG_VPORT,
+ lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
"1828 Vport Deleted.\n");
scsi_host_put(shost);
return VPORT_OK;
@@ -838,7 +828,8 @@
if (port_iterator->load_flag & FC_UNLOADING)
continue;
if (!scsi_host_get(lpfc_shost_from_vport(port_iterator))) {
- lpfc_printf_vlog(port_iterator, KERN_ERR, LOG_VPORT,
+ lpfc_printf_vlog(port_iterator, KERN_ERR,
+ LOG_TRACE_EVENT,
"1801 Create vport work array FAILED: "
"cannot do scsi_host_get\n");
continue;
@@ -908,7 +899,8 @@
GFP_ATOMIC);
if (!ndlp->lat_data)
- lpfc_printf_vlog(vport, KERN_ERR, LOG_NODE,
+ lpfc_printf_vlog(vport, KERN_ERR,
+ LOG_TRACE_EVENT,
"0287 lpfc_alloc_bucket failed to "
"allocate statistical data buffer DID "
"0x%x\n", ndlp->nlp_DID);
diff --git a/drivers/scsi/megaraid.c b/drivers/scsi/megaraid.c
index f27ffd0..0484ee5 100644
--- a/drivers/scsi/megaraid.c
+++ b/drivers/scsi/megaraid.c
@@ -124,7 +124,7 @@
/**
* mega_setup_mailbox()
- * @adapter - pointer to our soft state
+ * @adapter: pointer to our soft state
*
* Allocates a 8 byte aligned memory for the handshake mailbox.
*/
@@ -347,7 +347,7 @@
/**
* mega_runpendq()
- * @adapter - pointer to our soft state
+ * @adapter: pointer to our soft state
*
* Runs through the list of pending requests.
*/
@@ -413,8 +413,8 @@
/**
* mega_allocate_scb()
- * @adapter - pointer to our soft state
- * @cmd - scsi command from the mid-layer
+ * @adapter: pointer to our soft state
+ * @cmd: scsi command from the mid-layer
*
* Allocate a SCB structure. This is the central structure for controller
* commands.
@@ -444,9 +444,9 @@
/**
* mega_get_ldrv_num()
- * @adapter - pointer to our soft state
- * @cmd - scsi mid layer command
- * @channel - channel on the controller
+ * @adapter: pointer to our soft state
+ * @cmd: scsi mid layer command
+ * @channel: channel on the controller
*
* Calculate the logical drive number based on the information in scsi command
* and the channel number.
@@ -503,9 +503,9 @@
/**
* mega_build_cmd()
- * @adapter - pointer to our soft state
- * @cmd - Prepare using this scsi command
- * @busy - busy flag if no resources
+ * @adapter: pointer to our soft state
+ * @cmd: Prepare using this scsi command
+ * @busy: busy flag if no resources
*
* Prepares a command and scatter gather list for the controller. This routine
* also finds out if the commands is intended for a logical drive or a
@@ -517,7 +517,6 @@
static scb_t *
mega_build_cmd(adapter_t *adapter, struct scsi_cmnd *cmd, int *busy)
{
- mega_ext_passthru *epthru;
mega_passthru *pthru;
scb_t *scb;
mbox_t *mbox;
@@ -905,7 +904,7 @@
if( adapter->support_ext_cdb ) {
- epthru = mega_prepare_extpassthru(adapter, scb, cmd,
+ mega_prepare_extpassthru(adapter, scb, cmd,
channel, target);
mbox->m_out.cmd = MEGA_MBOXCMD_EXTPTHRU;
@@ -937,11 +936,11 @@
/**
* mega_prepare_passthru()
- * @adapter - pointer to our soft state
- * @scb - our scsi control block
- * @cmd - scsi command from the mid-layer
- * @channel - actual channel on the controller
- * @target - actual id on the controller.
+ * @adapter: pointer to our soft state
+ * @scb: our scsi control block
+ * @cmd: scsi command from the mid-layer
+ * @channel: actual channel on the controller
+ * @target: actual id on the controller.
*
* prepare a command for the scsi physical devices.
*/
@@ -1000,11 +999,11 @@
/**
* mega_prepare_extpassthru()
- * @adapter - pointer to our soft state
- * @scb - our scsi control block
- * @cmd - scsi command from the mid-layer
- * @channel - actual channel on the controller
- * @target - actual id on the controller.
+ * @adapter: pointer to our soft state
+ * @scb: our scsi control block
+ * @cmd: scsi command from the mid-layer
+ * @channel: actual channel on the controller
+ * @target: actual id on the controller.
*
* prepare a command for the scsi physical devices. This rountine prepares
* commands for devices which can take extended CDBs (>10 bytes)
@@ -1085,8 +1084,8 @@
/**
* issue_scb()
- * @adapter - pointer to our soft state
- * @scb - scsi control block
+ * @adapter: pointer to our soft state
+ * @scb: scsi control block
*
* Post a command to the card if the mailbox is available, otherwise return
* busy. We also take the scb from the pending list if the mailbox is
@@ -1166,8 +1165,8 @@
/**
* issue_scb_block()
- * @adapter - pointer to our soft state
- * @raw_mbox - the mailbox
+ * @adapter: pointer to our soft state
+ * @raw_mbox: the mailbox
*
* Issue a scb in synchronous and non-interrupt mode
*/
@@ -1247,8 +1246,8 @@
/**
* megaraid_isr_iomapped()
- * @irq - irq
- * @devp - pointer to our soft state
+ * @irq: irq
+ * @devp: pointer to our soft state
*
* Interrupt service routine for io-mapped controllers.
* Find out if our device is interrupting. If yes, acknowledge the interrupt
@@ -1323,8 +1322,8 @@
/**
* megaraid_isr_memmapped()
- * @irq - irq
- * @devp - pointer to our soft state
+ * @irq: irq
+ * @devp: pointer to our soft state
*
* Interrupt service routine for memory-mapped controllers.
* Find out if our device is interrupting. If yes, acknowledge the interrupt
@@ -1401,10 +1400,10 @@
}
/**
* mega_cmd_done()
- * @adapter - pointer to our soft state
- * @completed - array of ids of completed commands
- * @nstatus - number of completed commands
- * @status - status of the last command completed
+ * @adapter: pointer to our soft state
+ * @completed: array of ids of completed commands
+ * @nstatus: number of completed commands
+ * @status: status of the last command completed
*
* Complete the commands and call the scsi mid-layer callback hooks.
*/
@@ -1921,9 +1920,9 @@
/**
* megaraid_abort_and_reset()
- * @adapter - megaraid soft state
- * @cmd - scsi command to be aborted or reset
- * @aor - abort or reset flag
+ * @adapter: megaraid soft state
+ * @cmd: scsi command to be aborted or reset
+ * @aor: abort or reset flag
*
* Try to locate the scsi command in the pending queue. If found and is not
* issued to the controller, abort/reset it. Otherwise return failure
@@ -2021,8 +2020,8 @@
/**
* mega_allocate_inquiry()
- * @dma_handle - handle returned for dma address
- * @pdev - handle to pci device
+ * @dma_handle: handle returned for dma address
+ * @pdev: handle to pci device
*
* allocates memory for inquiry structure
*/
@@ -2045,8 +2044,8 @@
/**
* proc_show_config()
- * @m - Synthetic file construction data
- * @v - File iterator
+ * @m: Synthetic file construction data
+ * @v: File iterator
*
* Display configuration information about the controller.
*/
@@ -2109,8 +2108,8 @@
/**
* proc_show_stat()
- * @m - Synthetic file construction data
- * @v - File iterator
+ * @m: Synthetic file construction data
+ * @v: File iterator
*
* Display statistical information about the I/O activity.
*/
@@ -2143,8 +2142,8 @@
/**
* proc_show_mbox()
- * @m - Synthetic file construction data
- * @v - File iterator
+ * @m: Synthetic file construction data
+ * @v: File iterator
*
* Display mailbox information for the last command issued. This information
* is good for debugging.
@@ -2171,8 +2170,8 @@
/**
* proc_show_rebuild_rate()
- * @m - Synthetic file construction data
- * @v - File iterator
+ * @m: Synthetic file construction data
+ * @v: File iterator
*
* Display current rebuild rate
*/
@@ -2214,8 +2213,8 @@
/**
* proc_show_battery()
- * @m - Synthetic file construction data
- * @v - File iterator
+ * @m: Synthetic file construction data
+ * @v: File iterator
*
* Display information about the battery module on the controller.
*/
@@ -2317,9 +2316,9 @@
/**
* proc_show_pdrv()
- * @m - Synthetic file construction data
- * @page - buffer to write the data in
- * @adapter - pointer to our soft state
+ * @m: Synthetic file construction data
+ * @adapter: pointer to our soft state
+ * @channel: channel
*
* Display information about the physical drives.
*/
@@ -2433,8 +2432,8 @@
/**
* proc_show_pdrv_ch0()
- * @m - Synthetic file construction data
- * @v - File iterator
+ * @m: Synthetic file construction data
+ * @v: File iterator
*
* Display information about the physical drives on physical channel 0.
*/
@@ -2447,8 +2446,8 @@
/**
* proc_show_pdrv_ch1()
- * @m - Synthetic file construction data
- * @v - File iterator
+ * @m: Synthetic file construction data
+ * @v: File iterator
*
* Display information about the physical drives on physical channel 1.
*/
@@ -2461,8 +2460,8 @@
/**
* proc_show_pdrv_ch2()
- * @m - Synthetic file construction data
- * @v - File iterator
+ * @m: Synthetic file construction data
+ * @v: File iterator
*
* Display information about the physical drives on physical channel 2.
*/
@@ -2475,8 +2474,8 @@
/**
* proc_show_pdrv_ch3()
- * @m - Synthetic file construction data
- * @v - File iterator
+ * @m: Synthetic file construction data
+ * @v: File iterator
*
* Display information about the physical drives on physical channel 3.
*/
@@ -2489,10 +2488,10 @@
/**
* proc_show_rdrv()
- * @m - Synthetic file construction data
- * @adapter - pointer to our soft state
- * @start - starting logical drive to display
- * @end - ending logical drive to display
+ * @m: Synthetic file construction data
+ * @adapter: pointer to our soft state
+ * @start: starting logical drive to display
+ * @end: ending logical drive to display
*
* We do not print the inquiry information since its already available through
* /proc/scsi/scsi interface
@@ -2674,8 +2673,8 @@
/**
* proc_show_rdrv_10()
- * @m - Synthetic file construction data
- * @v - File iterator
+ * @m: Synthetic file construction data
+ * @v: File iterator
*
* Display real time information about the logical drives 0 through 9.
*/
@@ -2688,8 +2687,8 @@
/**
* proc_show_rdrv_20()
- * @m - Synthetic file construction data
- * @v - File iterator
+ * @m: Synthetic file construction data
+ * @v: File iterator
*
* Display real time information about the logical drives 0 through 9.
*/
@@ -2702,8 +2701,8 @@
/**
* proc_show_rdrv_30()
- * @m - Synthetic file construction data
- * @v - File iterator
+ * @m: Synthetic file construction data
+ * @v: File iterator
*
* Display real time information about the logical drives 0 through 9.
*/
@@ -2716,8 +2715,8 @@
/**
* proc_show_rdrv_40()
- * @m - Synthetic file construction data
- * @v - File iterator
+ * @m: Synthetic file construction data
+ * @v: File iterator
*
* Display real time information about the logical drives 0 through 9.
*/
@@ -2729,8 +2728,8 @@
/**
* mega_create_proc_entry()
- * @index - index in soft state array
- * @parent - parent node for this /proc entry
+ * @index: index in soft state array
+ * @parent: parent node for this /proc entry
*
* Creates /proc entries for our controllers.
*/
@@ -2785,7 +2784,7 @@
#endif
-/**
+/*
* megaraid_biosparam()
*
* Return the disk geometry for a particular disk
@@ -2854,7 +2853,7 @@
/**
* mega_init_scb()
- * @adapter - pointer to our soft state
+ * @adapter: pointer to our soft state
*
* Allocate memory for the various pointers in the scb structures:
* scatter-gather list pointer, passthru and extended passthru structure
@@ -2934,8 +2933,8 @@
/**
* megadev_open()
- * @inode - unused
- * @filep - unused
+ * @inode: unused
+ * @filep: unused
*
* Routines for the character/ioctl interface to the driver. Find out if this
* is a valid open.
@@ -2954,10 +2953,9 @@
/**
* megadev_ioctl()
- * @inode - Our device inode
- * @filep - unused
- * @cmd - ioctl command
- * @arg - user buffer
+ * @filep: Our device file
+ * @cmd: ioctl command
+ * @arg: user buffer
*
* ioctl entry point for our private ioctl interface. We move the data in from
* the user space, prepare the command (if necessary, convert the old MIMD
@@ -2977,14 +2975,13 @@
void *data = NULL; /* data to be transferred */
dma_addr_t data_dma_hndl; /* dma handle for data xfer area */
megacmd_t mc;
- megastat_t __user *ustats;
- int num_ldrv;
+#if MEGA_HAVE_STATS
+ megastat_t __user *ustats = NULL;
+ int num_ldrv = 0;
+#endif
u32 uxferaddr = 0;
struct pci_dev *pdev;
- ustats = NULL; /* avoid compilation warnings */
- num_ldrv = 0;
-
/*
* Make sure only USCSICMD are issued through this interface.
* MIMD application would still fire different command.
@@ -3370,8 +3367,8 @@
/**
* mega_m_to_n()
- * @arg - user address
- * @uioc - new ioctl structure
+ * @arg: user address
+ * @uioc: new ioctl structure
*
* A thin layer to convert older mimd interface ioctl structure to NIT ioctl
* structure
@@ -3498,8 +3495,8 @@
/*
* mega_n_to_m()
- * @arg - user address
- * @mc - mailbox command
+ * @arg: user address
+ * @mc: mailbox command
*
* Updates the status information to the application, depending on application
* conforms to older mimd ioctl interface or newer NIT ioctl interface
@@ -3565,7 +3562,7 @@
/**
* mega_is_bios_enabled()
- * @adapter - pointer to our soft state
+ * @adapter: pointer to our soft state
*
* issue command to find out if the BIOS is enabled for this controller
*/
@@ -3574,7 +3571,6 @@
{
unsigned char raw_mbox[sizeof(struct mbox_out)];
mbox_t *mbox;
- int ret;
mbox = (mbox_t *)raw_mbox;
@@ -3587,8 +3583,7 @@
raw_mbox[0] = IS_BIOS_ENABLED;
raw_mbox[2] = GET_BIOS;
-
- ret = issue_scb_block(adapter, raw_mbox);
+ issue_scb_block(adapter, raw_mbox);
return *(char *)adapter->mega_buffer;
}
@@ -3596,7 +3591,7 @@
/**
* mega_enum_raid_scsi()
- * @adapter - pointer to our soft state
+ * @adapter: pointer to our soft state
*
* Find out what channels are RAID/SCSI. This information is used to
* differentiate the virtual channels and physical channels and to support
@@ -3651,7 +3646,7 @@
/**
* mega_get_boot_drv()
- * @adapter - pointer to our soft state
+ * @adapter: pointer to our soft state
*
* Find out which device is the boot device. Note, any logical drive or any
* phyical device (e.g., a CDROM) can be designated as a boot device.
@@ -3718,7 +3713,7 @@
/**
* mega_support_random_del()
- * @adapter - pointer to our soft state
+ * @adapter: pointer to our soft state
*
* Find out if this controller supports random deletion and addition of
* logical drives
@@ -3748,7 +3743,7 @@
/**
* mega_support_ext_cdb()
- * @adapter - pointer to our soft state
+ * @adapter: pointer to our soft state
*
* Find out if this firmware support cdblen > 10
*/
@@ -3776,8 +3771,8 @@
/**
* mega_del_logdrv()
- * @adapter - pointer to our soft state
- * @logdrv - logical drive to be deleted
+ * @adapter: pointer to our soft state
+ * @logdrv: logical drive to be deleted
*
* Delete the specified logical drive. It is the responsibility of the user
* app to let the OS know about this operation.
@@ -3862,7 +3857,7 @@
/**
* mega_get_max_sgl()
- * @adapter - pointer to our soft state
+ * @adapter: pointer to our soft state
*
* Find out the maximum number of scatter-gather elements supported by this
* version of the firmware
@@ -3908,7 +3903,7 @@
/**
* mega_support_cluster()
- * @adapter - pointer to our soft state
+ * @adapter: pointer to our soft state
*
* Find out if this firmware support cluster calls.
*/
@@ -3950,8 +3945,8 @@
#ifdef CONFIG_PROC_FS
/**
* mega_adapinq()
- * @adapter - pointer to our soft state
- * @dma_handle - DMA address of the buffer
+ * @adapter: pointer to our soft state
+ * @dma_handle: DMA address of the buffer
*
* Issue internal commands while interrupts are available.
* We only issue direct mailbox commands from within the driver. ioctl()
@@ -3983,11 +3978,12 @@
}
-/** mega_internal_dev_inquiry()
- * @adapter - pointer to our soft state
- * @ch - channel for this device
- * @tgt - ID of this device
- * @buf_dma_handle - DMA address of the buffer
+/**
+ * mega_internal_dev_inquiry()
+ * @adapter: pointer to our soft state
+ * @ch: channel for this device
+ * @tgt: ID of this device
+ * @buf_dma_handle: DMA address of the buffer
*
* Issue the scsi inquiry for the specified device.
*/
@@ -4056,9 +4052,9 @@
/**
* mega_internal_command()
- * @adapter - pointer to our soft state
- * @mc - the mailbox command
- * @pthru - Passthru structure for DCDB commands
+ * @adapter: pointer to our soft state
+ * @mc: the mailbox command
+ * @pthru: Passthru structure for DCDB commands
*
* Issue the internal commands in interrupt mode.
* The last argument is the address of the passthru structure if the command
diff --git a/drivers/scsi/megaraid/megaraid_mbox.c b/drivers/scsi/megaraid/megaraid_mbox.c
index 8f918df..19469a2 100644
--- a/drivers/scsi/megaraid/megaraid_mbox.c
+++ b/drivers/scsi/megaraid/megaraid_mbox.c
@@ -3304,7 +3304,6 @@
* megaraid_mbox_display_scb - display SCB information, mostly debug purposes
* @adapter : controller's soft state
* @scb : SCB to be displayed
- * @level : debug level for console print
*
* Diplay information about the given SCB iff the current debug level is
* verbose.
@@ -3972,7 +3971,8 @@
/**
* megaraid_sysfs_show_app_hndl - display application handle for this adapter
- * @cdev : class device object representation for the host
+ * @dev : class device object representation for the host
+ * @attr : device attribute (unused)
* @buf : buffer to send data to
*
* Display the handle used by the applications while executing management
diff --git a/drivers/scsi/megaraid/megaraid_mm.c b/drivers/scsi/megaraid/megaraid_mm.c
index e83163c..8df5344 100644
--- a/drivers/scsi/megaraid/megaraid_mm.c
+++ b/drivers/scsi/megaraid/megaraid_mm.c
@@ -95,7 +95,6 @@
/**
* mraid_mm_ioctl - module entry-point for ioctls
- * @inode : inode (ignored)
* @filep : file operations pointer (ignored)
* @cmd : ioctl command
* @arg : user ioctl packet
diff --git a/drivers/scsi/megaraid/megaraid_sas.h b/drivers/scsi/megaraid/megaraid_sas.h
index af2c7a2..5e4137f 100644
--- a/drivers/scsi/megaraid/megaraid_sas.h
+++ b/drivers/scsi/megaraid/megaraid_sas.h
@@ -2709,4 +2709,24 @@
int ocr_context);
int megasas_irqpoll(struct irq_poll *irqpoll, int budget);
void megasas_dump_fusion_io(struct scsi_cmnd *scmd);
+u32 megasas_readl(struct megasas_instance *instance,
+ const volatile void __iomem *addr);
+struct megasas_cmd *megasas_get_cmd(struct megasas_instance *instance);
+void megasas_return_cmd(struct megasas_instance *instance,
+ struct megasas_cmd *cmd);
+int megasas_issue_polled(struct megasas_instance *instance,
+ struct megasas_cmd *cmd);
+void megaraid_sas_kill_hba(struct megasas_instance *instance);
+void megasas_check_and_restore_queue_depth(struct megasas_instance *instance);
+void megasas_start_timer(struct megasas_instance *instance);
+int megasas_sriov_start_heartbeat(struct megasas_instance *instance,
+ int initial);
+int megasas_alloc_cmds(struct megasas_instance *instance);
+void megasas_free_cmds(struct megasas_instance *instance);
+
+void megasas_init_debugfs(void);
+void megasas_exit_debugfs(void);
+void megasas_setup_debugfs(struct megasas_instance *instance);
+void megasas_destroy_debugfs(struct megasas_instance *instance);
+
#endif /*LSI_MEGARAID_SAS_H */
diff --git a/drivers/scsi/megaraid/megaraid_sas_base.c b/drivers/scsi/megaraid/megaraid_sas_base.c
index 0066833..861f714 100644
--- a/drivers/scsi/megaraid/megaraid_sas_base.c
+++ b/drivers/scsi/megaraid/megaraid_sas_base.c
@@ -202,10 +202,6 @@
static spinlock_t poll_aen_lock;
extern struct dentry *megasas_debugfs_root;
-extern void megasas_init_debugfs(void);
-extern void megasas_exit_debugfs(void);
-extern void megasas_setup_debugfs(struct megasas_instance *instance);
-extern void megasas_destroy_debugfs(struct megasas_instance *instance);
void
megasas_complete_cmd(struct megasas_instance *instance, struct megasas_cmd *cmd,
@@ -427,14 +423,14 @@
evt_detail->description);
}
-/**
-* The following functions are defined for xscale
-* (deviceid : 1064R, PERC5) controllers
-*/
+/*
+ * The following functions are defined for xscale
+ * (deviceid : 1064R, PERC5) controllers
+ */
/**
* megasas_enable_intr_xscale - Enables interrupts
- * @regs: MFI register set
+ * @instance: Adapter soft state
*/
static inline void
megasas_enable_intr_xscale(struct megasas_instance *instance)
@@ -450,7 +446,7 @@
/**
* megasas_disable_intr_xscale -Disables interrupt
- * @regs: MFI register set
+ * @instance: Adapter soft state
*/
static inline void
megasas_disable_intr_xscale(struct megasas_instance *instance)
@@ -466,7 +462,7 @@
/**
* megasas_read_fw_status_reg_xscale - returns the current FW status value
- * @regs: MFI register set
+ * @instance: Adapter soft state
*/
static u32
megasas_read_fw_status_reg_xscale(struct megasas_instance *instance)
@@ -475,7 +471,7 @@
}
/**
* megasas_clear_interrupt_xscale - Check & clear interrupt
- * @regs: MFI register set
+ * @instance: Adapter soft state
*/
static int
megasas_clear_intr_xscale(struct megasas_instance *instance)
@@ -509,9 +505,10 @@
/**
* megasas_fire_cmd_xscale - Sends command to the FW
- * @frame_phys_addr : Physical address of cmd
- * @frame_count : Number of frames for the command
- * @regs : MFI register set
+ * @instance: Adapter soft state
+ * @frame_phys_addr : Physical address of cmd
+ * @frame_count : Number of frames for the command
+ * @regs : MFI register set
*/
static inline void
megasas_fire_cmd_xscale(struct megasas_instance *instance,
@@ -529,7 +526,8 @@
/**
* megasas_adp_reset_xscale - For controller reset
- * @regs: MFI register set
+ * @instance: Adapter soft state
+ * @regs: MFI register set
*/
static int
megasas_adp_reset_xscale(struct megasas_instance *instance,
@@ -570,7 +568,8 @@
/**
* megasas_check_reset_xscale - For controller reset check
- * @regs: MFI register set
+ * @instance: Adapter soft state
+ * @regs: MFI register set
*/
static int
megasas_check_reset_xscale(struct megasas_instance *instance,
@@ -599,19 +598,19 @@
.issue_dcmd = megasas_issue_dcmd,
};
-/**
-* This is the end of set of functions & definitions specific
-* to xscale (deviceid : 1064R, PERC5) controllers
-*/
+/*
+ * This is the end of set of functions & definitions specific
+ * to xscale (deviceid : 1064R, PERC5) controllers
+ */
-/**
-* The following functions are defined for ppc (deviceid : 0x60)
-* controllers
-*/
+/*
+ * The following functions are defined for ppc (deviceid : 0x60)
+ * controllers
+ */
/**
* megasas_enable_intr_ppc - Enables interrupts
- * @regs: MFI register set
+ * @instance: Adapter soft state
*/
static inline void
megasas_enable_intr_ppc(struct megasas_instance *instance)
@@ -629,7 +628,7 @@
/**
* megasas_disable_intr_ppc - Disable interrupt
- * @regs: MFI register set
+ * @instance: Adapter soft state
*/
static inline void
megasas_disable_intr_ppc(struct megasas_instance *instance)
@@ -645,7 +644,7 @@
/**
* megasas_read_fw_status_reg_ppc - returns the current FW status value
- * @regs: MFI register set
+ * @instance: Adapter soft state
*/
static u32
megasas_read_fw_status_reg_ppc(struct megasas_instance *instance)
@@ -655,7 +654,7 @@
/**
* megasas_clear_interrupt_ppc - Check & clear interrupt
- * @regs: MFI register set
+ * @instance: Adapter soft state
*/
static int
megasas_clear_intr_ppc(struct megasas_instance *instance)
@@ -688,9 +687,10 @@
/**
* megasas_fire_cmd_ppc - Sends command to the FW
- * @frame_phys_addr : Physical address of cmd
- * @frame_count : Number of frames for the command
- * @regs : MFI register set
+ * @instance: Adapter soft state
+ * @frame_phys_addr: Physical address of cmd
+ * @frame_count: Number of frames for the command
+ * @regs: MFI register set
*/
static inline void
megasas_fire_cmd_ppc(struct megasas_instance *instance,
@@ -708,7 +708,8 @@
/**
* megasas_check_reset_ppc - For controller reset check
- * @regs: MFI register set
+ * @instance: Adapter soft state
+ * @regs: MFI register set
*/
static int
megasas_check_reset_ppc(struct megasas_instance *instance,
@@ -738,7 +739,7 @@
/**
* megasas_enable_intr_skinny - Enables interrupts
- * @regs: MFI register set
+ * @instance: Adapter soft state
*/
static inline void
megasas_enable_intr_skinny(struct megasas_instance *instance)
@@ -756,7 +757,7 @@
/**
* megasas_disable_intr_skinny - Disables interrupt
- * @regs: MFI register set
+ * @instance: Adapter soft state
*/
static inline void
megasas_disable_intr_skinny(struct megasas_instance *instance)
@@ -772,7 +773,7 @@
/**
* megasas_read_fw_status_reg_skinny - returns the current FW status value
- * @regs: MFI register set
+ * @instance: Adapter soft state
*/
static u32
megasas_read_fw_status_reg_skinny(struct megasas_instance *instance)
@@ -782,7 +783,7 @@
/**
* megasas_clear_interrupt_skinny - Check & clear interrupt
- * @regs: MFI register set
+ * @instance: Adapter soft state
*/
static int
megasas_clear_intr_skinny(struct megasas_instance *instance)
@@ -825,9 +826,10 @@
/**
* megasas_fire_cmd_skinny - Sends command to the FW
- * @frame_phys_addr : Physical address of cmd
- * @frame_count : Number of frames for the command
- * @regs : MFI register set
+ * @instance: Adapter soft state
+ * @frame_phys_addr: Physical address of cmd
+ * @frame_count: Number of frames for the command
+ * @regs: MFI register set
*/
static inline void
megasas_fire_cmd_skinny(struct megasas_instance *instance,
@@ -847,7 +849,8 @@
/**
* megasas_check_reset_skinny - For controller reset check
- * @regs: MFI register set
+ * @instance: Adapter soft state
+ * @regs: MFI register set
*/
static int
megasas_check_reset_skinny(struct megasas_instance *instance,
@@ -876,14 +879,14 @@
};
-/**
-* The following functions are defined for gen2 (deviceid : 0x78 0x79)
-* controllers
-*/
+/*
+ * The following functions are defined for gen2 (deviceid : 0x78 0x79)
+ * controllers
+ */
/**
* megasas_enable_intr_gen2 - Enables interrupts
- * @regs: MFI register set
+ * @instance: Adapter soft state
*/
static inline void
megasas_enable_intr_gen2(struct megasas_instance *instance)
@@ -902,7 +905,7 @@
/**
* megasas_disable_intr_gen2 - Disables interrupt
- * @regs: MFI register set
+ * @instance: Adapter soft state
*/
static inline void
megasas_disable_intr_gen2(struct megasas_instance *instance)
@@ -918,7 +921,7 @@
/**
* megasas_read_fw_status_reg_gen2 - returns the current FW status value
- * @regs: MFI register set
+ * @instance: Adapter soft state
*/
static u32
megasas_read_fw_status_reg_gen2(struct megasas_instance *instance)
@@ -928,7 +931,7 @@
/**
* megasas_clear_interrupt_gen2 - Check & clear interrupt
- * @regs: MFI register set
+ * @instance: Adapter soft state
*/
static int
megasas_clear_intr_gen2(struct megasas_instance *instance)
@@ -961,11 +964,13 @@
return mfiStatus;
}
+
/**
* megasas_fire_cmd_gen2 - Sends command to the FW
- * @frame_phys_addr : Physical address of cmd
- * @frame_count : Number of frames for the command
- * @regs : MFI register set
+ * @instance: Adapter soft state
+ * @frame_phys_addr: Physical address of cmd
+ * @frame_count: Number of frames for the command
+ * @regs: MFI register set
*/
static inline void
megasas_fire_cmd_gen2(struct megasas_instance *instance,
@@ -983,7 +988,8 @@
/**
* megasas_adp_reset_gen2 - For controller reset
- * @regs: MFI register set
+ * @instance: Adapter soft state
+ * @reg_set: MFI register set
*/
static int
megasas_adp_reset_gen2(struct megasas_instance *instance,
@@ -1043,7 +1049,8 @@
/**
* megasas_check_reset_gen2 - For controller reset check
- * @regs: MFI register set
+ * @instance: Adapter soft state
+ * @regs: MFI register set
*/
static int
megasas_check_reset_gen2(struct megasas_instance *instance,
@@ -1071,10 +1078,10 @@
.issue_dcmd = megasas_issue_dcmd,
};
-/**
-* This is the end of set of functions & definitions
-* specific to gen2 (deviceid : 0x78, 0x79) controllers
-*/
+/*
+ * This is the end of set of functions & definitions
+ * specific to gen2 (deviceid : 0x78, 0x79) controllers
+ */
/*
* Template added for TB (Fusion)
@@ -1609,7 +1616,7 @@
/**
* megasas_cmd_type - Checks if the cmd is for logical drive/sysPD
* and whether it's RW or non RW
- * @scmd: SCSI command
+ * @cmd: SCSI command
*
*/
inline int megasas_cmd_type(struct scsi_cmnd *cmd)
@@ -1749,8 +1756,8 @@
/**
* megasas_queue_command - Queue entry point
+ * @shost: adapter SCSI host
* @scmd: SCSI command to be queued
- * @done: Callback entry point
*/
static int
megasas_queue_command(struct Scsi_Host *shost, struct scsi_cmnd *scmd)
@@ -2916,11 +2923,7 @@
/**
* megasas_dump_reg_set - This function will print hexdump of register set
- * @buf: Buffer to be dumped
- * @sz: Size in bytes
- * @format: Different formats of dumping e.g. format=n will
- * cause only 'n' 32 bit words to be dumped in a
- * single line.
+ * @reg_set: Register set to be dumped
*/
inline void
megasas_dump_reg_set(void __iomem *reg_set)
@@ -2997,6 +3000,7 @@
/**
* megasas_reset_bus_host - Bus & host reset handler entry point
+ * @scmd: Mid-layer SCSI command
*/
static int megasas_reset_bus_host(struct scsi_cmnd *scmd)
{
@@ -3777,7 +3781,7 @@
megasas_register_aen(instance, seq_num, class_locale.word);
}
-/**
+/*
* Move the internal reset pending commands to a deferred queue.
*
* We move the commands pending at internal reset time to a
@@ -3785,7 +3789,7 @@
* completion of the internal reset sequence. if the internal reset
* did not complete in time, the kernel reset handler would flush
* these commands.
- **/
+ */
static void
megasas_internal_reset_defer_cmds(struct megasas_instance *instance)
{
@@ -3963,8 +3967,11 @@
tasklet_schedule(&instance->isr_tasklet);
return IRQ_HANDLED;
}
+
/**
* megasas_isr - isr entry point
+ * @irq: IRQ number
+ * @devp: IRQ context address
*/
static irqreturn_t megasas_isr(int irq, void *devp)
{
@@ -3986,6 +3993,7 @@
/**
* megasas_transition_to_ready - Move the FW to READY state
* @instance: Adapter soft state
+ * @ocr: Adapter reset state
*
* During the initialization, FW passes can potentially be in any one of
* several possible states. If the FW in operational, waiting-for-handshake
@@ -4743,7 +4751,7 @@
/**
* megasas_ld_list_query - Returns FW's ld_list structure
* @instance: Adapter soft state
- * @ld_list: ld_list structure
+ * @query_type: ld_list structure type
*
* Issues an internal command (DCMD) to get the FW's controller PD
* list structure. This information is mainly used to find out SYSTEM
@@ -5602,9 +5610,13 @@
&instance->irq_context[i])) {
dev_err(&instance->pdev->dev,
"Failed to register IRQ for vector %d.\n", i);
- for (j = 0; j < i; j++)
+ for (j = 0; j < i; j++) {
+ if (j < instance->low_latency_index_start)
+ irq_set_affinity_hint(
+ pci_irq_vector(pdev, j), NULL);
free_irq(pci_irq_vector(pdev, j),
&instance->irq_context[j]);
+ }
/* Retry irq register for IO_APIC*/
instance->msix_vectors = 0;
instance->msix_load_balance = false;
@@ -5642,6 +5654,9 @@
if (instance->msix_vectors)
for (i = 0; i < instance->msix_vectors; i++) {
+ if (i < instance->low_latency_index_start)
+ irq_set_affinity_hint(
+ pci_irq_vector(instance->pdev, i), NULL);
free_irq(pci_irq_vector(instance->pdev, i),
&instance->irq_context[i]);
}
@@ -5653,7 +5668,6 @@
/**
* megasas_setup_jbod_map - setup jbod map for FP seq_number.
* @instance: Adapter soft state
- * @is_probe: Driver probe check
*
* Return 0 on success.
*/
@@ -6494,7 +6508,7 @@
* megasas_register_aen - Registers for asynchronous event notification
* @instance: Adapter soft state
* @seq_num: The starting sequence number
- * @class_locale: Class of the event
+ * @class_locale_word: Class of the event
*
* This function subscribes for AEN for events beyond the @seq_num. It requests
* to be notified if and only if the event is of type @class_locale
@@ -7014,8 +7028,9 @@
* megasas_alloc_ctrl_dma_buffers - Allocate consistent DMA buffers during
* driver load time
*
- * @instance- Adapter soft instance
- * @return- O for SUCCESS
+ * @instance: Adapter soft instance
+ *
+ * @return: O for SUCCESS
*/
static inline
int megasas_alloc_ctrl_dma_buffers(struct megasas_instance *instance)
@@ -7931,7 +7946,7 @@
/**
* megasas_shutdown - Shutdown entry point
- * @device: Generic device structure
+ * @pdev: Generic device structure
*/
static void megasas_shutdown(struct pci_dev *pdev)
{
@@ -7956,8 +7971,10 @@
pci_free_irq_vectors(instance->pdev);
}
-/**
+/*
* megasas_mgmt_open - char node "open" entry point
+ * @inode: char node inode
+ * @filep: char node file
*/
static int megasas_mgmt_open(struct inode *inode, struct file *filep)
{
@@ -7970,8 +7987,11 @@
return 0;
}
-/**
+/*
* megasas_mgmt_fasync - Async notifier registration from applications
+ * @fd: char node file descriptor number
+ * @filep: char node file
+ * @mode: notifier on/off
*
* This function adds the calling process to a driver global queue. When an
* event occurs, SIGIO will be sent to all processes in this queue.
@@ -7997,9 +8017,11 @@
return rc;
}
-/**
+/*
* megasas_mgmt_poll - char node "poll" entry point
- * */
+ * @filep: char node file
+ * @wait: Events to poll for
+ */
static __poll_t megasas_mgmt_poll(struct file *file, poll_table *wait)
{
__poll_t mask;
@@ -8057,7 +8079,8 @@
/**
* megasas_mgmt_fw_ioctl - Issues management ioctls to FW
* @instance: Adapter soft state
- * @argp: User's ioctl packet
+ * @user_ioc: User's ioctl packet
+ * @ioc: ioctl packet
*/
static int
megasas_mgmt_fw_ioctl(struct megasas_instance *instance,
@@ -8397,6 +8420,9 @@
/**
* megasas_mgmt_ioctl - char node ioctl entry point
+ * @file: char device file pointer
+ * @cmd: ioctl command
+ * @arg: ioctl command arguments address
*/
static long
megasas_mgmt_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
diff --git a/drivers/scsi/megaraid/megaraid_sas_fp.c b/drivers/scsi/megaraid/megaraid_sas_fp.c
index 3b3d04d..b6c08d6 100644
--- a/drivers/scsi/megaraid/megaraid_sas_fp.c
+++ b/drivers/scsi/megaraid/megaraid_sas_fp.c
@@ -80,21 +80,20 @@
}
/**
- * @param dividend : Dividend
- * @param divisor : Divisor
+ * mega_div64_32 - Do a 64-bit division
+ * @dividend: Dividend
+ * @divisor: Divisor
*
* @return quotient
**/
static u64 mega_div64_32(uint64_t dividend, uint32_t divisor)
{
- u32 remainder;
- u64 d;
+ u64 d = dividend;
if (!divisor)
printk(KERN_ERR "megasas : DIVISOR is zero in mod fn\n");
- d = dividend;
- remainder = do_div(d, divisor);
+ do_div(d, divisor);
return d;
}
diff --git a/drivers/scsi/megaraid/megaraid_sas_fusion.c b/drivers/scsi/megaraid/megaraid_sas_fusion.c
index fcf03f7..0824410 100644
--- a/drivers/scsi/megaraid/megaraid_sas_fusion.c
+++ b/drivers/scsi/megaraid/megaraid_sas_fusion.c
@@ -48,9 +48,6 @@
#include "megaraid_sas.h"
-extern void megasas_free_cmds(struct megasas_instance *instance);
-extern struct megasas_cmd *megasas_get_cmd(struct megasas_instance
- *instance);
extern void
megasas_complete_cmd(struct megasas_instance *instance,
struct megasas_cmd *cmd, u8 alt_status);
@@ -58,24 +55,14 @@
wait_and_poll(struct megasas_instance *instance, struct megasas_cmd *cmd,
int seconds);
-void
-megasas_return_cmd(struct megasas_instance *instance, struct megasas_cmd *cmd);
-int megasas_alloc_cmds(struct megasas_instance *instance);
int
megasas_clear_intr_fusion(struct megasas_instance *instance);
-int
-megasas_issue_polled(struct megasas_instance *instance,
- struct megasas_cmd *cmd);
-void
-megasas_check_and_restore_queue_depth(struct megasas_instance *instance);
int megasas_transition_to_ready(struct megasas_instance *instance, int ocr);
-void megaraid_sas_kill_hba(struct megasas_instance *instance);
extern u32 megasas_dbg_lvl;
int megasas_sriov_start_heartbeat(struct megasas_instance *instance,
int initial);
-void megasas_start_timer(struct megasas_instance *instance);
extern struct megasas_mgmt_info megasas_mgmt_info;
extern unsigned int resetwaittime;
extern unsigned int dual_qdepth_disable;
@@ -84,15 +71,13 @@
static inline
void megasas_configure_queue_sizes(struct megasas_instance *instance);
static void megasas_fusion_crash_dump(struct megasas_instance *instance);
-extern u32 megasas_readl(struct megasas_instance *instance,
- const volatile void __iomem *addr);
/**
* megasas_adp_reset_wait_for_ready - initiate chip reset and wait for
* controller to come to ready state
- * @instance - adapter's soft state
- * @do_adp_reset - If true, do a chip reset
- * @ocr_context - If called from OCR context this will
+ * @instance: adapter's soft state
+ * @do_adp_reset: If true, do a chip reset
+ * @ocr_context: If called from OCR context this will
* be set to 1, else 0
*
* This function initates a chip reset followed by a wait for controller to
@@ -146,10 +131,10 @@
/**
* megasas_check_same_4gb_region - check if allocation
* crosses same 4GB boundary or not
- * @instance - adapter's soft instance
- * start_addr - start address of DMA allocation
- * size - size of allocation in bytes
- * return - true : allocation does not cross same
+ * @instance: adapter's soft instance
+ * @start_addr: start address of DMA allocation
+ * @size: size of allocation in bytes
+ * @return: true : allocation does not cross same
* 4GB boundary
* false: allocation crosses same
* 4GB boundary
@@ -174,7 +159,7 @@
/**
* megasas_enable_intr_fusion - Enables interrupts
- * @regs: MFI register set
+ * @instance: adapter's soft instance
*/
static void
megasas_enable_intr_fusion(struct megasas_instance *instance)
@@ -196,7 +181,7 @@
/**
* megasas_disable_intr_fusion - Disables interrupt
- * @regs: MFI register set
+ * @instance: adapter's soft instance
*/
static void
megasas_disable_intr_fusion(struct megasas_instance *instance)
@@ -238,6 +223,7 @@
/**
* megasas_get_cmd_fusion - Get a command from the free pool
* @instance: Adapter soft state
+ * @blk_tag: Command tag
*
* Returns a blk_tag indexed mpt frame
*/
@@ -309,8 +295,8 @@
/**
* megasas_fusion_update_can_queue - Do all Adapter Queue depth related calculations here
- * @instance: Adapter soft state
- * fw_boot_context: Whether this function called during probe or after OCR
+ * @instance: Adapter soft state
+ * @fw_boot_context: Whether this function called during probe or after OCR
*
* This function is only for fusion controllers.
* Update host can queue, if firmware downgrade max supported firmware commands.
@@ -1016,6 +1002,7 @@
* wait_and_poll - Issues a polling command
* @instance: Adapter soft state
* @cmd: Command packet to be issued
+ * @seconds: Maximum poll time
*
* For polling, MFI requires the cmd_status to be set to 0xFF before posting.
*/
@@ -1906,6 +1893,7 @@
/**
* megasas_fault_detect_work - Worker function of
* FW fault handling workqueue.
+ * @work: FW fault work struct
*/
static void
megasas_fault_detect_work(struct work_struct *work)
@@ -1989,11 +1977,13 @@
/**
* map_cmd_status - Maps FW cmd status to OS cmd status
- * @cmd : Pointer to cmd
- * @status : status of cmd returned by FW
- * @ext_status : ext status of cmd returned by FW
+ * @fusion: fusion context
+ * @scmd: Pointer to cmd
+ * @status: status of cmd returned by FW
+ * @ext_status: ext status of cmd returned by FW
+ * @data_length: command data length
+ * @sense: command sense data
*/
-
static void
map_cmd_status(struct fusion_context *fusion,
struct scsi_cmnd *scmd, u8 status, u8 ext_status,
@@ -2234,7 +2224,7 @@
* @scp: SCSI command from the mid-layer
* @sgl_ptr: SGL to be filled in
* @cmd: cmd we are working on
- * @sge_count sge count
+ * @sge_count: sge count
*
*/
static void
@@ -2343,9 +2333,12 @@
/**
* megasas_set_pd_lba - Sets PD LBA
- * @cdb: CDB
+ * @io_request: IO request
* @cdb_len: cdb length
- * @start_blk: Start block of IO
+ * @io_info: IO information
+ * @scp: SCSI command
+ * @local_map_ptr: Raid map
+ * @ref_tag: Primary reference tag
*
* Used to set the PD LBA in CDB for FP IOs
*/
@@ -2603,10 +2596,12 @@
* affinity (cpu of the controller) and raid_flags in the raid context
* based on IO type.
*
+ * @fusion: Fusion context
* @praid_context: IO RAID context
* @raid: LD raid map
* @fp_possible: Is fast path possible?
* @is_read: Is read IO?
+ * @scsi_buff_len: SCSI command buffer length
*
*/
static void
@@ -2940,7 +2935,7 @@
/**
* megasas_build_ld_nonrw_fusion - prepares non rw ios for virtual disk
* @instance: Adapter soft state
- * @scp: SCSI command
+ * @scmd: SCSI command
* @cmd: Command to be prepared
*
* Prepares the io_request frame for non-rw io cmds for vd.
@@ -3028,7 +3023,7 @@
/**
* megasas_build_syspd_fusion - prepares rw/non-rw ios for syspd
* @instance: Adapter soft state
- * @scp: SCSI command
+ * @scmd: SCSI command
* @cmd: Command to be prepared
* @fp_possible: parameter to detect fast path or firmware path io.
*
@@ -3405,7 +3400,7 @@
* megasas_complete_r1_command -
* completes R1 FP write commands which has valid peer smid
* @instance: Adapter soft state
- * @cmd_fusion: MPT command frame
+ * @cmd: MPT command frame
*
*/
static inline void
@@ -3459,6 +3454,9 @@
/**
* complete_cmd_fusion - Completes command
* @instance: Adapter soft state
+ * @MSIxIndex: MSI number
+ * @irq_context: IRQ context
+ *
* Completes all commands that is in reply descriptor queue
*/
static int
@@ -3634,6 +3632,7 @@
/**
* megasas_enable_irq_poll() - enable irqpoll
+ * @instance: Adapter soft state
*/
static void megasas_enable_irq_poll(struct megasas_instance *instance)
{
@@ -3650,7 +3649,7 @@
/**
* megasas_sync_irqs - Synchronizes all IRQs owned by adapter
- * @instance: Adapter soft state
+ * @instance_addr: Adapter soft state address
*/
static void megasas_sync_irqs(unsigned long instance_addr)
{
@@ -3706,7 +3705,7 @@
/**
* megasas_complete_cmd_dpc_fusion - Completes command
- * @instance: Adapter soft state
+ * @instance_addr: Adapter soft state address
*
* Tasklet to complete cmds
*/
@@ -3729,6 +3728,8 @@
/**
* megasas_isr_fusion - isr entry point
+ * @irq: IRQ number
+ * @devp: IRQ context
*/
static irqreturn_t megasas_isr_fusion(int irq, void *devp)
{
@@ -3761,7 +3762,7 @@
/**
* build_mpt_mfi_pass_thru - builds a cmd fo MFI Pass thru
* @instance: Adapter soft state
- * mfi_cmd: megasas_cmd pointer
+ * @mfi_cmd: megasas_cmd pointer
*
*/
static void
@@ -3878,7 +3879,7 @@
/**
* megasas_read_fw_status_reg_fusion - returns the current FW status value
- * @regs: MFI register set
+ * @instance: Adapter soft state
*/
static u32
megasas_read_fw_status_reg_fusion(struct megasas_instance *instance)
@@ -3889,7 +3890,7 @@
/**
* megasas_alloc_host_crash_buffer - Host buffers for Crash dump collection from Firmware
* @instance: Controller's soft instance
- * return: Number of allocated host crash buffers
+ * @return: Number of allocated host crash buffers
*/
static void
megasas_alloc_host_crash_buffer(struct megasas_instance *instance)
@@ -3927,6 +3928,7 @@
/**
* megasas_adp_reset_fusion - For controller reset
+ * @instance: Controller's soft instance
* @regs: MFI register set
*/
static int
@@ -4004,6 +4006,7 @@
/**
* megasas_check_reset_fusion - For controller reset check
+ * @instance: Controller's soft instance
* @regs: MFI register set
*/
static int
@@ -4333,8 +4336,8 @@
/**
* megasas_tm_response_code - translation of device response code
- * @ioc: per adapter object
- * @mpi_reply: MPI reply returned by firmware
+ * @instance: Controller's soft instance
+ * @mpi_reply: MPI reply returned by firmware
*
* Return nothing.
*/
@@ -4389,9 +4392,9 @@
* @device_handle: device handle
* @channel: the channel assigned by the OS
* @id: the id assigned by the OS
- * @type: MPI2_SCSITASKMGMT_TASKTYPE__XXX (defined in megaraid_sas_fusion.c)
* @smid_task: smid assigned to the task
- * @m_type: TM_MUTEX_ON or TM_MUTEX_OFF
+ * @type: MPI2_SCSITASKMGMT_TASKTYPE__XXX (defined in megaraid_sas_fusion.c)
+ * @mr_device_priv_data: private data
* Context: user
*
* MegaRaid use MPT interface for Task Magement request.
diff --git a/drivers/scsi/mesh.c b/drivers/scsi/mesh.c
index f9f8f49..fd1d030 100644
--- a/drivers/scsi/mesh.c
+++ b/drivers/scsi/mesh.c
@@ -1045,6 +1045,8 @@
while ((in_8(&mr->bus_status1) & BS1_RST) != 0)
udelay(1);
printk("done\n");
+ if (ms->dma_started)
+ halt_dma(ms);
handle_reset(ms);
/* request_q is empty, no point in mesh_start() */
return;
@@ -1357,7 +1359,8 @@
ms->conn_tgt, ms->data_ptr, scsi_bufflen(cmd),
ms->tgts[ms->conn_tgt].data_goes_out);
}
- scsi_dma_unmap(cmd);
+ if (cmd)
+ scsi_dma_unmap(cmd);
ms->dma_started = 0;
}
@@ -1712,6 +1715,9 @@
spin_lock_irqsave(ms->host->host_lock, flags);
+ if (ms->dma_started)
+ halt_dma(ms);
+
/* Reset the controller & dbdma channel */
out_le32(&md->control, (RUN|PAUSE|FLUSH|WAKE) << 16); /* stop dma */
out_8(&mr->exception, 0xff); /* clear all exception bits */
diff --git a/drivers/scsi/mpt3sas/mpt3sas_base.c b/drivers/scsi/mpt3sas/mpt3sas_base.c
index 96b78fd..1d64524 100644
--- a/drivers/scsi/mpt3sas/mpt3sas_base.c
+++ b/drivers/scsi/mpt3sas/mpt3sas_base.c
@@ -190,7 +190,7 @@
/**
* _base_readl_aero - retry readl for max three times.
- * @addr - MPT Fusion system interface register address
+ * @addr: MPT Fusion system interface register address
*
* Retry the readl() for max three times if it gets zero value
* while reading the system interface register.
@@ -817,6 +817,7 @@
* mpt3sas_base_wait_for_coredump_completion - Wait until coredump
* completes or times out
* @ioc: per adapter object
+ * @caller: caller function name
*
* Returns 0 for success, non-zero for failure.
*/
@@ -1718,8 +1719,8 @@
/**
* _base_irqpoll - IRQ poll callback handler
- * @irqpoll - irq_poll object
- * @budget - irq poll weight
+ * @irqpoll: irq_poll object
+ * @budget: irq poll weight
*
* returns number of reply descriptors processed
*/
@@ -3048,8 +3049,8 @@
/**
* _base_check_and_enable_high_iops_queues - enable high iops mode
- * @ ioc - per adapter object
- * @ hba_msix_vector_count - msix vectors supported by HBA
+ * @ioc: per adapter object
+ * @hba_msix_vector_count: msix vectors supported by HBA
*
* Enable high iops queues only if
* - HBA is a SEA/AERO controller and
@@ -5621,6 +5622,7 @@
* _base_wait_for_doorbell_int - waiting for controller interrupt(generated by
* a write to the doorbell)
* @ioc: per adapter object
+ * @timeout: timeout in seconds
*
* Return: 0 for success, non-zero for failure.
*
@@ -5833,7 +5835,7 @@
/**
* mpt3sas_wait_for_ioc - IOC's operational state is checked here.
* @ioc: per adapter object
- * @wait_count: timeout in seconds
+ * @timeout: timeout in seconds
*
* Return: Waits up to timeout seconds for the IOC to
* become operational. Returns 0 if IOC is present
diff --git a/drivers/scsi/mpt3sas/mpt3sas_base.h b/drivers/scsi/mpt3sas/mpt3sas_base.h
index 4fca393..4ff876c 100644
--- a/drivers/scsi/mpt3sas/mpt3sas_base.h
+++ b/drivers/scsi/mpt3sas/mpt3sas_base.h
@@ -1770,7 +1770,7 @@
void mpt3sas_process_trigger_data(struct MPT3SAS_ADAPTER *ioc,
struct SL_WH_TRIGGERS_EVENT_DATA_T *event_data);
void mpt3sas_trigger_master(struct MPT3SAS_ADAPTER *ioc,
- u32 tigger_bitmask);
+ u32 trigger_bitmask);
void mpt3sas_trigger_event(struct MPT3SAS_ADAPTER *ioc, u16 event,
u16 log_entry_qualifier);
void mpt3sas_trigger_scsi(struct MPT3SAS_ADAPTER *ioc, u8 sense_key,
diff --git a/drivers/scsi/mpt3sas/mpt3sas_config.c b/drivers/scsi/mpt3sas/mpt3sas_config.c
index 62ddf53..11026e0 100644
--- a/drivers/scsi/mpt3sas/mpt3sas_config.c
+++ b/drivers/scsi/mpt3sas/mpt3sas_config.c
@@ -303,7 +303,6 @@
u8 retry_count, issue_host_reset = 0;
struct config_request mem;
u32 ioc_status = UINT_MAX;
- u8 issue_reset = 0;
mutex_lock(&ioc->config_cmds.mutex);
if (ioc->config_cmds.status != MPT3_CMD_NOT_USED) {
@@ -386,9 +385,9 @@
if (!(ioc->logging_level & MPT_DEBUG_CONFIG))
_config_display_some_debug(ioc,
smid, "config_request", NULL);
- mpt3sas_check_cmd_timeout(ioc,
- ioc->config_cmds.status, mpi_request,
- sizeof(Mpi2ConfigRequest_t)/4, issue_reset);
+ ioc_err(ioc, "%s: command timeout\n", __func__);
+ mpt3sas_base_check_cmd_timeout(ioc, ioc->config_cmds.status,
+ mpi_request, sizeof(Mpi2ConfigRequest_t) / 4);
retry_count++;
if (ioc->config_cmds.smid == smid)
mpt3sas_base_free_smid(ioc, smid);
diff --git a/drivers/scsi/mpt3sas/mpt3sas_ctl.c b/drivers/scsi/mpt3sas/mpt3sas_ctl.c
index 983e568..4326030 100644
--- a/drivers/scsi/mpt3sas/mpt3sas_ctl.c
+++ b/drivers/scsi/mpt3sas/mpt3sas_ctl.c
@@ -3662,8 +3662,9 @@
/**
* drv_support_bitmap_show - driver supported feature bitmap
- * @cdev - pointer to embedded class device
- * @buf - the buffer returned
+ * @cdev: pointer to embedded class device
+ * @attr: unused
+ * @buf: the buffer returned
*
* A sysfs 'read-only' shost attribute.
*/
@@ -3680,8 +3681,9 @@
/**
* enable_sdev_max_qd_show - display whether sdev max qd is enabled/disabled
- * @cdev - pointer to embedded class device
- * @buf - the buffer returned
+ * @cdev: pointer to embedded class device
+ * @attr: unused
+ * @buf: the buffer returned
*
* A sysfs read/write shost attribute. This attribute is used to set the
* targets queue depth to HBA IO queue depth if this attribute is enabled.
@@ -3698,8 +3700,10 @@
/**
* enable_sdev_max_qd_store - Enable/disable sdev max qd
- * @cdev - pointer to embedded class device
- * @buf - the buffer returned
+ * @cdev: pointer to embedded class device
+ * @attr: unused
+ * @buf: the buffer returned
+ * @count: unused
*
* A sysfs read/write shost attribute. This attribute is used to set the
* targets queue depth to HBA IO queue depth if this attribute is enabled.
diff --git a/drivers/scsi/mpt3sas/mpt3sas_trigger_diag.h b/drivers/scsi/mpt3sas/mpt3sas_trigger_diag.h
index 6586a46..405eada 100644
--- a/drivers/scsi/mpt3sas/mpt3sas_trigger_diag.h
+++ b/drivers/scsi/mpt3sas/mpt3sas_trigger_diag.h
@@ -69,7 +69,7 @@
#define MASTER_TRIGGER_TASK_MANAGMENT (0x00000004)
#define MASTER_TRIGGER_DEVICE_REMOVAL (0x00000008)
-/* fake firmware event for tigger */
+/* fake firmware event for trigger */
#define MPI3_EVENT_DIAGNOSTIC_TRIGGER_FIRED (0x6E)
/**
diff --git a/drivers/scsi/mvsas/mv_init.c b/drivers/scsi/mvsas/mv_init.c
index b0de3bd..978f528 100644
--- a/drivers/scsi/mvsas/mv_init.c
+++ b/drivers/scsi/mvsas/mv_init.c
@@ -177,15 +177,16 @@
static irqreturn_t mvs_interrupt(int irq, void *opaque)
{
- u32 core_nr;
u32 stat;
struct mvs_info *mvi;
struct sas_ha_struct *sha = opaque;
#ifndef CONFIG_SCSI_MVSAS_TASKLET
u32 i;
-#endif
+ u32 core_nr;
core_nr = ((struct mvs_prv_info *)sha->lldd_ha)->n_host;
+#endif
+
mvi = ((struct mvs_prv_info *)sha->lldd_ha)->mvi[0];
if (unlikely(!mvi))
@@ -299,7 +300,7 @@
int mvs_ioremap(struct mvs_info *mvi, int bar, int bar_ex)
{
- unsigned long res_start, res_len, res_flag, res_flag_ex = 0;
+ unsigned long res_start, res_len, res_flag_ex = 0;
struct pci_dev *pdev = mvi->pdev;
if (bar_ex != -1) {
/*
@@ -327,7 +328,6 @@
goto err_out;
}
- res_flag = pci_resource_flags(pdev, bar);
mvi->regs = ioremap(res_start, res_len);
if (!mvi->regs) {
diff --git a/drivers/scsi/myrs.c b/drivers/scsi/myrs.c
index 5c56664..103803e 100644
--- a/drivers/scsi/myrs.c
+++ b/drivers/scsi/myrs.c
@@ -87,7 +87,7 @@
return NULL;
}
-/**
+/*
* myrs_reset_cmd - clears critical fields in struct myrs_cmdblk
*/
static inline void myrs_reset_cmd(struct myrs_cmdblk *cmd_blk)
@@ -98,7 +98,7 @@
cmd_blk->status = 0;
}
-/**
+/*
* myrs_qcmd - queues Command for DAC960 V2 Series Controllers.
*/
static void myrs_qcmd(struct myrs_hba *cs, struct myrs_cmdblk *cmd_blk)
@@ -122,7 +122,7 @@
cs->next_cmd_mbox = next_mbox;
}
-/**
+/*
* myrs_exec_cmd - executes V2 Command and waits for completion.
*/
static void myrs_exec_cmd(struct myrs_hba *cs,
@@ -140,7 +140,7 @@
wait_for_completion(&complete);
}
-/**
+/*
* myrs_report_progress - prints progress message
*/
static void myrs_report_progress(struct myrs_hba *cs, unsigned short ldev_num,
@@ -153,7 +153,7 @@
(100 * (int)(blocks >> 7)) / (int)(size >> 7));
}
-/**
+/*
* myrs_get_ctlr_info - executes a Controller Information IOCTL Command
*/
static unsigned char myrs_get_ctlr_info(struct myrs_hba *cs)
@@ -214,7 +214,7 @@
return status;
}
-/**
+/*
* myrs_get_ldev_info - executes a Logical Device Information IOCTL Command
*/
static unsigned char myrs_get_ldev_info(struct myrs_hba *cs,
@@ -301,7 +301,7 @@
return status;
}
-/**
+/*
* myrs_get_pdev_info - executes a "Read Physical Device Information" Command
*/
static unsigned char myrs_get_pdev_info(struct myrs_hba *cs,
@@ -345,7 +345,7 @@
return status;
}
-/**
+/*
* myrs_dev_op - executes a "Device Operation" Command
*/
static unsigned char myrs_dev_op(struct myrs_hba *cs,
@@ -369,7 +369,7 @@
return status;
}
-/**
+/*
* myrs_translate_pdev - translates a Physical Device Channel and
* TargetID into a Logical Device.
*/
@@ -414,7 +414,7 @@
return status;
}
-/**
+/*
* myrs_get_event - executes a Get Event Command
*/
static unsigned char myrs_get_event(struct myrs_hba *cs,
@@ -476,7 +476,7 @@
return status;
}
-/**
+/*
* myrs_enable_mmio_mbox - enables the Memory Mailbox Interface
*/
static bool myrs_enable_mmio_mbox(struct myrs_hba *cs,
@@ -577,7 +577,7 @@
return (status == MYRS_STATUS_SUCCESS);
}
-/**
+/*
* myrs_get_config - reads the Configuration Information
*/
static int myrs_get_config(struct myrs_hba *cs)
@@ -682,7 +682,7 @@
return 0;
}
-/**
+/*
* myrs_log_event - prints a Controller Event message
*/
static struct {
@@ -2338,11 +2338,11 @@
return NULL;
}
-/**
+/*
* myrs_err_status reports Controller BIOS Messages passed through
- the Error Status Register when the driver performs the BIOS handshaking.
- It returns true for fatal errors and false otherwise.
-*/
+ * the Error Status Register when the driver performs the BIOS handshaking.
+ * It returns true for fatal errors and false otherwise.
+ */
static bool myrs_err_status(struct myrs_hba *cs, unsigned char status,
unsigned char parm0, unsigned char parm1)
diff --git a/drivers/scsi/pcmcia/nsp_cs.c b/drivers/scsi/pcmcia/nsp_cs.c
index d79ce97..8655ff1 100644
--- a/drivers/scsi/pcmcia/nsp_cs.c
+++ b/drivers/scsi/pcmcia/nsp_cs.c
@@ -134,6 +134,7 @@
scsi_set_resid(SCpnt, scsi_get_resid(SCpnt) + residInc);
}
+__printf(4, 5)
static void nsp_cs_message(const char *func, int line, char *type, char *fmt, ...)
{
va_list args;
@@ -689,14 +690,14 @@
{
unsigned int base = SCpnt->device->host->io_port;
unsigned int count;
- unsigned int l, m, h, dummy;
+ unsigned int l, m, h;
nsp_index_write(base, POINTERCLR, POINTER_CLEAR | ACK_COUNTER);
l = nsp_index_read(base, TRANSFERCOUNT);
m = nsp_index_read(base, TRANSFERCOUNT);
h = nsp_index_read(base, TRANSFERCOUNT);
- dummy = nsp_index_read(base, TRANSFERCOUNT); /* required this! */
+ nsp_index_read(base, TRANSFERCOUNT); /* required this! */
count = (h << 16) | (m << 8) | (l << 0);
diff --git a/drivers/scsi/pm8001/pm8001_ctl.c b/drivers/scsi/pm8001/pm8001_ctl.c
index 3c9f427..77c805d 100644
--- a/drivers/scsi/pm8001/pm8001_ctl.c
+++ b/drivers/scsi/pm8001/pm8001_ctl.c
@@ -47,6 +47,7 @@
/**
* pm8001_ctl_mpi_interface_rev_show - MPI interface revision number
* @cdev: pointer to embedded class device
+ * @attr: device attribute (unused)
* @buf: the buffer returned
*
* A sysfs 'read-only' shost attribute.
@@ -72,6 +73,7 @@
/**
* controller_fatal_error_show - check controller is under fatal err
* @cdev: pointer to embedded class device
+ * @attr: device attribute (unused)
* @buf: the buffer returned
*
* A sysfs 'read only' shost attribute.
@@ -91,6 +93,7 @@
/**
* pm8001_ctl_fw_version_show - firmware version
* @cdev: pointer to embedded class device
+ * @attr: device attribute (unused)
* @buf: the buffer returned
*
* A sysfs 'read-only' shost attribute.
@@ -121,6 +124,7 @@
/**
* pm8001_ctl_ila_version_show - ila version
* @cdev: pointer to embedded class device
+ * @attr: device attribute (unused)
* @buf: the buffer returned
*
* A sysfs 'read-only' shost attribute.
@@ -146,6 +150,7 @@
/**
* pm8001_ctl_inactive_fw_version_show - Inacative firmware version number
* @cdev: pointer to embedded class device
+ * @attr: device attribute (unused)
* @buf: the buffer returned
*
* A sysfs 'read-only' shost attribute.
@@ -172,6 +177,7 @@
/**
* pm8001_ctl_max_out_io_show - max outstanding io supported
* @cdev: pointer to embedded class device
+ * @attr: device attribute (unused)
* @buf: the buffer returned
*
* A sysfs 'read-only' shost attribute.
@@ -195,6 +201,7 @@
/**
* pm8001_ctl_max_devices_show - max devices support
* @cdev: pointer to embedded class device
+ * @attr: device attribute (unused)
* @buf: the buffer returned
*
* A sysfs 'read-only' shost attribute.
@@ -221,6 +228,7 @@
* pm8001_ctl_max_sg_list_show - max sg list supported iff not 0.0 for no
* hardware limitation
* @cdev: pointer to embedded class device
+ * @attr: device attribute (unused)
* @buf: the buffer returned
*
* A sysfs 'read-only' shost attribute.
@@ -265,6 +273,7 @@
/**
* pm8001_ctl_sas_spec_support_show - sas spec supported
* @cdev: pointer to embedded class device
+ * @attr: device attribute (unused)
* @buf: the buffer returned
*
* A sysfs 'read-only' shost attribute.
@@ -292,6 +301,7 @@
/**
* pm8001_ctl_sas_address_show - sas address
* @cdev: pointer to embedded class device
+ * @attr: device attribute (unused)
* @buf: the buffer returned
*
* This is the controller sas address
@@ -313,6 +323,7 @@
/**
* pm8001_ctl_logging_level_show - logging level
* @cdev: pointer to embedded class device
+ * @attr: device attribute (unused)
* @buf: the buffer returned
*
* A sysfs 'read/write' shost attribute.
@@ -326,6 +337,7 @@
return snprintf(buf, PAGE_SIZE, "%08xh\n", pm8001_ha->logging_level);
}
+
static ssize_t pm8001_ctl_logging_level_store(struct device *cdev,
struct device_attribute *attr, const char *buf, size_t count)
{
@@ -346,6 +358,7 @@
/**
* pm8001_ctl_aap_log_show - aap1 event log
* @cdev: pointer to embedded class device
+ * @attr: device attribute (unused)
* @buf: the buffer returned
*
* A sysfs 'read-only' shost attribute.
@@ -382,6 +395,7 @@
/**
* pm8001_ctl_ib_queue_log_show - Out bound Queue log
* @cdev:pointer to embedded class device
+ * @attr: device attribute (unused)
* @buf: the buffer returned
* A sysfs 'read-only' shost attribute.
*/
@@ -414,6 +428,7 @@
/**
* pm8001_ctl_ob_queue_log_show - Out bound Queue log
* @cdev:pointer to embedded class device
+ * @attr: device attribute (unused)
* @buf: the buffer returned
* A sysfs 'read-only' shost attribute.
*/
@@ -446,6 +461,7 @@
/**
* pm8001_ctl_bios_version_show - Bios version Display
* @cdev:pointer to embedded class device
+ * @attr: device attribute (unused)
* @buf:the buffer returned
* A sysfs 'read-only' shost attribute.
*/
@@ -483,6 +499,7 @@
/**
* event_log_size_show - event log size
* @cdev: pointer to embedded class device
+ * @attr: device attribute (unused)
* @buf: the buffer returned
*
* A sysfs read shost attribute.
@@ -501,6 +518,7 @@
/**
* pm8001_ctl_aap_log_show - IOP event log
* @cdev: pointer to embedded class device
+ * @attr: device attribute (unused)
* @buf: the buffer returned
*
* A sysfs 'read-only' shost attribute.
@@ -538,6 +556,7 @@
/**
** pm8001_ctl_fatal_log_show - fatal error logging
** @cdev:pointer to embedded class device
+ ** @attr: device attribute
** @buf: the buffer returned
**
** A sysfs 'read-only' shost attribute.
@@ -557,6 +576,7 @@
/**
** non_fatal_log_show - non fatal error logging
** @cdev:pointer to embedded class device
+ ** @attr: device attribute
** @buf: the buffer returned
**
** A sysfs 'read-only' shost attribute.
@@ -601,8 +621,9 @@
/**
** pm8001_ctl_gsm_log_show - gsm dump collection
** @cdev:pointer to embedded class device
+ ** @attr: device attribute (unused)
** @buf: the buffer returned
- **A sysfs 'read-only' shost attribute.
+ ** A sysfs 'read-only' shost attribute.
**/
static ssize_t pm8001_ctl_gsm_log_show(struct device *cdev,
struct device_attribute *attr, char *buf)
diff --git a/drivers/scsi/pm8001/pm8001_hwi.c b/drivers/scsi/pm8001/pm8001_hwi.c
index 0b44992..e9a9392 100644
--- a/drivers/scsi/pm8001/pm8001_hwi.c
+++ b/drivers/scsi/pm8001/pm8001_hwi.c
@@ -333,6 +333,7 @@
/**
* update_inbnd_queue_table - update the inbound queue table to the HBA.
* @pm8001_ha: our hba card information
+ * @number: entry in the queue
*/
static void update_inbnd_queue_table(struct pm8001_hba_info *pm8001_ha,
int number)
@@ -354,6 +355,7 @@
/**
* update_outbnd_queue_table - update the outbound queue table to the HBA.
* @pm8001_ha: our hba card information
+ * @number: entry in the queue
*/
static void update_outbnd_queue_table(struct pm8001_hba_info *pm8001_ha,
int number)
@@ -473,7 +475,7 @@
/**
* mpi_set_open_retry_interval_reg
* @pm8001_ha: our hba card information
- * @interval - interval time for each OPEN_REJECT (RETRY). The units are in 1us.
+ * @interval: interval time for each OPEN_REJECT (RETRY). The units are in 1us.
*/
static void mpi_set_open_retry_interval_reg(struct pm8001_hba_info *pm8001_ha,
u32 interval)
@@ -1231,6 +1233,7 @@
/**
* pm8001_chip_msix_interrupt_enable - enable PM8001 chip interrupt
* @pm8001_ha: our hba card information
+ * @int_vec_idx: interrupt number to enable
*/
static void
pm8001_chip_msix_interrupt_enable(struct pm8001_hba_info *pm8001_ha,
@@ -1249,6 +1252,7 @@
/**
* pm8001_chip_msix_interrupt_disable - disable PM8001 chip interrupt
* @pm8001_ha: our hba card information
+ * @int_vec_idx: interrupt number to disable
*/
static void
pm8001_chip_msix_interrupt_disable(struct pm8001_hba_info *pm8001_ha,
@@ -1264,6 +1268,7 @@
/**
* pm8001_chip_interrupt_enable - enable PM8001 chip interrupt
* @pm8001_ha: our hba card information
+ * @vec: unused
*/
static void
pm8001_chip_interrupt_enable(struct pm8001_hba_info *pm8001_ha, u8 vec)
@@ -1278,6 +1283,7 @@
/**
* pm8001_chip_intx_interrupt_disable- disable PM8001 chip interrupt
* @pm8001_ha: our hba card information
+ * @vec: unused
*/
static void
pm8001_chip_interrupt_disable(struct pm8001_hba_info *pm8001_ha, u8 vec)
@@ -4500,7 +4506,6 @@
/**
* pm8001_chip_phy_start_req - start phy via PHY_START COMMAND
* @pm8001_ha: our hba card information.
- * @num: the inbound queue number
* @phy_id: the phy id which we wanted to start up.
*/
static int
@@ -4536,7 +4541,6 @@
/**
* pm8001_chip_phy_stop_req - start phy via PHY_STOP COMMAND
* @pm8001_ha: our hba card information.
- * @num: the inbound queue number
* @phy_id: the phy id which we wanted to start up.
*/
static int pm8001_chip_phy_stop_req(struct pm8001_hba_info *pm8001_ha,
@@ -4556,7 +4560,7 @@
return ret;
}
-/**
+/*
* see comments on pm8001_mpi_reg_resp.
*/
static int pm8001_chip_reg_dev_req(struct pm8001_hba_info *pm8001_ha,
@@ -4616,7 +4620,7 @@
return rc;
}
-/**
+/*
* see comments on pm8001_mpi_reg_resp.
*/
int pm8001_chip_dereg_dev_req(struct pm8001_hba_info *pm8001_ha,
@@ -4641,9 +4645,8 @@
/**
* pm8001_chip_phy_ctl_req - support the local phy operation
* @pm8001_ha: our hba card information.
- * @num: the inbound queue number
- * @phy_id: the phy id which we wanted to operate
- * @phy_op:
+ * @phyId: the phy id which we wanted to operate
+ * @phy_op: the phy operation to request
*/
static int pm8001_chip_phy_ctl_req(struct pm8001_hba_info *pm8001_ha,
u32 phyId, u32 phy_op)
@@ -4679,8 +4682,7 @@
/**
* pm8001_chip_isr - PM8001 isr handler.
* @pm8001_ha: our hba card information.
- * @irq: irq number.
- * @stat: stat.
+ * @vec: IRQ number
*/
static irqreturn_t
pm8001_chip_isr(struct pm8001_hba_info *pm8001_ha, u8 vec)
@@ -4717,10 +4719,8 @@
return ret;
}
-/**
+/*
* pm8001_chip_abort_task - SAS abort task when error or exception happened.
- * @task: the task we wanted to aborted.
- * @flag: the abort flag.
*/
int pm8001_chip_abort_task(struct pm8001_hba_info *pm8001_ha,
struct pm8001_device *pm8001_dev, u8 flag, u32 task_tag, u32 cmd_tag)
@@ -4959,6 +4959,7 @@
* pm8001_chip_fw_flash_update_build - support the firmware update operation
* @pm8001_ha: our hba card information.
* @fw_flash_updata_info: firmware flash update param
+ * @tag: Tag to apply to the payload
*/
int
pm8001_chip_fw_flash_update_build(struct pm8001_hba_info *pm8001_ha,
diff --git a/drivers/scsi/pm8001/pm8001_init.c b/drivers/scsi/pm8001/pm8001_init.c
index 9e99262..20fa96c 100644
--- a/drivers/scsi/pm8001/pm8001_init.c
+++ b/drivers/scsi/pm8001/pm8001_init.c
@@ -57,7 +57,7 @@
static struct scsi_transport_template *pm8001_stt;
-/**
+/*
* chip info structure to identify chip key functionality as
* encryption available/not, no of ports, hw specific function ref
*/
@@ -80,7 +80,7 @@
struct workqueue_struct *pm8001_wq;
-/**
+/*
* The main structure which LLDD must register for scsi core.
*/
static struct scsi_host_template pm8001_sht = {
@@ -109,7 +109,7 @@
.track_queue_depth = 1,
};
-/**
+/*
* Sas layer call this function to execute specific task.
*/
static struct sas_domain_function_template pm8001_transport_ops = {
@@ -129,9 +129,9 @@
};
/**
- *pm8001_phy_init - initiate our adapter phys
- *@pm8001_ha: our hba structure.
- *@phy_id: phy id.
+ * pm8001_phy_init - initiate our adapter phys
+ * @pm8001_ha: our hba structure.
+ * @phy_id: phy id.
*/
static void pm8001_phy_init(struct pm8001_hba_info *pm8001_ha, int phy_id)
{
@@ -155,9 +155,8 @@
}
/**
- *pm8001_free - free hba
- *@pm8001_ha: our hba structure.
- *
+ * pm8001_free - free hba
+ * @pm8001_ha: our hba structure.
*/
static void pm8001_free(struct pm8001_hba_info *pm8001_ha)
{
@@ -205,6 +204,7 @@
* pm8001_interrupt_handler_msix - main MSIX interrupt handler.
* It obtains the vector number and calls the equivalent bottom
* half or services directly.
+ * @irq: interrupt number
* @opaque: the passed outbound queue/vector. Host structure is
* retrieved from the same.
*/
@@ -230,6 +230,7 @@
/**
* pm8001_interrupt_handler_intx - main INTx interrupt handler.
+ * @irq: interrupt number
* @dev_id: sas_ha structure. The HBA is retrieved from sas_has structure.
*/
@@ -257,8 +258,8 @@
/**
* pm8001_alloc - initiate our hba structure and 6 DMAs area.
- * @pm8001_ha:our hba structure.
- *
+ * @pm8001_ha: our hba structure.
+ * @ent: PCI device ID structure to match on
*/
static int pm8001_alloc(struct pm8001_hba_info *pm8001_ha,
const struct pci_device_id *ent)
@@ -615,7 +616,7 @@
/**
* pm8001_init_sas_add - initialize sas address
- * @chip_info: our ha struct.
+ * @pm8001_ha: our ha struct.
*
* Currently we just set the fixed SAS address to our HBA,for manufacture,
* it should read from the EEPROM
@@ -893,8 +894,7 @@
#ifdef PM8001_USE_MSIX
/**
* pm8001_setup_msix - enable MSI-X interrupt
- * @chip_info: our ha struct.
- * @irq_handler: irq_handler
+ * @pm8001_ha: our ha struct.
*/
static u32 pm8001_setup_msix(struct pm8001_hba_info *pm8001_ha)
{
@@ -975,7 +975,7 @@
/**
* pm8001_request_irq - register interrupt
- * @chip_info: our ha struct.
+ * @pm8001_ha: our ha struct.
*/
static u32 pm8001_request_irq(struct pm8001_hba_info *pm8001_ha)
{
diff --git a/drivers/scsi/pm8001/pm8001_sas.c b/drivers/scsi/pm8001/pm8001_sas.c
index b7cbc31..337e79d 100644
--- a/drivers/scsi/pm8001/pm8001_sas.c
+++ b/drivers/scsi/pm8001/pm8001_sas.c
@@ -129,6 +129,7 @@
*pphys_addr_lo = lower_32_bits(phys_align);
return 0;
}
+
/**
* pm8001_find_ha_by_dev - from domain device which come from sas layer to
* find out our hba struct.
@@ -366,17 +367,15 @@
return 0;
}
+#define DEV_IS_GONE(pm8001_dev) \
+ ((!pm8001_dev || (pm8001_dev->dev_type == SAS_PHY_UNUSED)))
/**
* pm8001_task_exec - queue the task(ssp, smp && ata) to the hardware.
* @task: the task to be execute.
- * @num: if can_queue great than 1, the task can be queued up. for SMP task,
- * we always execute one one time.
* @gfp_flags: gfp_flags.
* @is_tmf: if it is task management task.
* @tmf: the task management IU
*/
-#define DEV_IS_GONE(pm8001_dev) \
- ((!pm8001_dev || (pm8001_dev->dev_type == SAS_PHY_UNUSED)))
static int pm8001_task_exec(struct sas_task *task,
gfp_t gfp_flags, int is_tmf, struct pm8001_tmf_task *tmf)
{
@@ -577,6 +576,7 @@
/**
* pm8001_find_dev - find a matching pm8001_device
* @pm8001_ha: our hba card information
+ * @device_id: device ID to match against
*/
struct pm8001_device *pm8001_find_dev(struct pm8001_hba_info *pm8001_ha,
u32 device_id)
@@ -995,6 +995,7 @@
/**
* Standard mandates link reset for ATA (type 0) and hard reset for
* SSP (type 1) , only for RECOVERY
+ * @dev: the device structure for the device to reset.
*/
int pm8001_I_T_nexus_reset(struct domain_device *dev)
{
diff --git a/drivers/scsi/pm8001/pm80xx_hwi.c b/drivers/scsi/pm8001/pm80xx_hwi.c
index 05c944a..b42f41d 100644
--- a/drivers/scsi/pm8001/pm80xx_hwi.c
+++ b/drivers/scsi/pm8001/pm80xx_hwi.c
@@ -66,7 +66,7 @@
return 0;
}
-void pm80xx_pci_mem_copy(struct pm8001_hba_info *pm8001_ha, u32 soffset,
+static void pm80xx_pci_mem_copy(struct pm8001_hba_info *pm8001_ha, u32 soffset,
const void *destination,
u32 dw_count, u32 bus_base_number)
{
@@ -912,6 +912,7 @@
/**
* update_inbnd_queue_table - update the inbound queue table to the HBA.
* @pm8001_ha: our hba card information
+ * @number: entry in the queue
*/
static void update_inbnd_queue_table(struct pm8001_hba_info *pm8001_ha,
int number)
@@ -948,6 +949,7 @@
/**
* update_outbnd_queue_table - update the outbound queue table to the HBA.
* @pm8001_ha: our hba card information
+ * @number: entry in the queue
*/
static void update_outbnd_queue_table(struct pm8001_hba_info *pm8001_ha,
int number)
@@ -1711,6 +1713,7 @@
/**
* pm8001_chip_interrupt_enable - enable PM8001 chip interrupt
* @pm8001_ha: our hba card information
+ * @vec: interrupt number to enable
*/
static void
pm80xx_chip_interrupt_enable(struct pm8001_hba_info *pm8001_ha, u8 vec)
@@ -1729,6 +1732,7 @@
/**
* pm8001_chip_interrupt_disable- disable PM8001 chip interrupt
* @pm8001_ha: our hba card information
+ * @vec: interrupt number to disable
*/
static void
pm80xx_chip_interrupt_disable(struct pm8001_hba_info *pm8001_ha, u8 vec)
@@ -4822,7 +4826,6 @@
/**
* pm80xx_chip_phy_start_req - start phy via PHY_START COMMAND
* @pm8001_ha: our hba card information.
- * @num: the inbound queue number
* @phy_id: the phy id which we wanted to start up.
*/
static int
@@ -4864,7 +4867,6 @@
/**
* pm8001_chip_phy_stop_req - start phy via PHY_STOP COMMAND
* @pm8001_ha: our hba card information.
- * @num: the inbound queue number
* @phy_id: the phy id which we wanted to start up.
*/
static int pm80xx_chip_phy_stop_req(struct pm8001_hba_info *pm8001_ha,
@@ -4884,7 +4886,7 @@
return ret;
}
-/**
+/*
* see comments on pm8001_mpi_reg_resp.
*/
static int pm80xx_chip_reg_dev_req(struct pm8001_hba_info *pm8001_ha,
@@ -4957,9 +4959,8 @@
/**
* pm80xx_chip_phy_ctl_req - support the local phy operation
* @pm8001_ha: our hba card information.
- * @num: the inbound queue number
- * @phy_id: the phy id which we wanted to operate
- * @phy_op:
+ * @phyId: the phy id which we wanted to operate
+ * @phy_op: phy operation to request
*/
static int pm80xx_chip_phy_ctl_req(struct pm8001_hba_info *pm8001_ha,
u32 phyId, u32 phy_op)
@@ -4998,8 +4999,7 @@
/**
* pm8001_chip_isr - PM8001 isr handler.
* @pm8001_ha: our hba card information.
- * @irq: irq number.
- * @stat: stat.
+ * @vec: irq number.
*/
static irqreturn_t
pm80xx_chip_isr(struct pm8001_hba_info *pm8001_ha, u8 vec)
@@ -5013,8 +5013,9 @@
return IRQ_HANDLED;
}
-void mpi_set_phy_profile_req(struct pm8001_hba_info *pm8001_ha,
- u32 operation, u32 phyid, u32 length, u32 *buf)
+static void mpi_set_phy_profile_req(struct pm8001_hba_info *pm8001_ha,
+ u32 operation, u32 phyid,
+ u32 length, u32 *buf)
{
u32 tag , i, j = 0;
int rc;
diff --git a/drivers/scsi/ppa.c b/drivers/scsi/ppa.c
index a406cc8..0ae800c 100644
--- a/drivers/scsi/ppa.c
+++ b/drivers/scsi/ppa.c
@@ -779,7 +779,6 @@
(DID_OK << 16) + (h << 8) + (l & STATUS_MASK);
}
return 0; /* Finished */
- break;
default:
printk(KERN_ERR "ppa: Invalid scsi phase\n");
@@ -847,10 +846,8 @@
case 1: /* Have not connected to interface */
dev->cur_cmd = NULL; /* Forget the problem */
return SUCCESS;
- break;
default: /* SCSI command sent, can not abort */
return FAILED;
- break;
}
}
diff --git a/drivers/scsi/qedf/qedf_debugfs.c b/drivers/scsi/qedf/qedf_debugfs.c
index b88bed9..a3ed681 100644
--- a/drivers/scsi/qedf/qedf_debugfs.c
+++ b/drivers/scsi/qedf/qedf_debugfs.c
@@ -14,10 +14,9 @@
static struct dentry *qedf_dbg_root;
-/**
+/*
* qedf_dbg_host_init - setup the debugfs file for the pf
- * @pf: the pf that is starting up
- **/
+ */
void
qedf_dbg_host_init(struct qedf_dbg_ctx *qedf,
const struct qedf_debugfs_ops *dops,
@@ -42,10 +41,9 @@
}
}
-/**
+/*
* qedf_dbg_host_exit - clear out the pf's debugfs entries
- * @pf: the pf that is stopping
- **/
+ */
void
qedf_dbg_host_exit(struct qedf_dbg_ctx *qedf_dbg)
{
@@ -56,9 +54,9 @@
qedf_dbg->bdf_dentry = NULL;
}
-/**
+/*
* qedf_dbg_init - start up debugfs for the driver
- **/
+ */
void
qedf_dbg_init(char *drv_name)
{
@@ -68,9 +66,9 @@
qedf_dbg_root = debugfs_create_dir(drv_name, NULL);
}
-/**
+/*
* qedf_dbg_exit - clean out the driver's debugfs entries
- **/
+ */
void
qedf_dbg_exit(void)
{
diff --git a/drivers/scsi/qedf/qedf_io.c b/drivers/scsi/qedf/qedf_io.c
index 0f6a15c..acd9774 100644
--- a/drivers/scsi/qedf/qedf_io.c
+++ b/drivers/scsi/qedf/qedf_io.c
@@ -22,7 +22,6 @@
container_of(work, struct qedf_ioreq, timeout_work.work);
struct qedf_ctx *qedf;
struct qedf_rport *fcport;
- u8 op = 0;
if (io_req == NULL) {
QEDF_INFO(NULL, QEDF_LOG_IO, "io_req is NULL.\n");
@@ -89,7 +88,6 @@
io_req->event = QEDF_IOREQ_EV_ELS_TMO;
/* Call callback function to complete command */
if (io_req->cb_func && io_req->cb_arg) {
- op = io_req->cb_arg->op;
io_req->cb_func(io_req->cb_arg);
io_req->cb_arg = NULL;
}
@@ -487,7 +485,7 @@
int sg_count = 0;
int bd_count = 0;
u32 sg_len;
- u64 addr, end_addr;
+ u64 addr;
int i = 0;
sg_count = dma_map_sg(&qedf->pdev->dev, scsi_sglist(sc),
@@ -502,10 +500,9 @@
scsi_for_each_sg(sc, sg, sg_count, i) {
sg_len = (u32)sg_dma_len(sg);
addr = (u64)sg_dma_address(sg);
- end_addr = (u64)(addr + sg_len);
/*
- * Intermediate s/g element so check if start and end address
+ * Intermediate s/g element so check if start address
* is page aligned. Only required for writes and only if the
* number of scatter/gather elements is 8 or more.
*/
@@ -860,7 +857,6 @@
struct qedf_ctx *qedf = lport_priv(lport);
struct e4_fcoe_task_context *task_ctx;
u16 xid;
- enum fcoe_task_type req_type = 0;
struct fcoe_wqe *sqe;
u16 sqe_idx;
@@ -873,11 +869,9 @@
io_req->cpu = smp_processor_id();
if (sc_cmd->sc_data_direction == DMA_FROM_DEVICE) {
- req_type = FCOE_TASK_TYPE_READ_INITIATOR;
io_req->io_req_flags = QEDF_READ;
qedf->input_requests++;
} else if (sc_cmd->sc_data_direction == DMA_TO_DEVICE) {
- req_type = FCOE_TASK_TYPE_WRITE_INITIATOR;
io_req->io_req_flags = QEDF_WRITE;
qedf->output_requests++;
} else {
@@ -1130,8 +1124,6 @@
void qedf_scsi_completion(struct qedf_ctx *qedf, struct fcoe_cqe *cqe,
struct qedf_ioreq *io_req)
{
- u16 xid;
- struct e4_fcoe_task_context *task_ctx;
struct scsi_cmnd *sc_cmd;
struct fcoe_cqe_rsp_info *fcp_rsp;
struct qedf_rport *fcport;
@@ -1155,8 +1147,6 @@
return;
}
- xid = io_req->xid;
- task_ctx = qedf_get_task_mem(&qedf->tasks, xid);
sc_cmd = io_req->sc_cmd;
fcp_rsp = &cqe->cqe_info.rsp_info;
@@ -1342,7 +1332,6 @@
void qedf_scsi_done(struct qedf_ctx *qedf, struct qedf_ioreq *io_req,
int result)
{
- u16 xid;
struct scsi_cmnd *sc_cmd;
int refcount;
@@ -1364,7 +1353,6 @@
*/
clear_bit(QEDF_CMD_OUTSTANDING, &io_req->flags);
- xid = io_req->xid;
sc_cmd = io_req->sc_cmd;
if (!sc_cmd) {
@@ -1863,7 +1851,6 @@
struct fc_rport_priv *rdata;
struct qedf_ctx *qedf;
u16 xid;
- u32 r_a_tov = 0;
int rc = 0;
unsigned long flags;
struct fcoe_wqe *sqe;
@@ -1886,7 +1873,6 @@
goto out;
}
- r_a_tov = rdata->r_a_tov;
lport = qedf->lport;
if (lport->state != LPORT_ST_READY || !(lport->link_up)) {
@@ -1964,14 +1950,12 @@
struct qedf_ioreq *io_req)
{
uint32_t r_ctl;
- uint16_t xid;
int rc;
struct qedf_rport *fcport = io_req->fcport;
QEDF_INFO(&(qedf->dbg_ctx), QEDF_LOG_SCSI_TM, "Entered with xid = "
"0x%x cmd_type = %d\n", io_req->xid, io_req->cmd_type);
- xid = io_req->xid;
r_ctl = cqe->cqe_info.abts_info.r_ctl;
/* This was added at a point when we were scheduling abts_compl &
@@ -2159,8 +2143,6 @@
{
struct qedf_rport *fcport;
struct qedf_ctx *qedf;
- uint16_t xid;
- struct e4_fcoe_task_context *task;
int tmo = 0;
int rc = SUCCESS;
unsigned long flags;
@@ -2220,12 +2202,9 @@
refcount, fcport, fcport->rdata->ids.port_id);
/* Cleanup cmds re-use the same TID as the original I/O */
- xid = io_req->xid;
io_req->cmd_type = QEDF_CLEANUP;
io_req->return_scsi_cmd_on_abts = return_scsi_cmd_on_abts;
- task = qedf_get_task_mem(&qedf->tasks, xid);
-
init_completion(&io_req->cleanup_done);
spin_lock_irqsave(&fcport->rport_lock, flags);
@@ -2531,7 +2510,6 @@
struct fcoe_cqe *cqe)
{
unsigned long flags;
- uint16_t tmp;
uint16_t pktlen = cqe->cqe_info.unsolic_info.pkt_len;
u32 payload_len, crc;
struct fc_frame_header *fh;
@@ -2629,9 +2607,9 @@
qedf->bdq_prod_idx = 0;
writew(qedf->bdq_prod_idx, qedf->bdq_primary_prod);
- tmp = readw(qedf->bdq_primary_prod);
+ readw(qedf->bdq_primary_prod);
writew(qedf->bdq_prod_idx, qedf->bdq_secondary_prod);
- tmp = readw(qedf->bdq_secondary_prod);
+ readw(qedf->bdq_secondary_prod);
spin_unlock_irqrestore(&qedf->hba_lock, flags);
}
diff --git a/drivers/scsi/qedf/qedf_main.c b/drivers/scsi/qedf/qedf_main.c
index 6e77e49..3f04f2c 100644
--- a/drivers/scsi/qedf/qedf_main.c
+++ b/drivers/scsi/qedf/qedf_main.c
@@ -1041,9 +1041,8 @@
return rc;
}
-/**
+/*
* qedf_xmit - qedf FCoE frame transmit function
- *
*/
static int qedf_xmit(struct fc_lport *lport, struct fc_frame *fp)
{
@@ -1437,7 +1436,7 @@
kref_put(&rdata->kref, fc_rport_destroy);
}
-/**
+/*
* This event_callback is called after successful completion of libfc
* initiated target login. qedf can proceed with initiating the session
* establishment.
@@ -3222,7 +3221,6 @@
void *task_start, *task_end;
struct qed_slowpath_params slowpath_params;
struct qed_probe_params qed_params;
- u16 tmp;
/*
* When doing error recovery we didn't reap the lport so don't try
@@ -3416,9 +3414,9 @@
"Writing %d to primary and secondary BDQ doorbell registers.\n",
qedf->bdq_prod_idx);
writew(qedf->bdq_prod_idx, qedf->bdq_primary_prod);
- tmp = readw(qedf->bdq_primary_prod);
+ readw(qedf->bdq_primary_prod);
writew(qedf->bdq_prod_idx, qedf->bdq_secondary_prod);
- tmp = readw(qedf->bdq_secondary_prod);
+ readw(qedf->bdq_secondary_prod);
qed_ops->common->set_power_state(qedf->cdev, PCI_D0);
diff --git a/drivers/scsi/qedi/qedi_fw.c b/drivers/scsi/qedi/qedi_fw.c
index 946cebc..6ed7458 100644
--- a/drivers/scsi/qedi/qedi_fw.c
+++ b/drivers/scsi/qedi/qedi_fw.c
@@ -352,7 +352,6 @@
struct iscsi_cqe_unsolicited *cqe,
int count)
{
- u16 tmp;
u16 idx = 0;
struct scsi_bd *pbl;
@@ -381,10 +380,10 @@
qedi->bdq_prod_idx += count;
writew(qedi->bdq_prod_idx, qedi->bdq_primary_prod);
- tmp = readw(qedi->bdq_primary_prod);
+ readw(qedi->bdq_primary_prod);
writew(qedi->bdq_prod_idx, qedi->bdq_secondary_prod);
- tmp = readw(qedi->bdq_secondary_prod);
+ readw(qedi->bdq_secondary_prod);
}
static void qedi_unsol_pdu_adjust_bdq(struct qedi_ctx *qedi,
diff --git a/drivers/scsi/qedi/qedi_iscsi.c b/drivers/scsi/qedi/qedi_iscsi.c
index 425e665..c14ac78 100644
--- a/drivers/scsi/qedi/qedi_iscsi.c
+++ b/drivers/scsi/qedi/qedi_iscsi.c
@@ -1546,7 +1546,7 @@
},
};
-char *qedi_get_iscsi_error(enum iscsi_error_types err_code)
+static char *qedi_get_iscsi_error(enum iscsi_error_types err_code)
{
int i;
char *msg = NULL;
diff --git a/drivers/scsi/qedi/qedi_main.c b/drivers/scsi/qedi/qedi_main.c
index 81a3076..6f038ae 100644
--- a/drivers/scsi/qedi/qedi_main.c
+++ b/drivers/scsi/qedi/qedi_main.c
@@ -1155,7 +1155,6 @@
{
struct qedi_work *qedi_work;
struct qedi_conn *q_conn;
- struct iscsi_conn *conn;
struct qedi_cmd *qedi_cmd;
u32 iscsi_cid;
int rc = 0;
@@ -1168,7 +1167,6 @@
iscsi_cid);
return -1;
}
- conn = q_conn->cls_conn->dd_data;
switch (cqe->cqe_common.cqe_type) {
case ISCSI_CQE_TYPE_SOLICITED:
@@ -1962,7 +1960,7 @@
qedi_ops->ll2->start(qedi->cdev, ¶ms);
}
-/**
+/*
* qedi_get_nvram_block: - Scan through the iSCSI NVRAM block (while accounting
* for gaps) for the matching absolute-pf-id of the QEDI device.
*/
@@ -2429,7 +2427,6 @@
struct qed_probe_params qed_params;
void *task_start, *task_end;
int rc;
- u16 tmp;
if (mode != QEDI_MODE_RECOVERY) {
qedi = qedi_host_alloc(pdev);
@@ -2528,9 +2525,9 @@
"Writing %d to primary and secondary BDQ doorbell registers.\n",
qedi->bdq_prod_idx);
writew(qedi->bdq_prod_idx, qedi->bdq_primary_prod);
- tmp = readw(qedi->bdq_primary_prod);
+ readw(qedi->bdq_primary_prod);
writew(qedi->bdq_prod_idx, qedi->bdq_secondary_prod);
- tmp = readw(qedi->bdq_secondary_prod);
+ readw(qedi->bdq_secondary_prod);
ether_addr_copy(qedi->mac, qedi->dev_info.common.hw_mac);
QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_DISC, "MAC address is %pM.\n",
diff --git a/drivers/scsi/qla2xxx/qla_bsg.c b/drivers/scsi/qla2xxx/qla_bsg.c
index 88c0338..67efde1 100644
--- a/drivers/scsi/qla2xxx/qla_bsg.c
+++ b/drivers/scsi/qla2xxx/qla_bsg.c
@@ -223,8 +223,7 @@
/* validate fcp priority data */
- if (!qla24xx_fcp_prio_cfg_valid(vha,
- (struct qla_fcp_prio_cfg *) ha->fcp_prio_cfg, 1)) {
+ if (!qla24xx_fcp_prio_cfg_valid(vha, ha->fcp_prio_cfg, 1)) {
bsg_reply->result = (DID_ERROR << 16);
ret = -EINVAL;
/* If buffer was invalidatic int
diff --git a/drivers/scsi/qla2xxx/qla_dbg.c b/drivers/scsi/qla2xxx/qla_dbg.c
index 1900571..1be811a 100644
--- a/drivers/scsi/qla2xxx/qla_dbg.c
+++ b/drivers/scsi/qla2xxx/qla_dbg.c
@@ -11,10 +11,8 @@
* ----------------------------------------------------------------------
* | Level | Last Value Used | Holes |
* ----------------------------------------------------------------------
- * | Module Init and Probe | 0x0193 | 0x0146 |
- * | | | 0x015b-0x0160 |
- * | | | 0x016e |
- * | Mailbox commands | 0x1206 | 0x11a2-0x11ff |
+ * | Module Init and Probe | 0x0199 | |
+ * | Mailbox commands | 0x1206 | 0x11a5-0x11ff |
* | Device Discovery | 0x2134 | 0x210e-0x2116 |
* | | | 0x211a |
* | | | 0x211c-0x2128 |
@@ -26,11 +24,7 @@
* | | | 0x3036,0x3038 |
* | | | 0x303a |
* | DPC Thread | 0x4023 | 0x4002,0x4013 |
- * | Async Events | 0x5090 | 0x502b-0x502f |
- * | | | 0x5047 |
- * | | | 0x5084,0x5075 |
- * | | | 0x503d,0x5044 |
- * | | | 0x505f |
+ * | Async Events | 0x509c | |
* | Timer Routines | 0x6012 | |
* | User Space Interactions | 0x70e3 | 0x7018,0x702e |
* | | | 0x7020,0x7024 |
@@ -1063,7 +1057,7 @@
}
if (rval == QLA_SUCCESS)
- qla2xxx_copy_queues(ha, &fw->risc_ram[cnt]);
+ qla2xxx_copy_queues(ha, &fw->queue_dump[0]);
qla2xxx_dump_post_process(base_vha, rval);
}
@@ -2447,6 +2441,23 @@
/* Driver Debug Functions. */
/****************************************************************************/
+/* Write the debug message prefix into @pbuf. */
+static void ql_dbg_prefix(char *pbuf, int pbuf_size,
+ const scsi_qla_host_t *vha, uint msg_id)
+{
+ if (vha) {
+ const struct pci_dev *pdev = vha->hw->pdev;
+
+ /* <module-name> [<dev-name>]-<msg-id>:<host>: */
+ snprintf(pbuf, pbuf_size, "%s [%s]-%04x:%ld: ", QL_MSGHDR,
+ dev_name(&(pdev->dev)), msg_id, vha->host_no);
+ } else {
+ /* <module-name> [<dev-name>]-<msg-id>: : */
+ snprintf(pbuf, pbuf_size, "%s [%s]-%04x: : ", QL_MSGHDR,
+ "0000:00:00.0", msg_id);
+ }
+}
+
/*
* This function is for formatting and logging debug information.
* It is to be used when vha is available. It formats the message
@@ -2465,41 +2476,19 @@
{
va_list va;
struct va_format vaf;
+ char pbuf[64];
va_start(va, fmt);
vaf.fmt = fmt;
vaf.va = &va;
- if (!ql_mask_match(level)) {
- char pbuf[64];
+ ql_dbg_prefix(pbuf, ARRAY_SIZE(pbuf), vha, id);
- if (vha != NULL) {
- const struct pci_dev *pdev = vha->hw->pdev;
- /* <module-name> <msg-id>:<host> Message */
- snprintf(pbuf, sizeof(pbuf), "%s [%s]-%04x:%ld: ",
- QL_MSGHDR, dev_name(&(pdev->dev)), id,
- vha->host_no);
- } else {
- snprintf(pbuf, sizeof(pbuf), "%s [%s]-%04x: : ",
- QL_MSGHDR, "0000:00:00.0", id);
- }
- pbuf[sizeof(pbuf) - 1] = 0;
+ if (!ql_mask_match(level))
trace_ql_dbg_log(pbuf, &vaf);
- va_end(va);
- return;
- }
-
- if (vha != NULL) {
- const struct pci_dev *pdev = vha->hw->pdev;
- /* <module-name> <pci-name> <msg-id>:<host> Message */
- pr_warn("%s [%s]-%04x:%ld: %pV",
- QL_MSGHDR, dev_name(&(pdev->dev)), id + ql_dbg_offset,
- vha->host_no, &vaf);
- } else {
- pr_warn("%s [%s]-%04x: : %pV",
- QL_MSGHDR, "0000:00:00.0", id + ql_dbg_offset, &vaf);
- }
+ else
+ pr_warn("%s%pV", pbuf, &vaf);
va_end(va);
@@ -2524,6 +2513,7 @@
{
va_list va;
struct va_format vaf;
+ char pbuf[128];
if (pdev == NULL)
return;
@@ -2535,9 +2525,8 @@
vaf.fmt = fmt;
vaf.va = &va;
- /* <module-name> <dev-name>:<msg-id> Message */
- pr_warn("%s [%s]-%04x: : %pV",
- QL_MSGHDR, dev_name(&(pdev->dev)), id + ql_dbg_offset, &vaf);
+ ql_dbg_prefix(pbuf, ARRAY_SIZE(pbuf), NULL, id + ql_dbg_offset);
+ pr_warn("%s%pV", pbuf, &vaf);
va_end(va);
}
@@ -2565,16 +2554,7 @@
if (level > ql_errlev)
return;
- if (vha != NULL) {
- const struct pci_dev *pdev = vha->hw->pdev;
- /* <module-name> <msg-id>:<host> Message */
- snprintf(pbuf, sizeof(pbuf), "%s [%s]-%04x:%ld: ",
- QL_MSGHDR, dev_name(&(pdev->dev)), id, vha->host_no);
- } else {
- snprintf(pbuf, sizeof(pbuf), "%s [%s]-%04x: : ",
- QL_MSGHDR, "0000:00:00.0", id);
- }
- pbuf[sizeof(pbuf) - 1] = 0;
+ ql_dbg_prefix(pbuf, ARRAY_SIZE(pbuf), vha, id);
va_start(va, fmt);
@@ -2625,10 +2605,7 @@
if (level > ql_errlev)
return;
- /* <module-name> <dev-name>:<msg-id> Message */
- snprintf(pbuf, sizeof(pbuf), "%s [%s]-%04x: : ",
- QL_MSGHDR, dev_name(&(pdev->dev)), id);
- pbuf[sizeof(pbuf) - 1] = 0;
+ ql_dbg_prefix(pbuf, ARRAY_SIZE(pbuf), NULL, id);
va_start(va, fmt);
@@ -2679,7 +2656,6 @@
"mbox[%d] %#04x\n", i, rd_reg_word(mbx_reg));
}
-
void
ql_dump_buffer(uint level, scsi_qla_host_t *vha, uint id, const void *buf,
uint size)
@@ -2724,16 +2700,7 @@
if (level > ql_errlev)
return;
- if (qpair != NULL) {
- const struct pci_dev *pdev = qpair->pdev;
- /* <module-name> <msg-id>:<host> Message */
- snprintf(pbuf, sizeof(pbuf), "%s [%s]-%04x: ",
- QL_MSGHDR, dev_name(&(pdev->dev)), id);
- } else {
- snprintf(pbuf, sizeof(pbuf), "%s [%s]-%04x: : ",
- QL_MSGHDR, "0000:00:00.0", id);
- }
- pbuf[sizeof(pbuf) - 1] = 0;
+ ql_dbg_prefix(pbuf, ARRAY_SIZE(pbuf), qpair ? qpair->vha : NULL, id);
va_start(va, fmt);
@@ -2777,6 +2744,7 @@
{
va_list va;
struct va_format vaf;
+ char pbuf[128];
if (!ql_mask_match(level))
return;
@@ -2786,16 +2754,9 @@
vaf.fmt = fmt;
vaf.va = &va;
- if (qpair != NULL) {
- const struct pci_dev *pdev = qpair->pdev;
- /* <module-name> <pci-name> <msg-id>:<host> Message */
- pr_warn("%s [%s]-%04x: %pV",
- QL_MSGHDR, dev_name(&(pdev->dev)), id + ql_dbg_offset,
- &vaf);
- } else {
- pr_warn("%s [%s]-%04x: : %pV",
- QL_MSGHDR, "0000:00:00.0", id + ql_dbg_offset, &vaf);
- }
+ ql_dbg_prefix(pbuf, ARRAY_SIZE(pbuf), qpair ? qpair->vha : NULL,
+ id + ql_dbg_offset);
+ pr_warn("%s%pV", pbuf, &vaf);
va_end(va);
diff --git a/drivers/scsi/qla2xxx/qla_dbg.h b/drivers/scsi/qla2xxx/qla_dbg.h
index 54ed020..91eb690 100644
--- a/drivers/scsi/qla2xxx/qla_dbg.h
+++ b/drivers/scsi/qla2xxx/qla_dbg.h
@@ -53,6 +53,7 @@
__be16 fpm_b0_reg[64];
__be16 fpm_b1_reg[64];
__be16 risc_ram[0xf000];
+ u8 queue_dump[];
};
struct qla24xx_fw_dump {
diff --git a/drivers/scsi/qla2xxx/qla_def.h b/drivers/scsi/qla2xxx/qla_def.h
index 42dbf90..8c92af5 100644
--- a/drivers/scsi/qla2xxx/qla_def.h
+++ b/drivers/scsi/qla2xxx/qla_def.h
@@ -34,6 +34,8 @@
#include <scsi/scsi_transport_fc.h>
#include <scsi/scsi_bsg_fc.h>
+#include <uapi/scsi/fc/fc_els.h>
+
/* Big endian Fibre Channel S_ID (source ID) or D_ID (destination ID). */
typedef struct {
uint8_t domain;
@@ -1053,6 +1055,7 @@
#define MBA_LIP_F8 0x8016 /* Received a LIP F8. */
#define MBA_LOOP_INIT_ERR 0x8017 /* Loop Initialization Error. */
#define MBA_FABRIC_AUTH_REQ 0x801b /* Fabric Authentication Required. */
+#define MBA_CONGN_NOTI_RECV 0x801e /* Congestion Notification Received */
#define MBA_SCSI_COMPLETION 0x8020 /* SCSI Command Complete. */
#define MBA_CTIO_COMPLETION 0x8021 /* CTIO Complete. */
#define MBA_IP_COMPLETION 0x8022 /* IP Transmit Command Complete. */
@@ -1304,7 +1307,6 @@
#define RNID_TYPE_ASIC_TEMP 0xC
#define ELS_CMD_MAP_SIZE 32
-#define ELS_COMMAND_RDP 0x18
/*
* Firmware state codes from get firmware state mailbox command
@@ -1509,6 +1511,25 @@
uint8_t reserved_3[26];
} init_cb_t;
+/* Special Features Control Block */
+struct init_sf_cb {
+ uint8_t format;
+ uint8_t reserved0;
+ /*
+ * BIT 15-14 = Reserved
+ * BIT_13 = SAN Congestion Management (1 - Enabled, 0 - Disabled)
+ * BIT_12 = Remote Write Optimization (1 - Enabled, 0 - Disabled)
+ * BIT 11-0 = Reserved
+ */
+ uint16_t flags;
+ uint8_t reserved1[32];
+ uint16_t discard_OHRB_timeout_value;
+ uint16_t remote_write_opt_queue_num;
+ uint8_t reserved2[40];
+ uint8_t scm_related_parameter[16];
+ uint8_t reserved3[32];
+};
+
/*
* Get Link Status mailbox command return buffer.
*/
@@ -2182,6 +2203,8 @@
struct dsd64 rsp_dsd;
} ms_iocb_entry_t;
+#define SCM_EDC_ACC_RECEIVED BIT_6
+#define SCM_RDF_ACC_RECEIVED BIT_7
/*
* ISP queue - Mailbox Command entry structure definition.
@@ -3851,6 +3874,12 @@
uint32_t n2n_bigger:1;
uint32_t secure_adapter:1;
uint32_t secure_fw:1;
+ /* Supported by Adapter */
+ uint32_t scm_supported_a:1;
+ /* Supported by Firmware */
+ uint32_t scm_supported_f:1;
+ /* Enabled in Driver */
+ uint32_t scm_enabled:1;
} flags;
uint16_t max_exchg;
@@ -4168,6 +4197,13 @@
int init_cb_size;
dma_addr_t ex_init_cb_dma;
struct ex_init_cb_81xx *ex_init_cb;
+ dma_addr_t sf_init_cb_dma;
+ struct init_sf_cb *sf_init_cb;
+
+ void *scm_fpin_els_buff;
+ uint64_t scm_fpin_els_buff_size;
+ bool scm_fpin_valid;
+ bool scm_fpin_payload_size;
void *async_pd;
dma_addr_t async_pd_dma;
@@ -4230,6 +4266,12 @@
#define FW_ATTR_H_NVME BIT_10
#define FW_ATTR_H_NVME_UPDATED BIT_14
+ /* About firmware SCM support */
+#define FW_ATTR_EXT0_SCM_SUPPORTED BIT_12
+ /* Brocade fabric attached */
+#define FW_ATTR_EXT0_SCM_BROCADE 0x00001000
+ /* Cisco fabric attached */
+#define FW_ATTR_EXT0_SCM_CISCO 0x00002000
uint16_t fw_attributes_ext[2];
uint32_t fw_memory_size;
uint32_t fw_transfer_size;
@@ -4522,15 +4564,31 @@
#define QLA_SET_DATA_RATE_NOLR 1
#define QLA_SET_DATA_RATE_LR 2 /* Set speed and initiate LR */
+#define QLA_DEFAULT_PAYLOAD_SIZE 64
+/*
+ * This item might be allocated with a size > sizeof(struct purex_item).
+ * The "size" variable gives the size of the payload (which
+ * is variable) starting at "iocb".
+ */
struct purex_item {
struct list_head list;
struct scsi_qla_host *vha;
- void (*process_item)(struct scsi_qla_host *vha, void *pkt);
+ void (*process_item)(struct scsi_qla_host *vha,
+ struct purex_item *pkt);
+ atomic_t in_use;
+ uint16_t size;
struct {
uint8_t iocb[64];
} iocb;
};
+#define SCM_FLAG_RDF_REJECT 0x00
+#define SCM_FLAG_RDF_COMPLETED 0x01
+
+#define QLA_CON_PRIMITIVE_RECEIVED 0x1
+#define QLA_CONGESTION_ARB_WARNING 0x1
+#define QLA_CONGESTION_ARB_ALARM 0X2
+
/*
* Qlogic scsi host structure
*/
@@ -4725,6 +4783,7 @@
struct list_head head;
spinlock_t lock;
} purex_list;
+ struct purex_item default_item;
struct name_list_extended gnl;
/* Count of active session/fcport */
@@ -4738,6 +4797,7 @@
__le16 dport_data[4];
struct list_head gpnid_list;
struct fab_scan scan;
+ uint8_t scm_fabric_connection_flags;
unsigned int irq_offset;
} scsi_qla_host_t;
diff --git a/drivers/scsi/qla2xxx/qla_fw.h b/drivers/scsi/qla2xxx/qla_fw.h
index d1e12a2..bba1b77 100644
--- a/drivers/scsi/qla2xxx/qla_fw.h
+++ b/drivers/scsi/qla2xxx/qla_fw.h
@@ -610,7 +610,7 @@
__le32 residual_len; /* FW calc residual transfer length. */
union {
- uint16_t reserved_1;
+ __le16 reserved_1;
__le16 nvme_rsp_pyld_len;
};
@@ -723,6 +723,8 @@
struct dsd64 dsd[2];
};
+#define PURX_ELS_HEADER_SIZE 0x18
+
/*
* ISP queue - PUREX IOCB entry structure definition
*/
@@ -2020,7 +2022,9 @@
* BIT 0 = Extended BB credits for LR
* BIT 1 = Virtual Fabric Enable
* BIT 2-5 = Distance Support if BIT 0 is on
- * BIT 6-15 = Unused
+ * BIT 6 = Prefer FCP
+ * BIT 7 = SCM Disabled if BIT is set (1)
+ * BIT 8-15 = Unused
*/
uint16_t enhanced_features;
diff --git a/drivers/scsi/qla2xxx/qla_gbl.h b/drivers/scsi/qla2xxx/qla_gbl.h
index 061f91b..0ced18f 100644
--- a/drivers/scsi/qla2xxx/qla_gbl.h
+++ b/drivers/scsi/qla2xxx/qla_gbl.h
@@ -127,6 +127,7 @@
void qla_do_iidma_work(struct scsi_qla_host *vha, fc_port_t *fcport);
int qla2x00_reserve_mgmt_server_loop_id(scsi_qla_host_t *);
void qla_rscn_replay(fc_port_t *fcport);
+void qla24xx_free_purex_item(struct purex_item *item);
extern bool qla24xx_risc_firmware_invalid(uint32_t *);
/*
@@ -229,7 +230,8 @@
int qla24xx_post_gnl_work(struct scsi_qla_host *, fc_port_t *);
int qla24xx_post_relogin_work(struct scsi_qla_host *vha);
void qla2x00_wait_for_sess_deletion(scsi_qla_host_t *);
-void qla24xx_process_purex_rdp(struct scsi_qla_host *vha, void *pkt);
+void qla24xx_process_purex_rdp(struct scsi_qla_host *vha,
+ struct purex_item *pkt);
/*
* Global Functions in qla_mid.c source file.
diff --git a/drivers/scsi/qla2xxx/qla_init.c b/drivers/scsi/qla2xxx/qla_init.c
index 2436a17..57a2d76 100644
--- a/drivers/scsi/qla2xxx/qla_init.c
+++ b/drivers/scsi/qla2xxx/qla_init.c
@@ -3749,7 +3749,7 @@
}
/* Enable PUREX PASSTHRU */
- if (ql2xrdpenable)
+ if (ql2xrdpenable || ha->flags.scm_supported_f)
qla25xx_set_els_cmds_supported(vha);
} else
goto failed;
@@ -3962,7 +3962,7 @@
ha->fw_options[2] &= ~BIT_8;
}
- if (ql2xrdpenable)
+ if (ql2xrdpenable || ha->flags.scm_supported_f)
ha->fw_options[1] |= ADD_FO1_ENABLE_PUREX_IOCB;
/* Enable Async 8130/8131 events -- transceiver insertion/removal */
@@ -6996,36 +6996,41 @@
static int
qla2x00_restart_isp(scsi_qla_host_t *vha)
{
- int status = 0;
+ int status;
struct qla_hw_data *ha = vha->hw;
/* If firmware needs to be loaded */
if (qla2x00_isp_firmware(vha)) {
vha->flags.online = 0;
status = ha->isp_ops->chip_diag(vha);
- if (!status)
- status = qla2x00_setup_chip(vha);
+ if (status)
+ return status;
+ status = qla2x00_setup_chip(vha);
+ if (status)
+ return status;
}
- if (!status && !(status = qla2x00_init_rings(vha))) {
- clear_bit(RESET_MARKER_NEEDED, &vha->dpc_flags);
- ha->flags.chip_reset_done = 1;
+ status = qla2x00_init_rings(vha);
+ if (status)
+ return status;
- /* Initialize the queues in use */
- qla25xx_init_queues(ha);
+ clear_bit(RESET_MARKER_NEEDED, &vha->dpc_flags);
+ ha->flags.chip_reset_done = 1;
- status = qla2x00_fw_ready(vha);
- if (!status) {
- /* Issue a marker after FW becomes ready. */
- qla2x00_marker(vha, ha->base_qpair, 0, 0, MK_SYNC_ALL);
- set_bit(LOOP_RESYNC_NEEDED, &vha->dpc_flags);
- }
+ /* Initialize the queues in use */
+ qla25xx_init_queues(ha);
+ status = qla2x00_fw_ready(vha);
+ if (status) {
/* if no cable then assume it's good */
- if ((vha->device_flags & DFLG_NO_CABLE))
- status = 0;
+ return vha->device_flags & DFLG_NO_CABLE ? 0 : status;
}
- return (status);
+
+ /* Issue a marker after FW becomes ready. */
+ qla2x00_marker(vha, ha->base_qpair, 0, 0, MK_SYNC_ALL);
+ set_bit(LOOP_RESYNC_NEEDED, &vha->dpc_flags);
+
+ return 0;
}
static int
@@ -8514,6 +8519,11 @@
icb->node_name[0] &= 0xF0;
}
+ if (IS_QLA28XX(ha) || IS_QLA27XX(ha)) {
+ if ((nv->enhanced_features & BIT_7) == 0)
+ ha->flags.scm_supported_a = 1;
+ }
+
/* Set host adapter parameters. */
ha->flags.disable_risc_code_load = 0;
ha->flags.enable_lip_reset = 0;
diff --git a/drivers/scsi/qla2xxx/qla_inline.h b/drivers/scsi/qla2xxx/qla_inline.h
index 1fb6cca..861dc52 100644
--- a/drivers/scsi/qla2xxx/qla_inline.h
+++ b/drivers/scsi/qla2xxx/qla_inline.h
@@ -11,7 +11,7 @@
* Continuation Type 1 IOCBs to allocate.
*
* @vha: HA context
- * @dsds: number of data segment decriptors needed
+ * @dsds: number of data segment descriptors needed
*
* Returns the number of IOCB entries needed to store @dsds.
*/
diff --git a/drivers/scsi/qla2xxx/qla_iocb.c b/drivers/scsi/qla2xxx/qla_iocb.c
index 8865c35..e3d2dea 100644
--- a/drivers/scsi/qla2xxx/qla_iocb.c
+++ b/drivers/scsi/qla2xxx/qla_iocb.c
@@ -44,7 +44,7 @@
* qla2x00_calc_iocbs_32() - Determine number of Command Type 2 and
* Continuation Type 0 IOCBs to allocate.
*
- * @dsds: number of data segment decriptors needed
+ * @dsds: number of data segment descriptors needed
*
* Returns the number of IOCB entries needed to store @dsds.
*/
@@ -66,7 +66,7 @@
* qla2x00_calc_iocbs_64() - Determine number of Command Type 3 and
* Continuation Type 1 IOCBs to allocate.
*
- * @dsds: number of data segment decriptors needed
+ * @dsds: number of data segment descriptors needed
*
* Returns the number of IOCB entries needed to store @dsds.
*/
@@ -669,7 +669,7 @@
* qla24xx_calc_dsd_lists() - Determine number of DSD list required
* for Command Type 6.
*
- * @dsds: number of data segment decriptors needed
+ * @dsds: number of data segment descriptors needed
*
* Returns the number of dsd list needed to store @dsds.
*/
@@ -2305,8 +2305,8 @@
pkt = req->ring_ptr;
memset(pkt, 0, REQUEST_ENTRY_SIZE);
if (IS_QLAFX00(ha)) {
- wrt_reg_byte((void __iomem *)&pkt->entry_count, req_cnt);
- wrt_reg_word((void __iomem *)&pkt->handle, handle);
+ wrt_reg_byte((u8 __force __iomem *)&pkt->entry_count, req_cnt);
+ wrt_reg_dword((__le32 __force __iomem *)&pkt->handle, handle);
} else {
pkt->entry_count = req_cnt;
pkt->handle = handle;
diff --git a/drivers/scsi/qla2xxx/qla_isr.c b/drivers/scsi/qla2xxx/qla_isr.c
index cf080054..27bcd34 100644
--- a/drivers/scsi/qla2xxx/qla_isr.c
+++ b/drivers/scsi/qla2xxx/qla_isr.c
@@ -22,6 +22,31 @@
static void qla2x00_status_cont_entry(struct rsp_que *, sts_cont_entry_t *);
static int qla2x00_error_entry(scsi_qla_host_t *, struct rsp_que *,
sts_entry_t *);
+static void qla27xx_process_purex_fpin(struct scsi_qla_host *vha,
+ struct purex_item *item);
+static struct purex_item *qla24xx_alloc_purex_item(scsi_qla_host_t *vha,
+ uint16_t size);
+static struct purex_item *qla24xx_copy_std_pkt(struct scsi_qla_host *vha,
+ void *pkt);
+static struct purex_item *qla27xx_copy_fpin_pkt(struct scsi_qla_host *vha,
+ void **pkt, struct rsp_que **rsp);
+
+static void
+qla27xx_process_purex_fpin(struct scsi_qla_host *vha, struct purex_item *item)
+{
+ void *pkt = &item->iocb;
+ uint16_t pkt_size = item->size;
+
+ ql_dbg(ql_dbg_init + ql_dbg_verbose, vha, 0x508d,
+ "%s: Enter\n", __func__);
+
+ ql_dbg(ql_dbg_init + ql_dbg_verbose, vha, 0x508e,
+ "-------- ELS REQ -------\n");
+ ql_dump_buffer(ql_dbg_init + ql_dbg_verbose, vha, 0x508f,
+ pkt, pkt_size);
+
+ fc_host_fpin_rcv(vha->host, pkt_size, (char *)pkt);
+}
const char *const port_state_str[] = {
"Unknown",
@@ -31,35 +56,11 @@
"ONLINE"
};
-static void qla24xx_purex_iocb(scsi_qla_host_t *vha, void *pkt,
- void (*process_item)(struct scsi_qla_host *vha, void *pkt))
-{
- struct purex_list *list = &vha->purex_list;
- struct purex_item *item;
- ulong flags;
-
- item = kzalloc(sizeof(*item), GFP_KERNEL);
- if (!item) {
- ql_log(ql_log_warn, vha, 0x5092,
- ">> Failed allocate purex list item.\n");
- return;
- }
-
- item->vha = vha;
- item->process_item = process_item;
- memcpy(&item->iocb, pkt, sizeof(item->iocb));
-
- spin_lock_irqsave(&list->lock, flags);
- list_add_tail(&item->list, &list->head);
- spin_unlock_irqrestore(&list->lock, flags);
-
- set_bit(PROCESS_PUREX_IOCB, &vha->dpc_flags);
-}
-
static void
-qla24xx_process_abts(struct scsi_qla_host *vha, void *pkt)
+qla24xx_process_abts(struct scsi_qla_host *vha, struct purex_item *pkt)
{
- struct abts_entry_24xx *abts = pkt;
+ struct abts_entry_24xx *abts =
+ (struct abts_entry_24xx *)&pkt->iocb;
struct qla_hw_data *ha = vha->hw;
struct els_entry_24xx *rsp_els;
struct abts_entry_24xx *abts_rsp;
@@ -789,6 +790,179 @@
}
}
+static struct purex_item *
+qla24xx_alloc_purex_item(scsi_qla_host_t *vha, uint16_t size)
+{
+ struct purex_item *item = NULL;
+ uint8_t item_hdr_size = sizeof(*item);
+
+ if (size > QLA_DEFAULT_PAYLOAD_SIZE) {
+ item = kzalloc(item_hdr_size +
+ (size - QLA_DEFAULT_PAYLOAD_SIZE), GFP_ATOMIC);
+ } else {
+ if (atomic_inc_return(&vha->default_item.in_use) == 1) {
+ item = &vha->default_item;
+ goto initialize_purex_header;
+ } else {
+ item = kzalloc(item_hdr_size, GFP_ATOMIC);
+ }
+ }
+ if (!item) {
+ ql_log(ql_log_warn, vha, 0x5092,
+ ">> Failed allocate purex list item.\n");
+
+ return NULL;
+ }
+
+initialize_purex_header:
+ item->vha = vha;
+ item->size = size;
+ return item;
+}
+
+static void
+qla24xx_queue_purex_item(scsi_qla_host_t *vha, struct purex_item *pkt,
+ void (*process_item)(struct scsi_qla_host *vha,
+ struct purex_item *pkt))
+{
+ struct purex_list *list = &vha->purex_list;
+ ulong flags;
+
+ pkt->process_item = process_item;
+
+ spin_lock_irqsave(&list->lock, flags);
+ list_add_tail(&pkt->list, &list->head);
+ spin_unlock_irqrestore(&list->lock, flags);
+
+ set_bit(PROCESS_PUREX_IOCB, &vha->dpc_flags);
+}
+
+/**
+ * qla24xx_copy_std_pkt() - Copy over purex ELS which is
+ * contained in a single IOCB.
+ * purex packet.
+ * @vha: SCSI driver HA context
+ * @pkt: ELS packet
+ */
+static struct purex_item
+*qla24xx_copy_std_pkt(struct scsi_qla_host *vha, void *pkt)
+{
+ struct purex_item *item;
+
+ item = qla24xx_alloc_purex_item(vha,
+ QLA_DEFAULT_PAYLOAD_SIZE);
+ if (!item)
+ return item;
+
+ memcpy(&item->iocb, pkt, sizeof(item->iocb));
+ return item;
+}
+
+/**
+ * qla27xx_copy_fpin_pkt() - Copy over fpin packets that can
+ * span over multiple IOCBs.
+ * @vha: SCSI driver HA context
+ * @pkt: ELS packet
+ * @rsp: Response queue
+ */
+static struct purex_item *
+qla27xx_copy_fpin_pkt(struct scsi_qla_host *vha, void **pkt,
+ struct rsp_que **rsp)
+{
+ struct purex_entry_24xx *purex = *pkt;
+ struct rsp_que *rsp_q = *rsp;
+ sts_cont_entry_t *new_pkt;
+ uint16_t no_bytes = 0, total_bytes = 0, pending_bytes = 0;
+ uint16_t buffer_copy_offset = 0;
+ uint16_t entry_count, entry_count_remaining;
+ struct purex_item *item;
+ void *fpin_pkt = NULL;
+
+ total_bytes = (le16_to_cpu(purex->frame_size) & 0x0FFF)
+ - PURX_ELS_HEADER_SIZE;
+ pending_bytes = total_bytes;
+ entry_count = entry_count_remaining = purex->entry_count;
+ no_bytes = (pending_bytes > sizeof(purex->els_frame_payload)) ?
+ sizeof(purex->els_frame_payload) : pending_bytes;
+ ql_log(ql_log_info, vha, 0x509a,
+ "FPIN ELS, frame_size 0x%x, entry count %d\n",
+ total_bytes, entry_count);
+
+ item = qla24xx_alloc_purex_item(vha, total_bytes);
+ if (!item)
+ return item;
+
+ fpin_pkt = &item->iocb;
+
+ memcpy(fpin_pkt, &purex->els_frame_payload[0], no_bytes);
+ buffer_copy_offset += no_bytes;
+ pending_bytes -= no_bytes;
+ --entry_count_remaining;
+
+ ((response_t *)purex)->signature = RESPONSE_PROCESSED;
+ wmb();
+
+ do {
+ while ((total_bytes > 0) && (entry_count_remaining > 0)) {
+ if (rsp_q->ring_ptr->signature == RESPONSE_PROCESSED) {
+ ql_dbg(ql_dbg_async, vha, 0x5084,
+ "Ran out of IOCBs, partial data 0x%x\n",
+ buffer_copy_offset);
+ cpu_relax();
+ continue;
+ }
+
+ new_pkt = (sts_cont_entry_t *)rsp_q->ring_ptr;
+ *pkt = new_pkt;
+
+ if (new_pkt->entry_type != STATUS_CONT_TYPE) {
+ ql_log(ql_log_warn, vha, 0x507a,
+ "Unexpected IOCB type, partial data 0x%x\n",
+ buffer_copy_offset);
+ break;
+ }
+
+ rsp_q->ring_index++;
+ if (rsp_q->ring_index == rsp_q->length) {
+ rsp_q->ring_index = 0;
+ rsp_q->ring_ptr = rsp_q->ring;
+ } else {
+ rsp_q->ring_ptr++;
+ }
+ no_bytes = (pending_bytes > sizeof(new_pkt->data)) ?
+ sizeof(new_pkt->data) : pending_bytes;
+ if ((buffer_copy_offset + no_bytes) <= total_bytes) {
+ memcpy(((uint8_t *)fpin_pkt +
+ buffer_copy_offset), new_pkt->data,
+ no_bytes);
+ buffer_copy_offset += no_bytes;
+ pending_bytes -= no_bytes;
+ --entry_count_remaining;
+ } else {
+ ql_log(ql_log_warn, vha, 0x5044,
+ "Attempt to copy more that we got, optimizing..%x\n",
+ buffer_copy_offset);
+ memcpy(((uint8_t *)fpin_pkt +
+ buffer_copy_offset), new_pkt->data,
+ total_bytes - buffer_copy_offset);
+ }
+
+ ((response_t *)new_pkt)->signature = RESPONSE_PROCESSED;
+ wmb();
+ }
+
+ if (pending_bytes != 0 || entry_count_remaining != 0) {
+ ql_log(ql_log_fatal, vha, 0x508b,
+ "Dropping partial FPIN, underrun bytes = 0x%x, entry cnts 0x%x\n",
+ total_bytes, entry_count_remaining);
+ qla24xx_free_purex_item(item);
+ return NULL;
+ }
+ } while (entry_count_remaining > 0);
+ host_to_fcp_swap((uint8_t *)&item->iocb, total_bytes);
+ return item;
+}
+
/**
* qla2x00_async_event() - Process aynchronous events.
* @vha: SCSI driver HA context
@@ -1302,6 +1476,19 @@
qla2x00_post_aen_work(vha, FCH_EVT_RSCN, rscn_entry);
}
break;
+ case MBA_CONGN_NOTI_RECV:
+ if (!ha->flags.scm_enabled ||
+ mb[1] != QLA_CON_PRIMITIVE_RECEIVED)
+ break;
+
+ if (mb[2] == QLA_CONGESTION_ARB_WARNING) {
+ ql_dbg(ql_dbg_async, vha, 0x509b,
+ "Congestion Warning %04x %04x.\n", mb[1], mb[2]);
+ } else if (mb[2] == QLA_CONGESTION_ARB_ALARM) {
+ ql_log(ql_log_warn, vha, 0x509b,
+ "Congestion Alarm %04x %04x.\n", mb[1], mb[2]);
+ }
+ break;
/* case MBA_RIO_RESPONSE: */
case MBA_ZIO_RESPONSE:
ql_dbg(ql_dbg_async, vha, 0x5015,
@@ -3229,6 +3416,8 @@
{
struct sts_entry_24xx *pkt;
struct qla_hw_data *ha = vha->hw;
+ struct purex_entry_24xx *purex_entry;
+ struct purex_item *pure_item;
if (!ha->flags.fw_started)
return;
@@ -3280,8 +3469,11 @@
break;
case ABTS_RECV_24XX:
if (qla_ini_mode_enabled(vha)) {
- qla24xx_purex_iocb(vha, pkt,
- qla24xx_process_abts);
+ pure_item = qla24xx_copy_std_pkt(vha, pkt);
+ if (!pure_item)
+ break;
+ qla24xx_queue_purex_item(vha, pure_item,
+ qla24xx_process_abts);
break;
}
if (IS_QLA83XX(ha) || IS_QLA27XX(ha) ||
@@ -3329,24 +3521,40 @@
(struct vp_ctrl_entry_24xx *)pkt);
break;
case PUREX_IOCB_TYPE:
- {
- struct purex_entry_24xx *purex = (void *)pkt;
-
- if (purex->els_frame_payload[3] != ELS_COMMAND_RDP) {
- ql_dbg(ql_dbg_init, vha, 0x5091,
- "Discarding ELS Request opcode %#x...\n",
- purex->els_frame_payload[3]);
+ purex_entry = (void *)pkt;
+ switch (purex_entry->els_frame_payload[3]) {
+ case ELS_RDP:
+ pure_item = qla24xx_copy_std_pkt(vha, pkt);
+ if (!pure_item)
+ break;
+ qla24xx_queue_purex_item(vha, pure_item,
+ qla24xx_process_purex_rdp);
break;
+ case ELS_FPIN:
+ if (!vha->hw->flags.scm_enabled) {
+ ql_log(ql_log_warn, vha, 0x5094,
+ "SCM not active for this port\n");
+ break;
+ }
+ pure_item = qla27xx_copy_fpin_pkt(vha,
+ (void **)&pkt, &rsp);
+ if (!pure_item)
+ break;
+ qla24xx_queue_purex_item(vha, pure_item,
+ qla27xx_process_purex_fpin);
+ break;
+
+ default:
+ ql_log(ql_log_warn, vha, 0x509c,
+ "Discarding ELS Request opcode 0x%x\n",
+ purex_entry->els_frame_payload[3]);
}
- qla24xx_purex_iocb(vha, pkt, qla24xx_process_purex_rdp);
break;
- }
default:
/* Type Not Supported. */
ql_dbg(ql_dbg_async, vha, 0x5042,
- "Received unknown response pkt type %x "
- "entry status=%x.\n",
- pkt->entry_type, pkt->entry_status);
+ "Received unknown response pkt type 0x%x entry status=%x.\n",
+ pkt->entry_type, pkt->entry_status);
break;
}
((response_t *)pkt)->signature = RESPONSE_PROCESSED;
diff --git a/drivers/scsi/qla2xxx/qla_mbx.c b/drivers/scsi/qla2xxx/qla_mbx.c
index df31ee0..7388343 100644
--- a/drivers/scsi/qla2xxx/qla_mbx.c
+++ b/drivers/scsi/qla2xxx/qla_mbx.c
@@ -59,6 +59,7 @@
{ MBC_IOCB_COMMAND_A64 },
{ MBC_GET_ADAPTER_LOOP_ID },
{ MBC_READ_SFP },
+ { MBC_SET_RNID_PARAMS },
{ MBC_GET_RNID_PARAMS },
{ MBC_GET_SET_ZIO_THRESHOLD },
};
@@ -1124,6 +1125,16 @@
(ha->flags.secure_fw) ? "Supported" :
"Not Supported");
}
+
+ if (ha->flags.scm_supported_a &&
+ (ha->fw_attributes_ext[0] & FW_ATTR_EXT0_SCM_SUPPORTED)) {
+ ha->flags.scm_supported_f = 1;
+ memset(ha->sf_init_cb, 0, sizeof(struct init_sf_cb));
+ ha->sf_init_cb->flags |= BIT_13;
+ }
+ ql_log(ql_log_info, vha, 0x11a3, "SCM in FW: %s\n",
+ (ha->flags.scm_supported_f) ? "Supported" :
+ "Not Supported");
}
failed:
@@ -1633,8 +1644,11 @@
mcp->in_mb |= MBX_13|MBX_12|MBX_11|MBX_10;
if (IS_FWI2_CAPABLE(vha->hw))
mcp->in_mb |= MBX_19|MBX_18|MBX_17|MBX_16;
- if (IS_QLA27XX(vha->hw) || IS_QLA28XX(vha->hw))
+ if (IS_QLA27XX(vha->hw) || IS_QLA28XX(vha->hw)) {
mcp->in_mb |= MBX_15;
+ mcp->out_mb |= MBX_7|MBX_21|MBX_22|MBX_23;
+ }
+
mcp->tov = MBX_TOV_SECONDS;
mcp->flags = 0;
rval = qla2x00_mailbox_command(vha, mcp);
@@ -1687,8 +1701,22 @@
}
}
- if (IS_QLA27XX(vha->hw) || IS_QLA28XX(vha->hw))
+ if (IS_QLA27XX(vha->hw) || IS_QLA28XX(vha->hw)) {
vha->bbcr = mcp->mb[15];
+ if (mcp->mb[7] & SCM_EDC_ACC_RECEIVED) {
+ ql_log(ql_log_info, vha, 0x11a4,
+ "SCM: EDC ELS completed, flags 0x%x\n",
+ mcp->mb[21]);
+ }
+ if (mcp->mb[7] & SCM_RDF_ACC_RECEIVED) {
+ vha->hw->flags.scm_enabled = 1;
+ vha->scm_fabric_connection_flags |=
+ SCM_FLAG_RDF_COMPLETED;
+ ql_log(ql_log_info, vha, 0x11a5,
+ "SCM: RDF ELS completed, flags 0x%x\n",
+ mcp->mb[23]);
+ }
+ }
}
return rval;
@@ -1801,6 +1829,17 @@
mcp->mb[14] = sizeof(*ha->ex_init_cb);
mcp->out_mb |= MBX_14|MBX_13|MBX_12|MBX_11|MBX_10;
}
+
+ if (ha->flags.scm_supported_f) {
+ mcp->mb[1] |= BIT_1;
+ mcp->mb[16] = MSW(ha->sf_init_cb_dma);
+ mcp->mb[17] = LSW(ha->sf_init_cb_dma);
+ mcp->mb[18] = MSW(MSD(ha->sf_init_cb_dma));
+ mcp->mb[19] = LSW(MSD(ha->sf_init_cb_dma));
+ mcp->mb[15] = sizeof(*ha->sf_init_cb);
+ mcp->out_mb |= MBX_19|MBX_18|MBX_17|MBX_16|MBX_15;
+ }
+
/* 1 and 2 should normally be captured. */
mcp->in_mb = MBX_2|MBX_1|MBX_0;
if (IS_QLA83XX(ha) || IS_QLA27XX(ha) || IS_QLA28XX(ha))
@@ -4866,6 +4905,7 @@
return rval;
}
+#define PUREX_CMD_COUNT 2
int
qla25xx_set_els_cmds_supported(scsi_qla_host_t *vha)
{
@@ -4874,12 +4914,12 @@
mbx_cmd_t *mcp = &mc;
uint8_t *els_cmd_map;
dma_addr_t els_cmd_map_dma;
- uint cmd_opcode = ELS_COMMAND_RDP;
- uint index = cmd_opcode / 8;
- uint bit = cmd_opcode % 8;
+ uint8_t cmd_opcode[PUREX_CMD_COUNT];
+ uint8_t i, index, purex_bit;
struct qla_hw_data *ha = vha->hw;
- if (!IS_QLA25XX(ha) && !IS_QLA2031(ha) && !IS_QLA27XX(ha))
+ if (!IS_QLA25XX(ha) && !IS_QLA2031(ha) &&
+ !IS_QLA27XX(ha) && !IS_QLA28XX(ha))
return QLA_SUCCESS;
ql_dbg(ql_dbg_mbx + ql_dbg_verbose, vha, 0x1197,
@@ -4893,7 +4933,17 @@
return QLA_MEMORY_ALLOC_FAILED;
}
- els_cmd_map[index] |= 1 << bit;
+ memset(els_cmd_map, 0, ELS_CMD_MAP_SIZE);
+
+ /* List of Purex ELS */
+ cmd_opcode[0] = ELS_FPIN;
+ cmd_opcode[1] = ELS_RDP;
+
+ for (i = 0; i < PUREX_CMD_COUNT; i++) {
+ index = cmd_opcode[i] / 8;
+ purex_bit = cmd_opcode[i] % 8;
+ els_cmd_map[index] |= 1 << purex_bit;
+ }
mcp->mb[0] = MBC_SET_RNID_PARAMS;
mcp->mb[1] = RNID_TYPE_ELS_CMD << 8;
diff --git a/drivers/scsi/qla2xxx/qla_nx.c b/drivers/scsi/qla2xxx/qla_nx.c
index 0baf55b..71273eb 100644
--- a/drivers/scsi/qla2xxx/qla_nx.c
+++ b/drivers/scsi/qla2xxx/qla_nx.c
@@ -966,26 +966,21 @@
static int
qla82xx_flash_wait_write_finish(struct qla_hw_data *ha)
{
- long timeout = 0;
- uint32_t done = 1 ;
uint32_t val;
- int ret = 0;
+ int i, ret;
scsi_qla_host_t *vha = pci_get_drvdata(ha->pdev);
qla82xx_wr_32(ha, QLA82XX_ROMUSB_ROM_ABYTE_CNT, 0);
- while ((done != 0) && (ret == 0)) {
+ for (i = 0; i < 50000; i++) {
ret = qla82xx_read_status_reg(ha, &val);
- done = val & 1;
- timeout++;
+ if (ret < 0 || (val & 1) == 0)
+ return ret;
udelay(10);
cond_resched();
- if (timeout >= 50000) {
- ql_log(ql_log_warn, vha, 0xb00d,
- "Timeout reached waiting for write finish.\n");
- return -1;
- }
}
- return ret;
+ ql_log(ql_log_warn, vha, 0xb00d,
+ "Timeout reached waiting for write finish.\n");
+ return -1;
}
static int
@@ -1172,6 +1167,7 @@
* Offset 4: Offset and number of addr/value pairs
* that present in CRB initialize sequence
*/
+ n = 0;
if (qla82xx_rom_fast_read(ha, 0, &n) != 0 || n != 0xcafecafeUL ||
qla82xx_rom_fast_read(ha, 4, &n) != 0) {
ql_log(ql_log_fatal, vha, 0x006e,
diff --git a/drivers/scsi/qla2xxx/qla_os.c b/drivers/scsi/qla2xxx/qla_os.c
index e92fad9..9b59f032 100644
--- a/drivers/scsi/qla2xxx/qla_os.c
+++ b/drivers/scsi/qla2xxx/qla_os.c
@@ -4218,6 +4218,16 @@
"ex_init_cb=%p.\n", ha->ex_init_cb);
}
+ /* Get consistent memory allocated for Special Features-CB. */
+ if (IS_QLA27XX(ha) || IS_QLA28XX(ha)) {
+ ha->sf_init_cb = dma_pool_alloc(ha->s_dma_pool, GFP_KERNEL,
+ &ha->sf_init_cb_dma);
+ if (!ha->sf_init_cb)
+ goto fail_sf_init_cb;
+ ql_dbg_pci(ql_dbg_init, ha->pdev, 0x0199,
+ "sf_init_cb=%p.\n", ha->sf_init_cb);
+ }
+
INIT_LIST_HEAD(&ha->gbl_dsd_list);
/* Get consistent memory allocated for Async Port-Database. */
@@ -4271,6 +4281,8 @@
fail_loop_id_map:
dma_pool_free(ha->s_dma_pool, ha->async_pd, ha->async_pd_dma);
fail_async_pd:
+ dma_pool_free(ha->s_dma_pool, ha->sf_init_cb, ha->sf_init_cb_dma);
+fail_sf_init_cb:
dma_pool_free(ha->s_dma_pool, ha->ex_init_cb, ha->ex_init_cb_dma);
fail_ex_init_cb:
kfree(ha->npiv_info);
@@ -4693,6 +4705,10 @@
ha->ms_iocb = NULL;
ha->ms_iocb_dma = 0;
+ if (ha->sf_init_cb)
+ dma_pool_free(ha->s_dma_pool,
+ ha->sf_init_cb, ha->sf_init_cb_dma);
+
if (ha->ex_init_cb)
dma_pool_free(ha->s_dma_pool,
ha->ex_init_cb, ha->ex_init_cb_dma);
@@ -4780,6 +4796,8 @@
kfree(ha->swl);
ha->swl = NULL;
kfree(ha->loop_id_map);
+ ha->sf_init_cb = NULL;
+ ha->sf_init_cb_dma = 0;
ha->loop_id_map = NULL;
}
@@ -5893,10 +5911,12 @@
* vha: SCSI qla host
* purex: RDP request received by HBA
*/
-void qla24xx_process_purex_rdp(struct scsi_qla_host *vha, void *pkt)
+void qla24xx_process_purex_rdp(struct scsi_qla_host *vha,
+ struct purex_item *item)
{
struct qla_hw_data *ha = vha->hw;
- struct purex_entry_24xx *purex = pkt;
+ struct purex_entry_24xx *purex =
+ (struct purex_entry_24xx *)&item->iocb;
dma_addr_t rsp_els_dma;
dma_addr_t rsp_payload_dma;
dma_addr_t stat_dma;
@@ -6306,6 +6326,15 @@
rsp_els, rsp_els_dma);
}
+void
+qla24xx_free_purex_item(struct purex_item *item)
+{
+ if (item == &item->vha->default_item)
+ memset(&item->vha->default_item, 0, sizeof(struct purex_item));
+ else
+ kfree(item);
+}
+
void qla24xx_process_purex_list(struct purex_list *list)
{
struct list_head head = LIST_HEAD_INIT(head);
@@ -6318,8 +6347,8 @@
list_for_each_entry_safe(item, next, &head, list) {
list_del(&item->list);
- item->process_item(item->vha, &item->iocb);
- kfree(item);
+ item->process_item(item->vha, item);
+ qla24xx_free_purex_item(item);
}
}
diff --git a/drivers/scsi/qla2xxx/qla_target.h b/drivers/scsi/qla2xxx/qla_target.h
index 010f125..1cff7c6 100644
--- a/drivers/scsi/qla2xxx/qla_target.h
+++ b/drivers/scsi/qla2xxx/qla_target.h
@@ -258,7 +258,7 @@
__be16 ox_id;
uint16_t rx_id;
__le32 parameter;
-} __packed;
+};
struct fcp_hdr_le {
le_id_t d_id;
@@ -273,7 +273,7 @@
__le16 rx_id;
__le16 ox_id;
__le32 parameter;
-} __packed;
+};
#define F_CTL_EXCH_CONTEXT_RESP BIT_23
#define F_CTL_SEQ_CONTEXT_RESIP BIT_22
diff --git a/drivers/scsi/qla2xxx/tcm_qla2xxx.c b/drivers/scsi/qla2xxx/tcm_qla2xxx.c
index 68183a9..44bfe162 100644
--- a/drivers/scsi/qla2xxx/tcm_qla2xxx.c
+++ b/drivers/scsi/qla2xxx/tcm_qla2xxx.c
@@ -1971,6 +1971,7 @@
BUILD_BUG_ON(sizeof(struct ctio_crc2_to_fw) != 64);
BUILD_BUG_ON(sizeof(struct ctio_crc_from_fw) != 64);
BUILD_BUG_ON(sizeof(struct ctio_to_2xxx) != 64);
+ BUILD_BUG_ON(sizeof(struct fcp_hdr) != 24);
BUILD_BUG_ON(sizeof(struct fcp_hdr_le) != 24);
BUILD_BUG_ON(sizeof(struct nack_to_isp) != 64);
diff --git a/drivers/scsi/qla4xxx/ql4_83xx.c b/drivers/scsi/qla4xxx/ql4_83xx.c
index 638f72c..de10e67 100644
--- a/drivers/scsi/qla4xxx/ql4_83xx.c
+++ b/drivers/scsi/qla4xxx/ql4_83xx.c
@@ -1406,16 +1406,16 @@
static void qla4_83xx_dump_pause_control_regs(struct scsi_qla_host *ha)
{
u32 val = 0, val1 = 0;
- int i, status = QLA_SUCCESS;
+ int i;
- status = qla4_83xx_rd_reg_indirect(ha, QLA83XX_SRE_SHIM_CONTROL, &val);
+ qla4_83xx_rd_reg_indirect(ha, QLA83XX_SRE_SHIM_CONTROL, &val);
DEBUG2(ql4_printk(KERN_INFO, ha, "SRE-Shim Ctrl:0x%x\n", val));
/* Port 0 Rx Buffer Pause Threshold Registers. */
DEBUG2(ql4_printk(KERN_INFO, ha,
"Port 0 Rx Buffer Pause Threshold Registers[TC7..TC0]:"));
for (i = 0; i < 8; i++) {
- status = qla4_83xx_rd_reg_indirect(ha,
+ qla4_83xx_rd_reg_indirect(ha,
QLA83XX_PORT0_RXB_PAUSE_THRS + (i * 0x4), &val);
DEBUG2(pr_info("0x%x ", val));
}
@@ -1426,7 +1426,7 @@
DEBUG2(ql4_printk(KERN_INFO, ha,
"Port 1 Rx Buffer Pause Threshold Registers[TC7..TC0]:"));
for (i = 0; i < 8; i++) {
- status = qla4_83xx_rd_reg_indirect(ha,
+ qla4_83xx_rd_reg_indirect(ha,
QLA83XX_PORT1_RXB_PAUSE_THRS + (i * 0x4), &val);
DEBUG2(pr_info("0x%x ", val));
}
@@ -1437,7 +1437,7 @@
DEBUG2(ql4_printk(KERN_INFO, ha,
"Port 0 RxB Traffic Class Max Cell Registers[3..0]:"));
for (i = 0; i < 4; i++) {
- status = qla4_83xx_rd_reg_indirect(ha,
+ qla4_83xx_rd_reg_indirect(ha,
QLA83XX_PORT0_RXB_TC_MAX_CELL + (i * 0x4), &val);
DEBUG2(pr_info("0x%x ", val));
}
@@ -1448,7 +1448,7 @@
DEBUG2(ql4_printk(KERN_INFO, ha,
"Port 1 RxB Traffic Class Max Cell Registers[3..0]:"));
for (i = 0; i < 4; i++) {
- status = qla4_83xx_rd_reg_indirect(ha,
+ qla4_83xx_rd_reg_indirect(ha,
QLA83XX_PORT1_RXB_TC_MAX_CELL + (i * 0x4), &val);
DEBUG2(pr_info("0x%x ", val));
}
@@ -1459,15 +1459,11 @@
DEBUG2(ql4_printk(KERN_INFO, ha,
"Port 0 RxB Rx Traffic Class Stats [TC7..TC0]"));
for (i = 7; i >= 0; i--) {
- status = qla4_83xx_rd_reg_indirect(ha,
- QLA83XX_PORT0_RXB_TC_STATS,
- &val);
+ qla4_83xx_rd_reg_indirect(ha, QLA83XX_PORT0_RXB_TC_STATS, &val);
val &= ~(0x7 << 29); /* Reset bits 29 to 31 */
qla4_83xx_wr_reg_indirect(ha, QLA83XX_PORT0_RXB_TC_STATS,
(val | (i << 29)));
- status = qla4_83xx_rd_reg_indirect(ha,
- QLA83XX_PORT0_RXB_TC_STATS,
- &val);
+ qla4_83xx_rd_reg_indirect(ha, QLA83XX_PORT0_RXB_TC_STATS, &val);
DEBUG2(pr_info("0x%x ", val));
}
@@ -1477,24 +1473,18 @@
DEBUG2(ql4_printk(KERN_INFO, ha,
"Port 1 RxB Rx Traffic Class Stats [TC7..TC0]"));
for (i = 7; i >= 0; i--) {
- status = qla4_83xx_rd_reg_indirect(ha,
- QLA83XX_PORT1_RXB_TC_STATS,
- &val);
+ qla4_83xx_rd_reg_indirect(ha, QLA83XX_PORT1_RXB_TC_STATS, &val);
val &= ~(0x7 << 29); /* Reset bits 29 to 31 */
qla4_83xx_wr_reg_indirect(ha, QLA83XX_PORT1_RXB_TC_STATS,
(val | (i << 29)));
- status = qla4_83xx_rd_reg_indirect(ha,
- QLA83XX_PORT1_RXB_TC_STATS,
- &val);
+ qla4_83xx_rd_reg_indirect(ha, QLA83XX_PORT1_RXB_TC_STATS, &val);
DEBUG2(pr_info("0x%x ", val));
}
DEBUG2(pr_info("\n"));
- status = qla4_83xx_rd_reg_indirect(ha, QLA83XX_PORT2_IFB_PAUSE_THRS,
- &val);
- status = qla4_83xx_rd_reg_indirect(ha, QLA83XX_PORT3_IFB_PAUSE_THRS,
- &val1);
+ qla4_83xx_rd_reg_indirect(ha, QLA83XX_PORT2_IFB_PAUSE_THRS, &val);
+ qla4_83xx_rd_reg_indirect(ha, QLA83XX_PORT3_IFB_PAUSE_THRS, &val1);
DEBUG2(ql4_printk(KERN_INFO, ha,
"IFB-Pause Thresholds: Port 2:0x%x, Port 3:0x%x\n",
diff --git a/drivers/scsi/qla4xxx/ql4_83xx.h b/drivers/scsi/qla4xxx/ql4_83xx.h
index 775fdf9..f34583e 100644
--- a/drivers/scsi/qla4xxx/ql4_83xx.h
+++ b/drivers/scsi/qla4xxx/ql4_83xx.h
@@ -87,23 +87,6 @@
#define QLA83XX_FW_API 0x356C
#define QLA83XX_DRV_OP_MODE 0x3570
-static const uint32_t qla4_83xx_reg_tbl[] = {
- QLA83XX_PEG_HALT_STATUS1,
- QLA83XX_PEG_HALT_STATUS2,
- QLA83XX_PEG_ALIVE_COUNTER,
- QLA83XX_CRB_DRV_ACTIVE,
- QLA83XX_CRB_DEV_STATE,
- QLA83XX_CRB_DRV_STATE,
- QLA83XX_CRB_DRV_SCRATCH,
- QLA83XX_CRB_DEV_PART_INFO1,
- QLA83XX_CRB_IDC_VER_MAJOR,
- QLA83XX_FW_VER_MAJOR,
- QLA83XX_FW_VER_MINOR,
- QLA83XX_FW_VER_SUB,
- QLA83XX_CMDPEG_STATE,
- QLA83XX_ASIC_TEMP,
-};
-
#define QLA83XX_CRB_WIN_BASE 0x3800
#define QLA83XX_CRB_WIN_FUNC(f) (QLA83XX_CRB_WIN_BASE+((f)*4))
#define QLA83XX_SEM_LOCK_BASE 0x3840
diff --git a/drivers/scsi/qla4xxx/ql4_bsg.c b/drivers/scsi/qla4xxx/ql4_bsg.c
index 415ee5eb..9231917 100644
--- a/drivers/scsi/qla4xxx/ql4_bsg.c
+++ b/drivers/scsi/qla4xxx/ql4_bsg.c
@@ -805,7 +805,7 @@
/**
* qla4xxx_process_vendor_specific - handle vendor specific bsg request
- * @job: iscsi_bsg_job to handle
+ * @bsg_job: iscsi_bsg_job to handle
**/
int qla4xxx_process_vendor_specific(struct bsg_job *bsg_job)
{
@@ -852,7 +852,7 @@
/**
* qla4xxx_bsg_request - handle bsg request from ISCSI transport
- * @job: iscsi_bsg_job to handle
+ * @bsg_job: iscsi_bsg_job to handle
*/
int qla4xxx_bsg_request(struct bsg_job *bsg_job)
{
diff --git a/drivers/scsi/qla4xxx/ql4_init.c b/drivers/scsi/qla4xxx/ql4_init.c
index 2bf5e3e..4a7ef97 100644
--- a/drivers/scsi/qla4xxx/ql4_init.c
+++ b/drivers/scsi/qla4xxx/ql4_init.c
@@ -14,7 +14,6 @@
static void ql4xxx_set_mac_number(struct scsi_qla_host *ha)
{
uint32_t value;
- uint8_t func_number;
unsigned long flags;
/* Get the function number */
@@ -22,7 +21,6 @@
value = readw(&ha->reg->ctrl_status);
spin_unlock_irqrestore(&ha->hardware_lock, flags);
- func_number = (uint8_t) ((value >> 4) & 0x30);
switch (value & ISP_CONTROL_FN_MASK) {
case ISP_CONTROL_FN0_SCSI:
ha->mac_index = 1;
@@ -667,6 +665,9 @@
pci_set_master(ha->pdev);
status = pci_set_mwi(ha->pdev);
+ if (status)
+ ql4_printk(KERN_WARNING, ha, "Failed to set MWI\n");
+
/*
* We want to respect framework's setting of PCI configuration space
* command register and also want to make sure that all bits of
@@ -945,6 +946,7 @@
/**
* qla4xxx_initialize_adapter - initiailizes hba
* @ha: Pointer to host adapter structure.
+ * @is_reset: Is this init path or reset path
*
* This routine parforms all of the steps necessary to initialize the adapter.
*
@@ -1156,9 +1158,10 @@
/**
* qla4xxx_process_ddb_changed - process ddb state change
- * @ha - Pointer to host adapter structure.
- * @fw_ddb_index - Firmware's device database index
- * @state - Device state
+ * @ha: Pointer to host adapter structure.
+ * @fw_ddb_index: Firmware's device database index
+ * @state: Device state
+ * @conn_err: Unused
*
* This routine processes a Decive Database Changed AEN Event.
**/
diff --git a/drivers/scsi/qla4xxx/ql4_iocb.c b/drivers/scsi/qla4xxx/ql4_iocb.c
index 17222eb4..a8df2d7 100644
--- a/drivers/scsi/qla4xxx/ql4_iocb.c
+++ b/drivers/scsi/qla4xxx/ql4_iocb.c
@@ -78,7 +78,7 @@
* @ha: Pointer to host adapter structure.
* @ddb_entry: Pointer to device database entry
* @lun: SCSI LUN
- * @marker_type: marker identifier
+ * @mrkr_mod: marker identifier
*
* This routine issues a marker IOCB.
**/
diff --git a/drivers/scsi/qla4xxx/ql4_isr.c b/drivers/scsi/qla4xxx/ql4_isr.c
index d2cd33d..ade5eaf 100644
--- a/drivers/scsi/qla4xxx/ql4_isr.c
+++ b/drivers/scsi/qla4xxx/ql4_isr.c
@@ -582,7 +582,7 @@
/**
* qla4_83xx_loopback_in_progress: Is loopback in progress?
* @ha: Pointer to host adapter structure.
- * @ret: 1 = loopback in progress, 0 = loopback not in progress
+ * returns: 1 = loopback in progress, 0 = loopback not in progress
**/
static int qla4_83xx_loopback_in_progress(struct scsi_qla_host *ha)
{
@@ -651,7 +651,7 @@
/**
* qla4xxx_isr_decode_mailbox - decodes mailbox status
* @ha: Pointer to host adapter structure.
- * @mailbox_status: Mailbox status.
+ * @mbox_status: Mailbox status.
*
* This routine decodes the mailbox status during the ISR.
* Hardware_lock locked upon entry. runs in interrupt context.
@@ -1044,6 +1044,7 @@
/**
* qla4_82xx_interrupt_service_routine - isr
* @ha: pointer to host adapter structure.
+ * @intr_status: Local interrupt status/type.
*
* This is the main interrupt service routine.
* hardware_lock locked upon entry. runs in interrupt context.
@@ -1069,6 +1070,7 @@
/**
* qla4xxx_interrupt_service_routine - isr
* @ha: pointer to host adapter structure.
+ * @intr_status: Local interrupt status/type.
*
* This is the main interrupt service routine.
* hardware_lock locked upon entry. runs in interrupt context.
diff --git a/drivers/scsi/qla4xxx/ql4_mbx.c b/drivers/scsi/qla4xxx/ql4_mbx.c
index 02636b4..bc8de7d 100644
--- a/drivers/scsi/qla4xxx/ql4_mbx.c
+++ b/drivers/scsi/qla4xxx/ql4_mbx.c
@@ -47,7 +47,7 @@
/**
* qla4xxx_is_intr_poll_mode – Are we allowed to poll for interrupts?
* @ha: Pointer to host adapter structure.
- * @ret: 1=polling mode, 0=non-polling mode
+ * returns: 1=polling mode, 0=non-polling mode
**/
static int qla4xxx_is_intr_poll_mode(struct scsi_qla_host *ha)
{
@@ -810,7 +810,7 @@
return QLA_SUCCESS;
}
-/**
+/*
* qla4xxx_get_fwddb_entry - retrieves firmware ddb entry
* @ha: Pointer to host adapter structure.
* @fw_ddb_index: Firmware's device database index
@@ -1259,8 +1259,7 @@
/**
* qla4xxx_reset_target - issues target Reset
* @ha: Pointer to host adapter structure.
- * @db_entry: Pointer to device database entry
- * @un_entry: Pointer to lun entry structure
+ * @ddb_entry: Pointer to device database entry
*
* This routine performs a TARGET RESET on the specified target.
* The caller must ensure that the ddb_entry pointers
diff --git a/drivers/scsi/qla4xxx/ql4_nx.c b/drivers/scsi/qla4xxx/ql4_nx.c
index 5a31877..038e19b 100644
--- a/drivers/scsi/qla4xxx/ql4_nx.c
+++ b/drivers/scsi/qla4xxx/ql4_nx.c
@@ -974,10 +974,10 @@
return ret;
}
-/**
+/*
* This routine does CRB initialize sequence
* to put the ISP into operational state
- **/
+ */
static int
qla4_82xx_pinit_from_rom(struct scsi_qla_host *ha, int verbose)
{
@@ -2645,7 +2645,7 @@
uint32_t addr1, addr2, value, data, temp, wrval;
uint8_t stride, stride2;
uint16_t count;
- uint32_t poll, mask, data_size, modify_mask;
+ uint32_t poll, mask, modify_mask;
uint32_t wait_count = 0;
uint32_t *data_ptr = *d_ptr;
struct qla8044_minidump_entry_rddfe *rddfe;
@@ -2661,7 +2661,6 @@
poll = le32_to_cpu(rddfe->poll);
mask = le32_to_cpu(rddfe->mask);
modify_mask = le32_to_cpu(rddfe->modify_mask);
- data_size = le32_to_cpu(rddfe->data_size);
addr2 = addr1 + stride;
@@ -2742,7 +2741,7 @@
uint8_t stride1, stride2;
uint32_t addr3, addr4, addr5, addr6, addr7;
uint16_t count, loop_cnt;
- uint32_t poll, mask;
+ uint32_t mask;
uint32_t *data_ptr = *d_ptr;
struct qla8044_minidump_entry_rdmdio *rdmdio;
@@ -2754,7 +2753,6 @@
stride2 = le32_to_cpu(rdmdio->stride_2);
count = le32_to_cpu(rdmdio->count);
- poll = le32_to_cpu(rdmdio->poll);
mask = le32_to_cpu(rdmdio->mask);
value2 = le32_to_cpu(rdmdio->value_2);
@@ -2813,7 +2811,7 @@
struct qla8xxx_minidump_entry_hdr *entry_hdr,
uint32_t **d_ptr)
{
- uint32_t addr1, addr2, value1, value2, poll, mask, r_value;
+ uint32_t addr1, addr2, value1, value2, poll, r_value;
struct qla8044_minidump_entry_pollwr *pollwr_hdr;
uint32_t wait_count = 0;
uint32_t rval = QLA_SUCCESS;
@@ -2825,7 +2823,6 @@
value2 = le32_to_cpu(pollwr_hdr->value_2);
poll = le32_to_cpu(pollwr_hdr->poll);
- mask = le32_to_cpu(pollwr_hdr->mask);
while (wait_count < poll) {
ha->isp_ops->rd_reg_indirect(ha, addr1, &r_value);
@@ -3220,6 +3217,7 @@
/**
* qla4_8xxx_uevent_emit - Send uevent when the firmware dump is ready.
* @ha: pointer to adapter structure
+ * @code: uevent code to act upon
**/
static void qla4_8xxx_uevent_emit(struct scsi_qla_host *ha, u32 code)
{
@@ -3688,9 +3686,9 @@
return dwptr;
}
-/**
+/*
* Address and length are byte address
- **/
+ */
static uint8_t *
qla4_82xx_read_optrom_data(struct scsi_qla_host *ha, uint8_t *buf,
uint32_t offset, uint32_t length)
diff --git a/drivers/scsi/qla4xxx/ql4_nx.h b/drivers/scsi/qla4xxx/ql4_nx.h
index 98fe786..b7a6e7f 100644
--- a/drivers/scsi/qla4xxx/ql4_nx.h
+++ b/drivers/scsi/qla4xxx/ql4_nx.h
@@ -599,23 +599,6 @@
QLA8XXX_CRB_TEMP_STATE,
};
-static const uint32_t qla4_82xx_reg_tbl[] = {
- QLA82XX_PEG_HALT_STATUS1,
- QLA82XX_PEG_HALT_STATUS2,
- QLA82XX_PEG_ALIVE_COUNTER,
- QLA82XX_CRB_DRV_ACTIVE,
- QLA82XX_CRB_DEV_STATE,
- QLA82XX_CRB_DRV_STATE,
- QLA82XX_CRB_DRV_SCRATCH,
- QLA82XX_CRB_DEV_PART_INFO,
- QLA82XX_CRB_DRV_IDC_VERSION,
- QLA82XX_FW_VERSION_MAJOR,
- QLA82XX_FW_VERSION_MINOR,
- QLA82XX_FW_VERSION_SUB,
- CRB_CMDPEG_STATE,
- CRB_TEMP_STATE,
-};
-
/* Every driver should use these Device State */
#define QLA8XXX_DEV_COLD 1
#define QLA8XXX_DEV_INITIALIZING 2
diff --git a/drivers/scsi/qla4xxx/ql4_os.c b/drivers/scsi/qla4xxx/ql4_os.c
index 5dc697c..bab87e4 100644
--- a/drivers/scsi/qla4xxx/ql4_os.c
+++ b/drivers/scsi/qla4xxx/ql4_os.c
@@ -188,6 +188,40 @@
static struct qla4_8xxx_legacy_intr_set legacy_intr[] =
QLA82XX_LEGACY_INTR_CONFIG;
+static const uint32_t qla4_82xx_reg_tbl[] = {
+ QLA82XX_PEG_HALT_STATUS1,
+ QLA82XX_PEG_HALT_STATUS2,
+ QLA82XX_PEG_ALIVE_COUNTER,
+ QLA82XX_CRB_DRV_ACTIVE,
+ QLA82XX_CRB_DEV_STATE,
+ QLA82XX_CRB_DRV_STATE,
+ QLA82XX_CRB_DRV_SCRATCH,
+ QLA82XX_CRB_DEV_PART_INFO,
+ QLA82XX_CRB_DRV_IDC_VERSION,
+ QLA82XX_FW_VERSION_MAJOR,
+ QLA82XX_FW_VERSION_MINOR,
+ QLA82XX_FW_VERSION_SUB,
+ CRB_CMDPEG_STATE,
+ CRB_TEMP_STATE,
+};
+
+static const uint32_t qla4_83xx_reg_tbl[] = {
+ QLA83XX_PEG_HALT_STATUS1,
+ QLA83XX_PEG_HALT_STATUS2,
+ QLA83XX_PEG_ALIVE_COUNTER,
+ QLA83XX_CRB_DRV_ACTIVE,
+ QLA83XX_CRB_DEV_STATE,
+ QLA83XX_CRB_DRV_STATE,
+ QLA83XX_CRB_DRV_SCRATCH,
+ QLA83XX_CRB_DEV_PART_INFO1,
+ QLA83XX_CRB_IDC_VER_MAJOR,
+ QLA83XX_FW_VER_MAJOR,
+ QLA83XX_FW_VER_MINOR,
+ QLA83XX_FW_VER_SUB,
+ QLA83XX_CMDPEG_STATE,
+ QLA83XX_ASIC_TEMP,
+};
+
static struct scsi_host_template qla4xxx_driver_template = {
.module = THIS_MODULE,
.name = DRIVER_NAME,
@@ -1845,12 +1879,10 @@
static enum blk_eh_timer_return qla4xxx_eh_cmd_timed_out(struct scsi_cmnd *sc)
{
struct iscsi_cls_session *session;
- struct iscsi_session *sess;
unsigned long flags;
enum blk_eh_timer_return ret = BLK_EH_DONE;
session = starget_to_session(scsi_target(sc->device));
- sess = session->dd_data;
spin_lock_irqsave(&session->lock, flags);
if (session->state == ISCSI_SESSION_FAILED)
@@ -3059,7 +3091,6 @@
struct ddb_entry *ddb_entry;
uint16_t ddb_index;
struct iscsi_session *sess;
- struct sockaddr *dst_addr;
int ret;
if (!ep) {
@@ -3068,7 +3099,6 @@
}
qla_ep = ep->dd_data;
- dst_addr = (struct sockaddr *)&qla_ep->dst_addr;
ha = to_qla_host(qla_ep->host);
DEBUG2(ql4_printk(KERN_INFO, ha, "%s: host: %ld\n", __func__,
ha->host_no));
@@ -4515,7 +4545,7 @@
/**
* qla4xxx_timer - checks every second for work to do.
- * @ha: Pointer to host adapter structure.
+ * @t: Context to obtain pointer to host adapter structure.
**/
static void qla4xxx_timer(struct timer_list *t)
{
@@ -5269,7 +5299,7 @@
/**
* qla4xxx_do_dpc - dpc routine
- * @data: in our case pointer to adapter structure
+ * @work: Context to obtain pointer to host adapter structure.
*
* This routine is a task that is schedule by the interrupt handler
* to perform the background processing for interrupts. We put it
@@ -5492,7 +5522,7 @@
int qla4_8xxx_iospace_config(struct scsi_qla_host *ha)
{
int status = 0;
- unsigned long mem_base, mem_len, db_base, db_len;
+ unsigned long mem_base, mem_len;
struct pci_dev *pdev = ha->pdev;
status = pci_request_regions(pdev, DRIVER_NAME);
@@ -5536,9 +5566,6 @@
((uint8_t *)ha->nx_pcibase);
}
- db_base = pci_resource_start(pdev, 4); /* doorbell is on bar 4 */
- db_len = pci_resource_len(pdev, 4);
-
return 0;
iospace_error_exit:
return -ENOMEM;
@@ -6249,14 +6276,12 @@
static void qla4xxx_get_param_ddb(struct ddb_entry *ddb_entry,
struct ql4_tuple_ddb *tddb)
{
- struct scsi_qla_host *ha;
struct iscsi_cls_session *cls_sess;
struct iscsi_cls_conn *cls_conn;
struct iscsi_session *sess;
struct iscsi_conn *conn;
DEBUG2(printk(KERN_INFO "Func: %s\n", __func__));
- ha = ddb_entry->ha;
cls_sess = ddb_entry->sess;
sess = cls_sess->dd_data;
cls_conn = ddb_entry->conn;
@@ -8591,7 +8616,7 @@
/**
* qla4xxx_probe_adapter - callback function to probe HBA
* @pdev: pointer to pci_dev structure
- * @pci_device_id: pointer to pci_device entry
+ * @ent: pointer to pci_device entry
*
* This routine will probe for Qlogic 4xxx iSCSI host adapters.
* It returns zero if successful. It also initializes all data necessary for
@@ -8977,7 +9002,7 @@
}
/**
* qla4xxx_remove_adapter - callback function to remove adapter.
- * @pci_dev: PCI device pointer
+ * @pdev: PCI device pointer
**/
static void qla4xxx_remove_adapter(struct pci_dev *pdev)
{
@@ -9144,8 +9169,8 @@
/**
* qla4xxx_eh_wait_for_commands - wait for active cmds to finish.
* @ha: pointer to HBA
- * @t: target id
- * @l: lun id
+ * @stgt: pointer to SCSI target
+ * @sdev: pointer to SCSI device
*
* This function waits for all outstanding commands to a lun to complete. It
* returns 0 if all pending commands are returned and 1 otherwise.
@@ -9615,6 +9640,7 @@
* qla4xxx_pci_mmio_enabled() gets called if
* qla4xxx_pci_error_detected() returns PCI_ERS_RESULT_CAN_RECOVER
* and read/write to the device still works.
+ * @pdev: PCI device pointer
**/
static pci_ers_result_t
qla4xxx_pci_mmio_enabled(struct pci_dev *pdev)
diff --git a/drivers/scsi/scsi.c b/drivers/scsi/scsi.c
index 56c24a7..24619c3 100644
--- a/drivers/scsi/scsi.c
+++ b/drivers/scsi/scsi.c
@@ -754,9 +754,6 @@
{
int error;
- error = scsi_init_queue();
- if (error)
- return error;
error = scsi_init_procfs();
if (error)
goto cleanup_queue;
diff --git a/drivers/scsi/scsi_debug.c b/drivers/scsi/scsi_debug.c
index 843cccb..064ed68 100644
--- a/drivers/scsi/scsi_debug.c
+++ b/drivers/scsi/scsi_debug.c
@@ -9,7 +9,7 @@
*
* Copyright (C) 2001 - 2020 Douglas Gilbert
*
- * For documentation see http://sg.danny.cz/sg/sdebug26.html
+ * For documentation see http://sg.danny.cz/sg/scsi_debug.html
*/
@@ -60,8 +60,8 @@
#include "scsi_logging.h"
/* make sure inq_product_rev string corresponds to this version */
-#define SDEBUG_VERSION "0189" /* format to fit INQUIRY revision field */
-static const char *sdebug_version_date = "20200421";
+#define SDEBUG_VERSION "0190" /* format to fit INQUIRY revision field */
+static const char *sdebug_version_date = "20200710";
#define MY_NAME "scsi_debug"
@@ -151,6 +151,7 @@
#define DEF_STRICT 0
#define DEF_STATISTICS false
#define DEF_SUBMIT_QUEUES 1
+#define DEF_TUR_MS_TO_READY 0
#define DEF_UUID_CTL 0
#define JDELAY_OVERRIDDEN -9999
@@ -187,21 +188,8 @@
SDEBUG_OPT_SHORT_TRANSFER | \
SDEBUG_OPT_HOST_BUSY | \
SDEBUG_OPT_CMD_ABORT)
-/* When "every_nth" > 0 then modulo "every_nth" commands:
- * - a missing response is simulated if SDEBUG_OPT_TIMEOUT is set
- * - a RECOVERED_ERROR is simulated on successful read and write
- * commands if SDEBUG_OPT_RECOVERED_ERR is set.
- * - a TRANSPORT_ERROR is simulated on successful read and write
- * commands if SDEBUG_OPT_TRANSPORT_ERR is set.
- * - similarly for DIF_ERR, DIX_ERR, SHORT_TRANSFER, HOST_BUSY and
- * CMD_ABORT
- *
- * When "every_nth" < 0 then after "- every_nth" commands the selected
- * error will be injected. The error will be injected on every subsequent
- * command until some other action occurs; for example, the user writing
- * a new value (other than -1 or 1) to every_nth:
- * echo 0 > /sys/bus/pseudo/drivers/scsi_debug/every_nth
- */
+#define SDEBUG_OPT_RECOV_DIF_DIX (SDEBUG_OPT_RECOVERED_ERR | \
+ SDEBUG_OPT_DIF_ERR | SDEBUG_OPT_DIX_ERR)
/* As indicated in SAM-5 and SPC-4 Unit Attentions (UAs) are returned in
* priority order. In the subset implemented here lower numbers have higher
@@ -301,7 +289,7 @@
struct sdebug_host_info *sdbg_host;
unsigned long uas_bm[1];
atomic_t num_in_q;
- atomic_t stopped;
+ atomic_t stopped; /* 1: by SSU, 2: device start */
bool used;
/* For ZBC devices */
@@ -314,6 +302,7 @@
unsigned int nr_exp_open;
unsigned int nr_closed;
unsigned int max_open;
+ ktime_t create_ts; /* time since bootup that this device was created */
struct sdeb_zone_state *zstate;
};
@@ -344,6 +333,7 @@
struct execute_work ew;
int sqa_idx; /* index of sdebug_queue array */
int qc_idx; /* index of sdebug_queued_cmd array within sqa_idx */
+ int hc_idx; /* hostwide tag index */
int issuing_cpu;
bool init_hrt;
bool init_wq;
@@ -357,13 +347,6 @@
*/
struct sdebug_defer *sd_dp;
struct scsi_cmnd *a_cmnd;
- unsigned int inj_recovered:1;
- unsigned int inj_transport:1;
- unsigned int inj_dif:1;
- unsigned int inj_dix:1;
- unsigned int inj_short:1;
- unsigned int inj_host_busy:1;
- unsigned int inj_cmd_abort:1;
};
struct sdebug_queue {
@@ -377,6 +360,7 @@
static atomic_t sdebug_completions; /* count of deferred completions */
static atomic_t sdebug_miss_cpus; /* submission + completion cpus differ */
static atomic_t sdebug_a_tsf; /* 'almost task set full' counter */
+static atomic_t sdeb_inject_pending;
struct opcode_info_t {
u8 num_attached; /* 0 if this is it (i.e. a leaf); use 0xff */
@@ -759,6 +743,7 @@
static int sdebug_every_nth = DEF_EVERY_NTH;
static int sdebug_fake_rw = DEF_FAKE_RW;
static unsigned int sdebug_guard = DEF_GUARD;
+static int sdebug_host_max_queue; /* per host */
static int sdebug_lowest_aligned = DEF_LOWEST_ALIGNED;
static int sdebug_max_luns = DEF_MAX_LUNS;
static int sdebug_max_queue = SDEBUG_CANQUEUE; /* per submit queue */
@@ -777,6 +762,7 @@
static int sdebug_ptype = DEF_PTYPE; /* SCSI peripheral device type */
static int sdebug_scsi_level = DEF_SCSI_LEVEL;
static int sdebug_sector_size = DEF_SECTOR_SIZE;
+static int sdeb_tur_ms_to_ready = DEF_TUR_MS_TO_READY;
static int sdebug_virtual_gb = DEF_VIRTUAL_GB;
static int sdebug_vpd_use_hostno = DEF_VPD_USE_HOSTNO;
static unsigned int sdebug_lbpu = DEF_LBPU;
@@ -1729,75 +1715,68 @@
return ret;
}
+/* See resp_iec_m_pg() for how this data is manipulated */
static unsigned char iec_m_pg[] = {0x1c, 0xa, 0x08, 0, 0, 0, 0, 0,
0, 0, 0x0, 0x0};
static int resp_requests(struct scsi_cmnd *scp,
struct sdebug_dev_info *devip)
{
- unsigned char *sbuff;
unsigned char *cmd = scp->cmnd;
- unsigned char arr[SCSI_SENSE_BUFFERSIZE];
- bool dsense;
+ unsigned char arr[SCSI_SENSE_BUFFERSIZE]; /* assume >= 18 bytes */
+ bool dsense = !!(cmd[1] & 1);
+ int alloc_len = cmd[4];
int len = 18;
+ int stopped_state = atomic_read(&devip->stopped);
memset(arr, 0, sizeof(arr));
- dsense = !!(cmd[1] & 1);
- sbuff = scp->sense_buffer;
- if ((iec_m_pg[2] & 0x4) && (6 == (iec_m_pg[3] & 0xf))) {
+ if (stopped_state > 0) { /* some "pollable" data [spc6r02: 5.12.2] */
+ if (dsense) {
+ arr[0] = 0x72;
+ arr[1] = NOT_READY;
+ arr[2] = LOGICAL_UNIT_NOT_READY;
+ arr[3] = (stopped_state == 2) ? 0x1 : 0x2;
+ len = 8;
+ } else {
+ arr[0] = 0x70;
+ arr[2] = NOT_READY; /* NO_SENSE in sense_key */
+ arr[7] = 0xa; /* 18 byte sense buffer */
+ arr[12] = LOGICAL_UNIT_NOT_READY;
+ arr[13] = (stopped_state == 2) ? 0x1 : 0x2;
+ }
+ } else if ((iec_m_pg[2] & 0x4) && (6 == (iec_m_pg[3] & 0xf))) {
+ /* Information exceptions control mode page: TEST=1, MRIE=6 */
if (dsense) {
arr[0] = 0x72;
arr[1] = 0x0; /* NO_SENSE in sense_key */
arr[2] = THRESHOLD_EXCEEDED;
- arr[3] = 0xff; /* TEST set and MRIE==6 */
+ arr[3] = 0xff; /* Failure prediction(false) */
len = 8;
} else {
arr[0] = 0x70;
arr[2] = 0x0; /* NO_SENSE in sense_key */
arr[7] = 0xa; /* 18 byte sense buffer */
arr[12] = THRESHOLD_EXCEEDED;
- arr[13] = 0xff; /* TEST set and MRIE==6 */
+ arr[13] = 0xff; /* Failure prediction(false) */
}
- } else {
- memcpy(arr, sbuff, SCSI_SENSE_BUFFERSIZE);
- if (arr[0] >= 0x70 && dsense == sdebug_dsense)
- ; /* have sense and formats match */
- else if (arr[0] <= 0x70) {
- if (dsense) {
- memset(arr, 0, 8);
- arr[0] = 0x72;
- len = 8;
- } else {
- memset(arr, 0, 18);
- arr[0] = 0x70;
- arr[7] = 0xa;
- }
- } else if (dsense) {
- memset(arr, 0, 8);
- arr[0] = 0x72;
- arr[1] = sbuff[2]; /* sense key */
- arr[2] = sbuff[12]; /* asc */
- arr[3] = sbuff[13]; /* ascq */
+ } else { /* nothing to report */
+ if (dsense) {
len = 8;
+ memset(arr, 0, len);
+ arr[0] = 0x72;
} else {
- memset(arr, 0, 18);
+ memset(arr, 0, len);
arr[0] = 0x70;
- arr[2] = sbuff[1];
arr[7] = 0xa;
- arr[12] = sbuff[1];
- arr[13] = sbuff[3];
}
-
}
- mk_sense_buffer(scp, 0, NO_ADDITIONAL_SENSE, 0);
- return fill_from_dev_buffer(scp, arr, len);
+ return fill_from_dev_buffer(scp, arr, min_t(int, len, alloc_len));
}
-static int resp_start_stop(struct scsi_cmnd *scp,
- struct sdebug_dev_info *devip)
+static int resp_start_stop(struct scsi_cmnd *scp, struct sdebug_dev_info *devip)
{
unsigned char *cmd = scp->cmnd;
- int power_cond, stop;
+ int power_cond, want_stop, stopped_state;
bool changing;
power_cond = (cmd[4] & 0xf0) >> 4;
@@ -1805,10 +1784,33 @@
mk_sense_invalid_fld(scp, SDEB_IN_CDB, 4, 7);
return check_condition_result;
}
- stop = !(cmd[4] & 1);
- changing = atomic_read(&devip->stopped) == !stop;
- atomic_xchg(&devip->stopped, stop);
- if (!changing || cmd[1] & 0x1) /* state unchanged or IMMED set */
+ want_stop = !(cmd[4] & 1);
+ stopped_state = atomic_read(&devip->stopped);
+ if (stopped_state == 2) {
+ ktime_t now_ts = ktime_get_boottime();
+
+ if (ktime_to_ns(now_ts) > ktime_to_ns(devip->create_ts)) {
+ u64 diff_ns = ktime_to_ns(ktime_sub(now_ts, devip->create_ts));
+
+ if (diff_ns >= ((u64)sdeb_tur_ms_to_ready * 1000000)) {
+ /* tur_ms_to_ready timer extinguished */
+ atomic_set(&devip->stopped, 0);
+ stopped_state = 0;
+ }
+ }
+ if (stopped_state == 2) {
+ if (want_stop) {
+ stopped_state = 1; /* dummy up success */
+ } else { /* Disallow tur_ms_to_ready delay to be overridden */
+ mk_sense_invalid_fld(scp, SDEB_IN_CDB, 4, 0 /* START bit */);
+ return check_condition_result;
+ }
+ }
+ }
+ changing = (stopped_state != want_stop);
+ if (changing)
+ atomic_xchg(&devip->stopped, want_stop);
+ if (!changing || (cmd[1] & 0x1)) /* state unchanged or IMMED bit set in cdb */
return SDEG_RES_IMMED_MASK;
else
return 0;
@@ -3109,7 +3111,6 @@
struct sdeb_store_info *sip = devip2sip(devip, true);
rwlock_t *macc_lckp = sip ? &sip->macc_lck : &sdeb_fake_rw_lck;
u8 *cmd = scp->cmnd;
- struct sdebug_queued_cmd *sqcp;
switch (cmd[0]) {
case READ_16:
@@ -3162,15 +3163,11 @@
sdev_printk(KERN_ERR, scp->device, "Unprotected RD "
"to DIF device\n");
}
- if (unlikely(sdebug_any_injecting_opt)) {
- sqcp = (struct sdebug_queued_cmd *)scp->host_scribble;
-
- if (sqcp) {
- if (sqcp->inj_short)
- num /= 2;
- }
- } else
- sqcp = NULL;
+ if (unlikely((sdebug_opts & SDEBUG_OPT_SHORT_TRANSFER) &&
+ atomic_read(&sdeb_inject_pending))) {
+ num /= 2;
+ atomic_set(&sdeb_inject_pending, 0);
+ }
ret = check_device_access_params(scp, lba, num, false);
if (ret)
@@ -3211,21 +3208,20 @@
scsi_set_resid(scp, scsi_bufflen(scp) - ret);
- if (unlikely(sqcp)) {
- if (sqcp->inj_recovered) {
- mk_sense_buffer(scp, RECOVERED_ERROR,
- THRESHOLD_EXCEEDED, 0);
+ if (unlikely((sdebug_opts & SDEBUG_OPT_RECOV_DIF_DIX) &&
+ atomic_read(&sdeb_inject_pending))) {
+ if (sdebug_opts & SDEBUG_OPT_RECOVERED_ERR) {
+ mk_sense_buffer(scp, RECOVERED_ERROR, THRESHOLD_EXCEEDED, 0);
+ atomic_set(&sdeb_inject_pending, 0);
return check_condition_result;
- } else if (sqcp->inj_transport) {
- mk_sense_buffer(scp, ABORTED_COMMAND,
- TRANSPORT_PROBLEM, ACK_NAK_TO);
- return check_condition_result;
- } else if (sqcp->inj_dif) {
+ } else if (sdebug_opts & SDEBUG_OPT_DIF_ERR) {
/* Logical block guard check failed */
mk_sense_buffer(scp, ABORTED_COMMAND, 0x10, 1);
+ atomic_set(&sdeb_inject_pending, 0);
return illegal_condition_result;
- } else if (sqcp->inj_dix) {
+ } else if (SDEBUG_OPT_DIX_ERR & sdebug_opts) {
mk_sense_buffer(scp, ILLEGAL_REQUEST, 0x10, 1);
+ atomic_set(&sdeb_inject_pending, 0);
return illegal_condition_result;
}
}
@@ -3504,23 +3500,21 @@
"%s: write: cdb indicated=%u, IO sent=%d bytes\n",
my_name, num * sdebug_sector_size, ret);
- if (unlikely(sdebug_any_injecting_opt)) {
- struct sdebug_queued_cmd *sqcp =
- (struct sdebug_queued_cmd *)scp->host_scribble;
-
- if (sqcp) {
- if (sqcp->inj_recovered) {
- mk_sense_buffer(scp, RECOVERED_ERROR,
- THRESHOLD_EXCEEDED, 0);
- return check_condition_result;
- } else if (sqcp->inj_dif) {
- /* Logical block guard check failed */
- mk_sense_buffer(scp, ABORTED_COMMAND, 0x10, 1);
- return illegal_condition_result;
- } else if (sqcp->inj_dix) {
- mk_sense_buffer(scp, ILLEGAL_REQUEST, 0x10, 1);
- return illegal_condition_result;
- }
+ if (unlikely((sdebug_opts & SDEBUG_OPT_RECOV_DIF_DIX) &&
+ atomic_read(&sdeb_inject_pending))) {
+ if (sdebug_opts & SDEBUG_OPT_RECOVERED_ERR) {
+ mk_sense_buffer(scp, RECOVERED_ERROR, THRESHOLD_EXCEEDED, 0);
+ atomic_set(&sdeb_inject_pending, 0);
+ return check_condition_result;
+ } else if (sdebug_opts & SDEBUG_OPT_DIF_ERR) {
+ /* Logical block guard check failed */
+ mk_sense_buffer(scp, ABORTED_COMMAND, 0x10, 1);
+ atomic_set(&sdeb_inject_pending, 0);
+ return illegal_condition_result;
+ } else if (sdebug_opts & SDEBUG_OPT_DIX_ERR) {
+ mk_sense_buffer(scp, ILLEGAL_REQUEST, 0x10, 1);
+ atomic_set(&sdeb_inject_pending, 0);
+ return illegal_condition_result;
}
}
return 0;
@@ -3662,28 +3656,24 @@
"%s: write: cdb indicated=%u, IO sent=%d bytes\n",
my_name, num_by, ret);
- if (unlikely(sdebug_any_injecting_opt)) {
- struct sdebug_queued_cmd *sqcp =
- (struct sdebug_queued_cmd *)scp->host_scribble;
-
- if (sqcp) {
- if (sqcp->inj_recovered) {
- mk_sense_buffer(scp, RECOVERED_ERROR,
- THRESHOLD_EXCEEDED, 0);
- ret = illegal_condition_result;
- goto err_out_unlock;
- } else if (sqcp->inj_dif) {
- /* Logical block guard check failed */
- mk_sense_buffer(scp, ABORTED_COMMAND,
- 0x10, 1);
- ret = illegal_condition_result;
- goto err_out_unlock;
- } else if (sqcp->inj_dix) {
- mk_sense_buffer(scp, ILLEGAL_REQUEST,
- 0x10, 1);
- ret = illegal_condition_result;
- goto err_out_unlock;
- }
+ if (unlikely((sdebug_opts & SDEBUG_OPT_RECOV_DIF_DIX) &&
+ atomic_read(&sdeb_inject_pending))) {
+ if (sdebug_opts & SDEBUG_OPT_RECOVERED_ERR) {
+ mk_sense_buffer(scp, RECOVERED_ERROR, THRESHOLD_EXCEEDED, 0);
+ atomic_set(&sdeb_inject_pending, 0);
+ ret = check_condition_result;
+ goto err_out_unlock;
+ } else if (sdebug_opts & SDEBUG_OPT_DIF_ERR) {
+ /* Logical block guard check failed */
+ mk_sense_buffer(scp, ABORTED_COMMAND, 0x10, 1);
+ atomic_set(&sdeb_inject_pending, 0);
+ ret = illegal_condition_result;
+ goto err_out_unlock;
+ } else if (sdebug_opts & SDEBUG_OPT_DIX_ERR) {
+ mk_sense_buffer(scp, ILLEGAL_REQUEST, 0x10, 1);
+ atomic_set(&sdeb_inject_pending, 0);
+ ret = illegal_condition_result;
+ goto err_out_unlock;
}
}
sg_off += num_by;
@@ -4049,7 +4039,7 @@
mk_sense_buffer(scp, ILLEGAL_REQUEST, LBA_OUT_OF_RANGE, 0);
return check_condition_result;
}
- if (!write_since_sync || cmd[1] & 0x2)
+ if (!write_since_sync || (cmd[1] & 0x2))
res = SDEG_RES_IMMED_MASK;
else /* delay if write_since_sync and IMMED clear */
write_since_sync = false;
@@ -4707,15 +4697,28 @@
static struct sdebug_queue *get_queue(struct scsi_cmnd *cmnd)
{
- u32 tag = blk_mq_unique_tag(cmnd->request);
- u16 hwq = blk_mq_unique_tag_to_hwq(tag);
+ u16 hwq;
- pr_debug("tag=%#x, hwq=%d\n", tag, hwq);
- if (WARN_ON_ONCE(hwq >= submit_queues))
- hwq = 0;
+ if (sdebug_host_max_queue) {
+ /* Provide a simple method to choose the hwq */
+ hwq = smp_processor_id() % submit_queues;
+ } else {
+ u32 tag = blk_mq_unique_tag(cmnd->request);
+
+ hwq = blk_mq_unique_tag_to_hwq(tag);
+
+ pr_debug("tag=%#x, hwq=%d\n", tag, hwq);
+ if (WARN_ON_ONCE(hwq >= submit_queues))
+ hwq = 0;
+ }
return sdebug_q_arr + hwq;
}
+static u32 get_tag(struct scsi_cmnd *cmnd)
+{
+ return blk_mq_unique_tag(cmnd->request);
+}
+
/* Queued (deferred) command completions converge here. */
static void sdebug_q_cmd_complete(struct sdebug_defer *sd_dp)
{
@@ -4747,8 +4750,8 @@
scp = sqcp->a_cmnd;
if (unlikely(scp == NULL)) {
spin_unlock_irqrestore(&sqp->qc_lock, iflags);
- pr_err("scp is NULL, sqa_idx=%d, qc_idx=%d\n",
- sd_dp->sqa_idx, qc_idx);
+ pr_err("scp is NULL, sqa_idx=%d, qc_idx=%d, hc_idx=%d\n",
+ sd_dp->sqa_idx, qc_idx, sd_dp->hc_idx);
return;
}
devip = (struct sdebug_dev_info *)scp->device->hostdata;
@@ -4925,6 +4928,8 @@
devip->zmodel = BLK_ZONED_NONE;
}
devip->sdbg_host = sdbg_host;
+ devip->create_ts = ktime_get_boottime();
+ atomic_set(&devip->stopped, (sdeb_tur_ms_to_ready > 0 ? 2 : 0));
list_add_tail(&devip->dev_list, &sdbg_host->dev_info_list);
}
return devip;
@@ -5333,24 +5338,11 @@
atomic_set(&sdebug_a_tsf, 0);
}
-static void setup_inject(struct sdebug_queue *sqp,
- struct sdebug_queued_cmd *sqcp)
+static bool inject_on_this_cmd(void)
{
- if ((atomic_read(&sdebug_cmnd_count) % abs(sdebug_every_nth)) > 0) {
- if (sdebug_every_nth > 0)
- sqcp->inj_recovered = sqcp->inj_transport
- = sqcp->inj_dif
- = sqcp->inj_dix = sqcp->inj_short
- = sqcp->inj_host_busy = sqcp->inj_cmd_abort = 0;
- return;
- }
- sqcp->inj_recovered = !!(SDEBUG_OPT_RECOVERED_ERR & sdebug_opts);
- sqcp->inj_transport = !!(SDEBUG_OPT_TRANSPORT_ERR & sdebug_opts);
- sqcp->inj_dif = !!(SDEBUG_OPT_DIF_ERR & sdebug_opts);
- sqcp->inj_dix = !!(SDEBUG_OPT_DIX_ERR & sdebug_opts);
- sqcp->inj_short = !!(SDEBUG_OPT_SHORT_TRANSFER & sdebug_opts);
- sqcp->inj_host_busy = !!(SDEBUG_OPT_HOST_BUSY & sdebug_opts);
- sqcp->inj_cmd_abort = !!(SDEBUG_OPT_CMD_ABORT & sdebug_opts);
+ if (sdebug_every_nth == 0)
+ return false;
+ return (atomic_read(&sdebug_cmnd_count) % abs(sdebug_every_nth)) == 0;
}
#define INCLUSIVE_TIMING_MAX_NS 1000000 /* 1 millisecond */
@@ -5367,7 +5359,8 @@
int delta_jiff, int ndelay)
{
bool new_sd_dp;
- int k, num_in_q, qdepth, inject;
+ bool inject = false;
+ int k, num_in_q, qdepth;
unsigned long iflags;
u64 ns_from_boot = 0;
struct sdebug_queue *sqp;
@@ -5393,7 +5386,6 @@
}
num_in_q = atomic_read(&devip->num_in_q);
qdepth = cmnd->device->queue_depth;
- inject = 0;
if (unlikely((qdepth > 0) && (num_in_q >= qdepth))) {
if (scsi_result) {
spin_unlock_irqrestore(&sqp->qc_lock, iflags);
@@ -5407,7 +5399,7 @@
(atomic_inc_return(&sdebug_a_tsf) >=
abs(sdebug_every_nth))) {
atomic_set(&sdebug_a_tsf, 0);
- inject = 1;
+ inject = true;
scsi_result = device_qfull_result;
}
}
@@ -5430,35 +5422,47 @@
else
return SCSI_MLQUEUE_HOST_BUSY;
}
- __set_bit(k, sqp->in_use_bm);
+ set_bit(k, sqp->in_use_bm);
atomic_inc(&devip->num_in_q);
sqcp = &sqp->qc_arr[k];
sqcp->a_cmnd = cmnd;
cmnd->host_scribble = (unsigned char *)sqcp;
sd_dp = sqcp->sd_dp;
spin_unlock_irqrestore(&sqp->qc_lock, iflags);
- if (unlikely(sdebug_every_nth && sdebug_any_injecting_opt))
- setup_inject(sqp, sqcp);
- if (sd_dp == NULL) {
+ if (!sd_dp) {
sd_dp = kzalloc(sizeof(*sd_dp), GFP_ATOMIC);
- if (sd_dp == NULL)
+ if (!sd_dp) {
+ atomic_dec(&devip->num_in_q);
+ clear_bit(k, sqp->in_use_bm);
return SCSI_MLQUEUE_HOST_BUSY;
+ }
new_sd_dp = true;
} else {
new_sd_dp = false;
}
+ /* Set the hostwide tag */
+ if (sdebug_host_max_queue)
+ sd_dp->hc_idx = get_tag(cmnd);
+
if (ndelay > 0 && ndelay < INCLUSIVE_TIMING_MAX_NS)
ns_from_boot = ktime_get_boottime_ns();
/* one of the resp_*() response functions is called here */
- cmnd->result = pfp != NULL ? pfp(cmnd, devip) : 0;
+ cmnd->result = pfp ? pfp(cmnd, devip) : 0;
if (cmnd->result & SDEG_RES_IMMED_MASK) {
cmnd->result &= ~SDEG_RES_IMMED_MASK;
delta_jiff = ndelay = 0;
}
if (cmnd->result == 0 && scsi_result != 0)
cmnd->result = scsi_result;
+ if (cmnd->result == 0 && unlikely(sdebug_opts & SDEBUG_OPT_TRANSPORT_ERR)) {
+ if (atomic_read(&sdeb_inject_pending)) {
+ mk_sense_buffer(cmnd, ABORTED_COMMAND, TRANSPORT_PROBLEM, ACK_NAK_TO);
+ atomic_set(&sdeb_inject_pending, 0);
+ cmnd->result = check_condition_result;
+ }
+ }
if (unlikely(sdebug_verbose && cmnd->result))
sdev_printk(KERN_INFO, sdp, "%s: non-zero result=0x%x\n",
@@ -5524,21 +5528,20 @@
if (sdebug_statistics)
sd_dp->issuing_cpu = raw_smp_processor_id();
sd_dp->defer_t = SDEB_DEFER_WQ;
- if (unlikely(sqcp->inj_cmd_abort))
+ if (unlikely((sdebug_opts & SDEBUG_OPT_CMD_ABORT) &&
+ atomic_read(&sdeb_inject_pending)))
sd_dp->aborted = true;
schedule_work(&sd_dp->ew.work);
- if (unlikely(sqcp->inj_cmd_abort)) {
- sdev_printk(KERN_INFO, sdp, "abort request tag %d\n",
- cmnd->request->tag);
+ if (unlikely((sdebug_opts & SDEBUG_OPT_CMD_ABORT) &&
+ atomic_read(&sdeb_inject_pending))) {
+ sdev_printk(KERN_INFO, sdp, "abort request tag %d\n", cmnd->request->tag);
blk_abort_request(cmnd->request);
+ atomic_set(&sdeb_inject_pending, 0);
}
}
- if (unlikely((SDEBUG_OPT_Q_NOISE & sdebug_opts) &&
- (scsi_result == device_qfull_result)))
- sdev_printk(KERN_INFO, sdp,
- "%s: num_in_q=%d +1, %s%s\n", __func__,
- num_in_q, (inject ? "<inject> " : ""),
- "status: TASK SET FULL");
+ if (unlikely((SDEBUG_OPT_Q_NOISE & sdebug_opts) && scsi_result == device_qfull_result))
+ sdev_printk(KERN_INFO, sdp, "%s: num_in_q=%d +1, %s%s\n", __func__,
+ num_in_q, (inject ? "<inject> " : ""), "status: TASK SET FULL");
return 0;
respond_in_thread: /* call back to mid-layer using invocation thread */
@@ -5569,6 +5572,7 @@
module_param_named(fake_rw, sdebug_fake_rw, int, S_IRUGO | S_IWUSR);
module_param_named(guard, sdebug_guard, uint, S_IRUGO);
module_param_named(host_lock, sdebug_host_lock, bool, S_IRUGO | S_IWUSR);
+module_param_named(host_max_queue, sdebug_host_max_queue, int, S_IRUGO);
module_param_string(inq_product, sdebug_inq_product_id,
sizeof(sdebug_inq_product_id), S_IRUGO | S_IWUSR);
module_param_string(inq_rev, sdebug_inq_product_rev,
@@ -5605,6 +5609,7 @@
module_param_named(statistics, sdebug_statistics, bool, S_IRUGO | S_IWUSR);
module_param_named(strict, sdebug_strict, bool, S_IRUGO | S_IWUSR);
module_param_named(submit_queues, submit_queues, int, S_IRUGO);
+module_param_named(tur_ms_to_ready, sdeb_tur_ms_to_ready, int, S_IRUGO);
module_param_named(unmap_alignment, sdebug_unmap_alignment, int, S_IRUGO);
module_param_named(unmap_granularity, sdebug_unmap_granularity, int, S_IRUGO);
module_param_named(unmap_max_blocks, sdebug_unmap_max_blocks, int, S_IRUGO);
@@ -5639,6 +5644,8 @@
MODULE_PARM_DESC(fake_rw, "fake reads/writes instead of copying (def=0)");
MODULE_PARM_DESC(guard, "protection checksum: 0=crc, 1=ip (def=0)");
MODULE_PARM_DESC(host_lock, "host_lock is ignored (def=0)");
+MODULE_PARM_DESC(host_max_queue,
+ "host max # of queued cmds (0 to max(def) [max_queue fixed equal for !0])");
MODULE_PARM_DESC(inq_product, "SCSI INQUIRY product string (def=\"scsi_debug\")");
MODULE_PARM_DESC(inq_rev, "SCSI INQUIRY revision string (def=\""
SDEBUG_VERSION "\")");
@@ -5671,6 +5678,7 @@
MODULE_PARM_DESC(statistics, "collect statistics on commands, queues (def=0)");
MODULE_PARM_DESC(strict, "stricter checks: reserved field in cdb (def=0)");
MODULE_PARM_DESC(submit_queues, "support for block multi-queue (def=1)");
+MODULE_PARM_DESC(tur_ms_to_ready, "TEST UNIT READY millisecs before initial good status (def=0)");
MODULE_PARM_DESC(unmap_alignment, "lowest aligned thin provisioning lba (def=0)");
MODULE_PARM_DESC(unmap_granularity, "thin provisioning granularity in blocks (def=1)");
MODULE_PARM_DESC(unmap_max_blocks, "max # of blocks can be unmapped in one cmd (def=0xffffffff)");
@@ -6072,17 +6080,27 @@
size_t count)
{
int nth;
+ char work[20];
- if ((count > 0) && (1 == sscanf(buf, "%d", &nth))) {
- sdebug_every_nth = nth;
- if (nth && !sdebug_statistics) {
- pr_info("every_nth needs statistics=1, set it\n");
- sdebug_statistics = true;
+ if (sscanf(buf, "%10s", work) == 1) {
+ if (strncasecmp(work, "0x", 2) == 0) {
+ if (kstrtoint(work + 2, 16, &nth) == 0)
+ goto every_nth_done;
+ } else {
+ if (kstrtoint(work, 10, &nth) == 0)
+ goto every_nth_done;
}
- tweak_cmnd_count();
- return count;
}
return -EINVAL;
+
+every_nth_done:
+ sdebug_every_nth = nth;
+ if (nth && !sdebug_statistics) {
+ pr_info("every_nth needs statistics=1, set it\n");
+ sdebug_statistics = true;
+ }
+ tweak_cmnd_count();
+ return count;
}
static DRIVER_ATTR_RW(every_nth);
@@ -6138,7 +6156,8 @@
struct sdebug_queue *sqp;
if ((count > 0) && (1 == sscanf(buf, "%d", &n)) && (n > 0) &&
- (n <= SDEBUG_CANQUEUE)) {
+ (n <= SDEBUG_CANQUEUE) &&
+ (sdebug_host_max_queue == 0)) {
block_unblock_all_queues(true);
k = 0;
for (j = 0, sqp = sdebug_q_arr; j < submit_queues;
@@ -6161,6 +6180,17 @@
}
static DRIVER_ATTR_RW(max_queue);
+static ssize_t host_max_queue_show(struct device_driver *ddp, char *buf)
+{
+ return scnprintf(buf, PAGE_SIZE, "%d\n", sdebug_host_max_queue);
+}
+
+/*
+ * Since this is used for .can_queue, and we get the hc_idx tag from the bitmap
+ * in range [0, sdebug_host_max_queue), we can't change it.
+ */
+static DRIVER_ATTR_RO(host_max_queue);
+
static ssize_t no_uld_show(struct device_driver *ddp, char *buf)
{
return scnprintf(buf, PAGE_SIZE, "%d\n", sdebug_no_uld);
@@ -6487,6 +6517,12 @@
}
static DRIVER_ATTR_RO(zbc);
+static ssize_t tur_ms_to_ready_show(struct device_driver *ddp, char *buf)
+{
+ return scnprintf(buf, PAGE_SIZE, "%d\n", sdeb_tur_ms_to_ready);
+}
+static DRIVER_ATTR_RO(tur_ms_to_ready);
+
/* Note: The following array creates attribute files in the
/sys/bus/pseudo/drivers/scsi_debug directory. The advantage of these
files (over those found in the /sys/module/scsi_debug/parameters
@@ -6500,6 +6536,7 @@
&driver_attr_ptype.attr,
&driver_attr_dsense.attr,
&driver_attr_fake_rw.attr,
+ &driver_attr_host_max_queue.attr,
&driver_attr_no_lun_0.attr,
&driver_attr_num_tgts.attr,
&driver_attr_dev_size_mb.attr,
@@ -6528,6 +6565,7 @@
&driver_attr_strict.attr,
&driver_attr_uuid_ctl.attr,
&driver_attr_cdb_len.attr,
+ &driver_attr_tur_ms_to_ready.attr,
&driver_attr_zbc.attr,
NULL,
};
@@ -6610,6 +6648,26 @@
pr_err("submit_queues must be 1 or more\n");
return -EINVAL;
}
+
+ if ((sdebug_max_queue > SDEBUG_CANQUEUE) || (sdebug_max_queue < 1)) {
+ pr_err("max_queue must be in range [1, %d]\n", SDEBUG_CANQUEUE);
+ return -EINVAL;
+ }
+
+ if ((sdebug_host_max_queue > SDEBUG_CANQUEUE) ||
+ (sdebug_host_max_queue < 0)) {
+ pr_err("host_max_queue must be in range [0 %d]\n",
+ SDEBUG_CANQUEUE);
+ return -EINVAL;
+ }
+
+ if (sdebug_host_max_queue &&
+ (sdebug_max_queue != sdebug_host_max_queue)) {
+ sdebug_max_queue = sdebug_host_max_queue;
+ pr_warn("fixing max submit queue depth to host max queue depth, %d\n",
+ sdebug_max_queue);
+ }
+
sdebug_q_arr = kcalloc(submit_queues, sizeof(struct sdebug_queue),
GFP_KERNEL);
if (sdebug_q_arr == NULL)
@@ -7044,10 +7102,47 @@
return false;
}
-static bool fake_host_busy(struct scsi_cmnd *scp)
+/* Response to TUR or media access command when device stopped */
+static int resp_not_ready(struct scsi_cmnd *scp, struct sdebug_dev_info *devip)
{
- return (sdebug_opts & SDEBUG_OPT_HOST_BUSY) &&
- (atomic_read(&sdebug_cmnd_count) % abs(sdebug_every_nth)) == 0;
+ int stopped_state;
+ u64 diff_ns = 0;
+ ktime_t now_ts = ktime_get_boottime();
+ struct scsi_device *sdp = scp->device;
+
+ stopped_state = atomic_read(&devip->stopped);
+ if (stopped_state == 2) {
+ if (ktime_to_ns(now_ts) > ktime_to_ns(devip->create_ts)) {
+ diff_ns = ktime_to_ns(ktime_sub(now_ts, devip->create_ts));
+ if (diff_ns >= ((u64)sdeb_tur_ms_to_ready * 1000000)) {
+ /* tur_ms_to_ready timer extinguished */
+ atomic_set(&devip->stopped, 0);
+ return 0;
+ }
+ }
+ mk_sense_buffer(scp, NOT_READY, LOGICAL_UNIT_NOT_READY, 0x1);
+ if (sdebug_verbose)
+ sdev_printk(KERN_INFO, sdp,
+ "%s: Not ready: in process of becoming ready\n", my_name);
+ if (scp->cmnd[0] == TEST_UNIT_READY) {
+ u64 tur_nanosecs_to_ready = (u64)sdeb_tur_ms_to_ready * 1000000;
+
+ if (diff_ns <= tur_nanosecs_to_ready)
+ diff_ns = tur_nanosecs_to_ready - diff_ns;
+ else
+ diff_ns = tur_nanosecs_to_ready;
+ /* As per 20-061r2 approved for spc6 by T10 on 20200716 */
+ do_div(diff_ns, 1000000); /* diff_ns becomes milliseconds */
+ scsi_set_sense_information(scp->sense_buffer, SCSI_SENSE_BUFFERSIZE,
+ diff_ns);
+ return check_condition_result;
+ }
+ }
+ mk_sense_buffer(scp, NOT_READY, LOGICAL_UNIT_NOT_READY, 0x2);
+ if (sdebug_verbose)
+ sdev_printk(KERN_INFO, sdp, "%s: Not ready: initializing command required\n",
+ my_name);
+ return check_condition_result;
}
static int scsi_debug_queuecommand(struct Scsi_Host *shost,
@@ -7058,7 +7153,6 @@
const struct opcode_info_t *oip;
const struct opcode_info_t *r_oip;
struct sdebug_dev_info *devip;
-
u8 *cmd = scp->cmnd;
int (*r_pfp)(struct scsi_cmnd *, struct sdebug_dev_info *);
int (*pfp)(struct scsi_cmnd *, struct sdebug_dev_info *) = NULL;
@@ -7068,10 +7162,15 @@
u16 sa;
u8 opcode = cmd[0];
bool has_wlun_rl;
+ bool inject_now;
scsi_set_resid(scp, 0);
- if (sdebug_statistics)
+ if (sdebug_statistics) {
atomic_inc(&sdebug_cmnd_count);
+ inject_now = inject_on_this_cmd();
+ } else {
+ inject_now = false;
+ }
if (unlikely(sdebug_verbose &&
!(SDEBUG_OPT_NO_CDB_NOISE & sdebug_opts))) {
char b[120];
@@ -7089,7 +7188,7 @@
sdev_printk(KERN_INFO, sdp, "%s: tag=%#x, cmd %s\n", my_name,
blk_mq_unique_tag(scp->request), b);
}
- if (fake_host_busy(scp))
+ if (unlikely(inject_now && (sdebug_opts & SDEBUG_OPT_HOST_BUSY)))
return SCSI_MLQUEUE_HOST_BUSY;
has_wlun_rl = (sdp->lun == SCSI_W_LUN_REPORT_LUNS);
if (unlikely((sdp->lun >= sdebug_max_luns) && !has_wlun_rl))
@@ -7103,6 +7202,9 @@
if (NULL == devip)
goto err_out;
}
+ if (unlikely(inject_now && !atomic_read(&sdeb_inject_pending)))
+ atomic_set(&sdeb_inject_pending, 1);
+
na = oip->num_attached;
r_pfp = oip->pfp;
if (na) { /* multiple commands with this opcode */
@@ -7167,14 +7269,11 @@
if (errsts)
goto check_cond;
}
- if (unlikely((F_M_ACCESS & flags) && atomic_read(&devip->stopped))) {
- mk_sense_buffer(scp, NOT_READY, LOGICAL_UNIT_NOT_READY, 0x2);
- if (sdebug_verbose)
- sdev_printk(KERN_INFO, sdp, "%s reports: Not ready: "
- "%s\n", my_name, "initializing command "
- "required");
- errsts = check_condition_result;
- goto fini;
+ if (unlikely(((F_M_ACCESS & flags) || scp->cmnd[0] == TEST_UNIT_READY) &&
+ atomic_read(&devip->stopped))) {
+ errsts = resp_not_ready(scp, devip);
+ if (errsts)
+ goto fini;
}
if (sdebug_fake_rw && (F_FAKE_RW & flags))
goto fini;
@@ -7248,7 +7347,10 @@
sdbg_host = to_sdebug_host(dev);
- sdebug_driver_template.can_queue = sdebug_max_queue;
+ if (sdebug_host_max_queue)
+ sdebug_driver_template.can_queue = sdebug_host_max_queue;
+ else
+ sdebug_driver_template.can_queue = sdebug_max_queue;
if (!sdebug_clustering)
sdebug_driver_template.dma_boundary = PAGE_SIZE - 1;
@@ -7263,9 +7365,13 @@
my_name, submit_queues, nr_cpu_ids);
submit_queues = nr_cpu_ids;
}
- /* Decide whether to tell scsi subsystem that we want mq */
- /* Following should give the same answer for each host */
- hpnt->nr_hw_queues = submit_queues;
+ /*
+ * Decide whether to tell scsi subsystem that we want mq. The
+ * following should give the same answer for each host. If the host
+ * has a limit of hostwide max commands, then do not set.
+ */
+ if (!sdebug_host_max_queue)
+ hpnt->nr_hw_queues = submit_queues;
sdbg_host->shost = hpnt;
*((struct sdebug_host_info **)hpnt->hostdata) = sdbg_host;
diff --git a/drivers/scsi/scsi_lib.c b/drivers/scsi/scsi_lib.c
index c866a4f..7c6dd6f 100644
--- a/drivers/scsi/scsi_lib.c
+++ b/drivers/scsi/scsi_lib.c
@@ -52,7 +52,6 @@
#define SCSI_INLINE_SG_CNT 2
#endif
-static struct kmem_cache *scsi_sdb_cache;
static struct kmem_cache *scsi_sense_cache;
static struct kmem_cache *scsi_sense_isadma_cache;
static DEFINE_MUTEX(scsi_sense_cache_mutex);
@@ -390,7 +389,7 @@
spin_unlock_irqrestore(shost->host_lock, flags);
scsi_kick_queue(sdev->request_queue);
spin_lock_irqsave(shost->host_lock, flags);
-
+
scsi_device_put(sdev);
}
out:
@@ -1461,18 +1460,18 @@
scsi_log_completion(cmd, disposition);
switch (disposition) {
- case SUCCESS:
- scsi_finish_command(cmd);
- break;
- case NEEDS_RETRY:
- scsi_queue_insert(cmd, SCSI_MLQUEUE_EH_RETRY);
- break;
- case ADD_TO_MLQUEUE:
- scsi_queue_insert(cmd, SCSI_MLQUEUE_DEVICE_BUSY);
- break;
- default:
- scsi_eh_scmd_add(cmd);
- break;
+ case SUCCESS:
+ scsi_finish_command(cmd);
+ break;
+ case NEEDS_RETRY:
+ scsi_queue_insert(cmd, SCSI_MLQUEUE_EH_RETRY);
+ break;
+ case ADD_TO_MLQUEUE:
+ scsi_queue_insert(cmd, SCSI_MLQUEUE_DEVICE_BUSY);
+ break;
+ default:
+ scsi_eh_scmd_add(cmd);
+ break;
}
}
@@ -1953,24 +1952,10 @@
}
EXPORT_SYMBOL(scsi_unblock_requests);
-int __init scsi_init_queue(void)
-{
- scsi_sdb_cache = kmem_cache_create("scsi_data_buffer",
- sizeof(struct scsi_data_buffer),
- 0, 0, NULL);
- if (!scsi_sdb_cache) {
- printk(KERN_ERR "SCSI: can't init scsi sdb cache\n");
- return -ENOMEM;
- }
-
- return 0;
-}
-
void scsi_exit_queue(void)
{
kmem_cache_destroy(scsi_sense_cache);
kmem_cache_destroy(scsi_sense_isadma_cache);
- kmem_cache_destroy(scsi_sdb_cache);
}
/**
@@ -2037,7 +2022,6 @@
real_buffer[1] = data->medium_type;
real_buffer[2] = data->device_specific;
real_buffer[3] = data->block_descriptor_length;
-
cmd[0] = MODE_SELECT;
cmd[4] = len;
@@ -2123,7 +2107,7 @@
if (scsi_sense_valid(sshdr)) {
if ((sshdr->sense_key == ILLEGAL_REQUEST) &&
(sshdr->asc == 0x20) && (sshdr->ascq == 0)) {
- /*
+ /*
* Invalid command operation code
*/
sdev->use_10_for_ms = 0;
@@ -2132,7 +2116,7 @@
}
}
- if(scsi_status_is_good(result)) {
+ if (scsi_status_is_good(result)) {
if (unlikely(buffer[0] == 0x86 && buffer[1] == 0x0b &&
(modepage == 6 || modepage == 8))) {
/* Initio breakage? */
@@ -2142,7 +2126,7 @@
data->device_specific = 0;
data->longlba = 0;
data->block_descriptor_length = 0;
- } else if(use_10_for_ms) {
+ } else if (use_10_for_ms) {
data->length = buffer[0]*256 + buffer[1] + 2;
data->medium_type = buffer[2];
data->device_specific = buffer[3];
@@ -2225,7 +2209,7 @@
goto illegal;
}
break;
-
+
case SDEV_RUNNING:
switch (oldstate) {
case SDEV_CREATED:
@@ -2510,7 +2494,7 @@
* (which must be a legal transition). When the device is in this
* state, only special requests will be accepted, all others will
* be deferred. Since special requests may also be requeued requests,
- * a successful return doesn't guarantee the device will be
+ * a successful return doesn't guarantee the device will be
* totally quiescent.
*
* Must be called with user context, may sleep.
@@ -2636,10 +2620,10 @@
return err;
}
- /*
+ /*
* The device has transitioned to SDEV_BLOCK. Stop the
* block layer from calling the midlayer with this device's
- * request queue.
+ * request queue.
*/
blk_mq_quiesce_queue_nowait(q);
return 0;
@@ -2674,7 +2658,7 @@
return err;
}
-
+
void scsi_start_queue(struct scsi_device *sdev)
{
struct request_queue *q = sdev->request_queue;
@@ -2833,8 +2817,10 @@
mutex_lock(&sdev->state_mutex);
ret = scsi_internal_device_block_nowait(sdev);
mutex_unlock(&sdev->state_mutex);
- if (ret)
+ if (ret) {
+ scsi_device_put(sdev);
break;
+ }
}
/*
diff --git a/drivers/scsi/scsi_logging.c b/drivers/scsi/scsi_logging.c
index c91fa3f..8ea44c6 100644
--- a/drivers/scsi/scsi_logging.c
+++ b/drivers/scsi/scsi_logging.c
@@ -205,13 +205,9 @@
/* Print opcode in one line and use separate lines for CDB */
off += scnprintf(logbuf + off, logbuf_len - off, "\n");
dev_printk(KERN_INFO, &cmd->device->sdev_gendev, "%s", logbuf);
- scsi_log_release_buffer(logbuf);
for (k = 0; k < cmd->cmd_len; k += 16) {
size_t linelen = min(cmd->cmd_len - k, 16);
- logbuf = scsi_log_reserve_buffer(&logbuf_len);
- if (!logbuf)
- break;
off = sdev_format_header(logbuf, logbuf_len,
scmd_name(cmd),
cmd->request->tag);
@@ -224,9 +220,8 @@
}
dev_printk(KERN_INFO, &cmd->device->sdev_gendev, "%s",
logbuf);
- scsi_log_release_buffer(logbuf);
}
- return;
+ goto out;
}
if (!WARN_ON(off > logbuf_len - 49)) {
off += scnprintf(logbuf + off, logbuf_len - off, " ");
@@ -236,6 +231,7 @@
}
out_printk:
dev_printk(KERN_INFO, &cmd->device->sdev_gendev, "%s", logbuf);
+out:
scsi_log_release_buffer(logbuf);
}
EXPORT_SYMBOL(scsi_print_command);
diff --git a/drivers/scsi/scsi_pm.c b/drivers/scsi/scsi_pm.c
index 5f0ad8b..3717eea 100644
--- a/drivers/scsi/scsi_pm.c
+++ b/drivers/scsi/scsi_pm.c
@@ -80,10 +80,6 @@
dev_dbg(dev, "scsi resume: %d\n", err);
if (err == 0) {
- bool was_runtime_suspended;
-
- was_runtime_suspended = pm_runtime_suspended(dev);
-
pm_runtime_disable(dev);
err = pm_runtime_set_active(dev);
pm_runtime_enable(dev);
@@ -97,10 +93,8 @@
*/
if (!err && scsi_is_sdev_device(dev)) {
struct scsi_device *sdev = to_scsi_device(dev);
- if (was_runtime_suspended)
- blk_post_runtime_resume(sdev->request_queue, 0);
- else
- blk_set_runtime_active(sdev->request_queue);
+
+ blk_set_runtime_active(sdev->request_queue);
}
}
diff --git a/drivers/scsi/scsi_priv.h b/drivers/scsi/scsi_priv.h
index 22b6585..d12ada0 100644
--- a/drivers/scsi/scsi_priv.h
+++ b/drivers/scsi/scsi_priv.h
@@ -93,7 +93,6 @@
extern void scsi_start_queue(struct scsi_device *sdev);
extern int scsi_mq_setup_tags(struct Scsi_Host *shost);
extern void scsi_mq_destroy_tags(struct Scsi_Host *shost);
-extern int scsi_init_queue(void);
extern void scsi_exit_queue(void);
extern void scsi_evt_thread(struct work_struct *work);
struct request_queue;
diff --git a/drivers/scsi/scsi_transport_iscsi.c b/drivers/scsi/scsi_transport_iscsi.c
index 7ae5024..2eb3e4f 100644
--- a/drivers/scsi/scsi_transport_iscsi.c
+++ b/drivers/scsi/scsi_transport_iscsi.c
@@ -1623,7 +1623,6 @@
static DEFINE_MUTEX(conn_mutex);
static LIST_HEAD(sesslist);
-static LIST_HEAD(sessdestroylist);
static DEFINE_SPINLOCK(sesslock);
static LIST_HEAD(connlist);
static LIST_HEAD(connlist_err);
@@ -1978,10 +1977,11 @@
{
queue_work(iscsi_eh_timer_workq, &session->unblock_work);
/*
- * make sure all the events have completed before tell the driver
- * it is safe
+ * Blocking the session can be done from any context so we only
+ * queue the block work. Make sure the unblock work has completed
+ * because it flushes/cancels the other works and updates the state.
*/
- flush_workqueue(iscsi_eh_timer_workq);
+ flush_work(&session->unblock_work);
}
EXPORT_SYMBOL_GPL(iscsi_unblock_session);
@@ -2036,11 +2036,11 @@
spin_unlock_irqrestore(&session->lock, flags);
mutex_unlock(&ihost->mutex);
+ scsi_remove_target(&session->dev);
+
if (session->ida_used)
ida_simple_remove(&iscsi_sess_ida, target_id);
- scsi_remove_target(&session->dev);
-
unbind_session_exit:
iscsi_session_event(session, ISCSI_KEVENT_UNBIND_SESSION);
ISCSI_DBG_TRANS_SESSION(session, "Completed target removal\n");
@@ -2202,14 +2202,13 @@
ISCSI_DBG_TRANS_SESSION(session, "Removing session\n");
spin_lock_irqsave(&sesslock, flags);
- list_del(&session->sess_list);
+ if (!list_empty(&session->sess_list))
+ list_del(&session->sess_list);
spin_unlock_irqrestore(&sesslock, flags);
- /* make sure there are no blocks/unblocks queued */
- flush_workqueue(iscsi_eh_timer_workq);
- /* make sure the timedout callout is not running */
- if (!cancel_delayed_work(&session->recovery_work))
- flush_workqueue(iscsi_eh_timer_workq);
+ flush_work(&session->block_work);
+ flush_work(&session->unblock_work);
+ cancel_delayed_work_sync(&session->recovery_work);
/*
* If we are blocked let commands flow again. The lld or iscsi
* layer should set up the queuecommand to fail commands.
@@ -3291,7 +3290,7 @@
pr_err("%s could not find host no %u\n",
__func__, ev->u.set_flashnode.host_no);
err = -ENODEV;
- goto put_host;
+ goto exit_set_fnode;
}
idx = ev->u.set_flashnode.flashnode_idx;
@@ -3679,7 +3678,7 @@
/* Prevent this session from being found again */
spin_lock_irqsave(&sesslock, flags);
- list_move(&session->sess_list, &sessdestroylist);
+ list_del_init(&session->sess_list);
spin_unlock_irqrestore(&sesslock, flags);
queue_work(iscsi_destroy_workq, &session->destroy_work);
@@ -4766,7 +4765,9 @@
goto release_nls;
}
- iscsi_destroy_workq = create_singlethread_workqueue("iscsi_destroy");
+ iscsi_destroy_workq = alloc_workqueue("%s",
+ WQ_SYSFS | __WQ_LEGACY | WQ_MEM_RECLAIM | WQ_UNBOUND,
+ 1, "iscsi_destroy");
if (!iscsi_destroy_workq) {
err = -ENOMEM;
goto destroy_wq;
diff --git a/drivers/scsi/scsi_transport_sas.c b/drivers/scsi/scsi_transport_sas.c
index 182fd25..e443dee 100644
--- a/drivers/scsi/scsi_transport_sas.c
+++ b/drivers/scsi/scsi_transport_sas.c
@@ -563,7 +563,7 @@
{
struct sas_phy *phy = transport_class_to_phy(dev);
- return snprintf(buf, 20, "%d", phy->enabled);
+ return snprintf(buf, 20, "%d\n", phy->enabled);
}
static DEVICE_ATTR(enable, S_IRUGO | S_IWUSR, show_sas_phy_enable,
diff --git a/drivers/scsi/scsi_transport_srp.c b/drivers/scsi/scsi_transport_srp.c
index d4d1104..cba1cf6 100644
--- a/drivers/scsi/scsi_transport_srp.c
+++ b/drivers/scsi/scsi_transport_srp.c
@@ -395,6 +395,10 @@
}
}
+/*
+ * scsi_target_block() must have been called before this function is
+ * called to guarantee that no .queuecommand() calls are in progress.
+ */
static void __rport_fail_io_fast(struct srp_rport *rport)
{
struct Scsi_Host *shost = rport_to_shost(rport);
@@ -404,11 +408,7 @@
if (srp_rport_set_state(rport, SRP_RPORT_FAIL_FAST))
return;
- /*
- * Call scsi_target_block() to wait for ongoing shost->queuecommand()
- * calls before invoking i->f->terminate_rport_io().
- */
- scsi_target_block(rport->dev.parent);
+
scsi_target_unblock(rport->dev.parent, SDEV_TRANSPORT_OFFLINE);
/* Involve the LLD if possible to terminate all I/O on the rport. */
@@ -570,8 +570,6 @@
* failure timers if these had not yet been started.
*/
__rport_fail_io_fast(rport);
- scsi_target_unblock(&shost->shost_gendev,
- SDEV_TRANSPORT_OFFLINE);
__srp_start_tl_fail_timers(rport);
} else if (rport->state != SRP_RPORT_BLOCKED) {
scsi_target_unblock(&shost->shost_gendev,
diff --git a/drivers/scsi/sd.c b/drivers/scsi/sd.c
index d90feff..acde0ca 100644
--- a/drivers/scsi/sd.c
+++ b/drivers/scsi/sd.c
@@ -1479,7 +1479,7 @@
* @bdev: target block device
* @mode: FMODE_* mask
* @cmd: ioctl command number
- * @arg: this is third argument given to ioctl(2) system call.
+ * @p: this is third argument given to ioctl(2) system call.
* Often contains a pointer.
*
* Returns 0 if successful (some ioctls return positive numbers on
diff --git a/drivers/scsi/sd.h b/drivers/scsi/sd.h
index 3a74f4b..27c0f4e 100644
--- a/drivers/scsi/sd.h
+++ b/drivers/scsi/sd.h
@@ -229,17 +229,11 @@
#else /* CONFIG_BLK_DEV_ZONED */
-static inline int sd_zbc_init(void)
-{
- return 0;
-}
-
static inline int sd_zbc_init_disk(struct scsi_disk *sdkp)
{
return 0;
}
-static inline void sd_zbc_exit(void) {}
static inline void sd_zbc_release_disk(struct scsi_disk *sdkp) {}
static inline int sd_zbc_read_zones(struct scsi_disk *sdkp,
diff --git a/drivers/scsi/sd_zbc.c b/drivers/scsi/sd_zbc.c
index d8b2c49..4717e79 100644
--- a/drivers/scsi/sd_zbc.c
+++ b/drivers/scsi/sd_zbc.c
@@ -585,7 +585,7 @@
* sd_zbc_check_capacity - Check the device capacity
* @sdkp: Target disk
* @buf: command buffer
- * @zblock: zone size in number of blocks
+ * @zblocks: zone size in number of blocks
*
* Get the device zone size and check that the device capacity as reported
* by READ CAPACITY matches the max_lba value (plus one) of the report zones
@@ -742,7 +742,6 @@
max_append = min_t(u32, logical_to_sectors(sdkp->device, zone_blocks),
q->limits.max_segments << (PAGE_SHIFT - 9));
- max_append = min_t(u32, max_append, queue_max_hw_sectors(q));
blk_queue_max_zone_append_sectors(q, max_append);
diff --git a/drivers/scsi/storvsc_drv.c b/drivers/scsi/storvsc_drv.c
index 624467e..8f5f5dc 100644
--- a/drivers/scsi/storvsc_drv.c
+++ b/drivers/scsi/storvsc_drv.c
@@ -1038,7 +1038,7 @@
do_work = true;
process_err_fn = storvsc_device_scan;
/*
- * Retry the I/O that trigerred this.
+ * Retry the I/O that triggered this.
*/
set_host_byte(scmnd, DID_REQUEUE);
}
@@ -1105,6 +1105,10 @@
data_transfer_length = 0;
}
+ /* Validate data_transfer_length (from Hyper-V) */
+ if (data_transfer_length > cmd_request->payload->range.len)
+ data_transfer_length = cmd_request->payload->range.len;
+
scsi_set_resid(scmnd,
cmd_request->payload->range.len - data_transfer_length);
@@ -1145,6 +1149,11 @@
/* Copy over the status...etc */
stor_pkt->vm_srb.scsi_status = vstor_packet->vm_srb.scsi_status;
stor_pkt->vm_srb.srb_status = vstor_packet->vm_srb.srb_status;
+
+ /* Validate sense_info_length (from Hyper-V) */
+ if (vstor_packet->vm_srb.sense_info_length > sense_buffer_size)
+ vstor_packet->vm_srb.sense_info_length = sense_buffer_size;
+
stor_pkt->vm_srb.sense_info_length =
vstor_packet->vm_srb.sense_info_length;
@@ -1570,6 +1579,7 @@
request = &stor_device->reset_request;
vstor_packet = &request->vstor_packet;
+ memset(vstor_packet, 0, sizeof(struct vstor_packet));
init_completion(&request->wait_event);
@@ -1673,6 +1683,7 @@
/* Setup the cmd request */
cmd_request->cmd = scmnd;
+ memset(&cmd_request->vstor_packet, 0, sizeof(struct vstor_packet));
vm_srb = &cmd_request->vstor_packet.vm_srb;
vm_srb->win8_extension.time_out_value = 60;
diff --git a/drivers/scsi/sym53c8xx_2/sym_glue.c b/drivers/scsi/sym53c8xx_2/sym_glue.c
index 2ca018c..af20ad9 100644
--- a/drivers/scsi/sym53c8xx_2/sym_glue.c
+++ b/drivers/scsi/sym53c8xx_2/sym_glue.c
@@ -1774,6 +1774,7 @@
/**
* sym2_reset_workarounds - hardware-specific work-arounds
+ * @pdev: pointer to PCI device
*
* This routine is similar to sym_set_workarounds(), except
* that, at this point, we already know that the device was
diff --git a/drivers/scsi/sym53c8xx_2/sym_hipd.c b/drivers/scsi/sym53c8xx_2/sym_hipd.c
index a428cae..8410117 100644
--- a/drivers/scsi/sym53c8xx_2/sym_hipd.c
+++ b/drivers/scsi/sym53c8xx_2/sym_hipd.c
@@ -5352,8 +5352,10 @@
{
struct scsi_device *sdev;
struct scsi_cmnd *cmd;
+#ifdef SYM_OPT_HANDLE_DEVICE_QUEUEING
struct sym_tcb *tp;
struct sym_lcb *lp;
+#endif
int resid;
int i;
@@ -5370,11 +5372,13 @@
cp->host_status, cp->ssss_status, cp->host_flags);
}
+#ifdef SYM_OPT_HANDLE_DEVICE_QUEUEING
/*
* Get target and lun pointers.
*/
tp = &np->target[cp->target];
lp = sym_lp(tp, sdev->lun);
+#endif
/*
* Check for extended errors.
@@ -5481,8 +5485,10 @@
*/
void sym_complete_ok (struct sym_hcb *np, struct sym_ccb *cp)
{
+#ifdef SYM_OPT_HANDLE_DEVICE_QUEUEING
struct sym_tcb *tp;
struct sym_lcb *lp;
+#endif
struct scsi_cmnd *cmd;
int resid;
@@ -5498,11 +5504,13 @@
*/
cmd = cp->cmd;
+#ifdef SYM_OPT_HANDLE_DEVICE_QUEUEING
/*
* Get target and lun pointers.
*/
tp = &np->target[cp->target];
lp = sym_lp(tp, cp->lun);
+#endif
/*
* If all data have been transferred, given than no
diff --git a/drivers/scsi/ufs/Kconfig b/drivers/scsi/ufs/Kconfig
index d35378b..f639499 100644
--- a/drivers/scsi/ufs/Kconfig
+++ b/drivers/scsi/ufs/Kconfig
@@ -99,6 +99,7 @@
config SCSI_UFS_QCOM
tristate "QCOM specific hooks to UFS controller platform driver"
depends on SCSI_UFSHCD_PLATFORM && ARCH_QCOM
+ select QCOM_SCM
select RESET_CONTROLLER
help
This selects the QCOM specific additions to UFSHCD platform driver.
@@ -160,3 +161,24 @@
Select this if you need a bsg device node for your UFS controller.
If unsure, say N.
+
+config SCSI_UFS_EXYNOS
+ tristate "EXYNOS specific hooks to UFS controller platform driver"
+ depends on SCSI_UFSHCD_PLATFORM && (ARCH_EXYNOS || COMPILE_TEST)
+ select PHY_SAMSUNG_UFS
+ help
+ This selects the EXYNOS specific additions to UFSHCD platform driver.
+ UFS host on EXYNOS includes HCI and UNIPRO layer, and associates with
+ UFS-PHY driver.
+
+ Select this if you have UFS host controller on EXYNOS chipset.
+ If unsure, say N.
+
+config SCSI_UFS_CRYPTO
+ bool "UFS Crypto Engine Support"
+ depends on SCSI_UFSHCD && BLK_INLINE_ENCRYPTION
+ help
+ Enable Crypto Engine Support in UFS.
+ Enabling this makes it possible for the kernel to use the crypto
+ capabilities of the UFS device (if present) to perform crypto
+ operations on data being transferred to/from the device.
diff --git a/drivers/scsi/ufs/Makefile b/drivers/scsi/ufs/Makefile
index 94c6c5d..4679af1 100644
--- a/drivers/scsi/ufs/Makefile
+++ b/drivers/scsi/ufs/Makefile
@@ -3,10 +3,14 @@
obj-$(CONFIG_SCSI_UFS_DWC_TC_PCI) += tc-dwc-g210-pci.o ufshcd-dwc.o tc-dwc-g210.o
obj-$(CONFIG_SCSI_UFS_DWC_TC_PLATFORM) += tc-dwc-g210-pltfrm.o ufshcd-dwc.o tc-dwc-g210.o
obj-$(CONFIG_SCSI_UFS_CDNS_PLATFORM) += cdns-pltfrm.o
-obj-$(CONFIG_SCSI_UFS_QCOM) += ufs-qcom.o
+obj-$(CONFIG_SCSI_UFS_QCOM) += ufs_qcom.o
+ufs_qcom-y += ufs-qcom.o
+ufs_qcom-$(CONFIG_SCSI_UFS_CRYPTO) += ufs-qcom-ice.o
+obj-$(CONFIG_SCSI_UFS_EXYNOS) += ufs-exynos.o
obj-$(CONFIG_SCSI_UFSHCD) += ufshcd-core.o
ufshcd-core-y += ufshcd.o ufs-sysfs.o
ufshcd-core-$(CONFIG_SCSI_UFS_BSG) += ufs_bsg.o
+ufshcd-core-$(CONFIG_SCSI_UFS_CRYPTO) += ufshcd-crypto.o
obj-$(CONFIG_SCSI_UFSHCD_PCI) += ufshcd-pci.o
obj-$(CONFIG_SCSI_UFSHCD_PLATFORM) += ufshcd-pltfrm.o
obj-$(CONFIG_SCSI_UFS_HISI) += ufs-hisi.o
diff --git a/drivers/scsi/ufs/ufs-exynos.c b/drivers/scsi/ufs/ufs-exynos.c
new file mode 100644
index 0000000..8f1b6f6
--- /dev/null
+++ b/drivers/scsi/ufs/ufs-exynos.c
@@ -0,0 +1,1297 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * UFS Host Controller driver for Exynos specific extensions
+ *
+ * Copyright (C) 2014-2015 Samsung Electronics Co., Ltd.
+ * Author: Seungwon Jeon <essuuj@gmail.com>
+ * Author: Alim Akhtar <alim.akhtar@samsung.com>
+ *
+ */
+
+#include <linux/clk.h>
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/of_address.h>
+#include <linux/phy/phy.h>
+#include <linux/platform_device.h>
+
+#include "ufshcd.h"
+#include "ufshcd-pltfrm.h"
+#include "ufshci.h"
+#include "unipro.h"
+
+#include "ufs-exynos.h"
+
+/*
+ * Exynos's Vendor specific registers for UFSHCI
+ */
+#define HCI_TXPRDT_ENTRY_SIZE 0x00
+#define PRDT_PREFECT_EN BIT(31)
+#define PRDT_SET_SIZE(x) ((x) & 0x1F)
+#define HCI_RXPRDT_ENTRY_SIZE 0x04
+#define HCI_1US_TO_CNT_VAL 0x0C
+#define CNT_VAL_1US_MASK 0x3FF
+#define HCI_UTRL_NEXUS_TYPE 0x40
+#define HCI_UTMRL_NEXUS_TYPE 0x44
+#define HCI_SW_RST 0x50
+#define UFS_LINK_SW_RST BIT(0)
+#define UFS_UNIPRO_SW_RST BIT(1)
+#define UFS_SW_RST_MASK (UFS_UNIPRO_SW_RST | UFS_LINK_SW_RST)
+#define HCI_DATA_REORDER 0x60
+#define HCI_UNIPRO_APB_CLK_CTRL 0x68
+#define UNIPRO_APB_CLK(v, x) (((v) & ~0xF) | ((x) & 0xF))
+#define HCI_AXIDMA_RWDATA_BURST_LEN 0x6C
+#define HCI_GPIO_OUT 0x70
+#define HCI_ERR_EN_PA_LAYER 0x78
+#define HCI_ERR_EN_DL_LAYER 0x7C
+#define HCI_ERR_EN_N_LAYER 0x80
+#define HCI_ERR_EN_T_LAYER 0x84
+#define HCI_ERR_EN_DME_LAYER 0x88
+#define HCI_CLKSTOP_CTRL 0xB0
+#define REFCLK_STOP BIT(2)
+#define UNIPRO_MCLK_STOP BIT(1)
+#define UNIPRO_PCLK_STOP BIT(0)
+#define CLK_STOP_MASK (REFCLK_STOP |\
+ UNIPRO_MCLK_STOP |\
+ UNIPRO_PCLK_STOP)
+#define HCI_MISC 0xB4
+#define REFCLK_CTRL_EN BIT(7)
+#define UNIPRO_PCLK_CTRL_EN BIT(6)
+#define UNIPRO_MCLK_CTRL_EN BIT(5)
+#define HCI_CORECLK_CTRL_EN BIT(4)
+#define CLK_CTRL_EN_MASK (REFCLK_CTRL_EN |\
+ UNIPRO_PCLK_CTRL_EN |\
+ UNIPRO_MCLK_CTRL_EN)
+/* Device fatal error */
+#define DFES_ERR_EN BIT(31)
+#define DFES_DEF_L2_ERRS (UIC_DATA_LINK_LAYER_ERROR_RX_BUF_OF |\
+ UIC_DATA_LINK_LAYER_ERROR_PA_INIT)
+#define DFES_DEF_L3_ERRS (UIC_NETWORK_UNSUPPORTED_HEADER_TYPE |\
+ UIC_NETWORK_BAD_DEVICEID_ENC |\
+ UIC_NETWORK_LHDR_TRAP_PACKET_DROPPING)
+#define DFES_DEF_L4_ERRS (UIC_TRANSPORT_UNSUPPORTED_HEADER_TYPE |\
+ UIC_TRANSPORT_UNKNOWN_CPORTID |\
+ UIC_TRANSPORT_NO_CONNECTION_RX |\
+ UIC_TRANSPORT_BAD_TC)
+
+enum {
+ UNIPRO_L1_5 = 0,/* PHY Adapter */
+ UNIPRO_L2, /* Data Link */
+ UNIPRO_L3, /* Network */
+ UNIPRO_L4, /* Transport */
+ UNIPRO_DME, /* DME */
+};
+
+/*
+ * UNIPRO registers
+ */
+#define UNIPRO_COMP_VERSION 0x000
+#define UNIPRO_DME_PWR_REQ 0x090
+#define UNIPRO_DME_PWR_REQ_POWERMODE 0x094
+#define UNIPRO_DME_PWR_REQ_LOCALL2TIMER0 0x098
+#define UNIPRO_DME_PWR_REQ_LOCALL2TIMER1 0x09C
+#define UNIPRO_DME_PWR_REQ_LOCALL2TIMER2 0x0A0
+#define UNIPRO_DME_PWR_REQ_REMOTEL2TIMER0 0x0A4
+#define UNIPRO_DME_PWR_REQ_REMOTEL2TIMER1 0x0A8
+#define UNIPRO_DME_PWR_REQ_REMOTEL2TIMER2 0x0AC
+
+/*
+ * UFS Protector registers
+ */
+#define UFSPRSECURITY 0x010
+#define NSSMU BIT(14)
+#define UFSPSBEGIN0 0x200
+#define UFSPSEND0 0x204
+#define UFSPSLUN0 0x208
+#define UFSPSCTRL0 0x20C
+
+#define CNTR_DIV_VAL 40
+
+static void exynos_ufs_auto_ctrl_hcc(struct exynos_ufs *ufs, bool en);
+static void exynos_ufs_ctrl_clkstop(struct exynos_ufs *ufs, bool en);
+
+static inline void exynos_ufs_enable_auto_ctrl_hcc(struct exynos_ufs *ufs)
+{
+ exynos_ufs_auto_ctrl_hcc(ufs, true);
+}
+
+static inline void exynos_ufs_disable_auto_ctrl_hcc(struct exynos_ufs *ufs)
+{
+ exynos_ufs_auto_ctrl_hcc(ufs, false);
+}
+
+static inline void exynos_ufs_disable_auto_ctrl_hcc_save(
+ struct exynos_ufs *ufs, u32 *val)
+{
+ *val = hci_readl(ufs, HCI_MISC);
+ exynos_ufs_auto_ctrl_hcc(ufs, false);
+}
+
+static inline void exynos_ufs_auto_ctrl_hcc_restore(
+ struct exynos_ufs *ufs, u32 *val)
+{
+ hci_writel(ufs, *val, HCI_MISC);
+}
+
+static inline void exynos_ufs_gate_clks(struct exynos_ufs *ufs)
+{
+ exynos_ufs_ctrl_clkstop(ufs, true);
+}
+
+static inline void exynos_ufs_ungate_clks(struct exynos_ufs *ufs)
+{
+ exynos_ufs_ctrl_clkstop(ufs, false);
+}
+
+static int exynos7_ufs_drv_init(struct device *dev, struct exynos_ufs *ufs)
+{
+ return 0;
+}
+
+static int exynos7_ufs_pre_link(struct exynos_ufs *ufs)
+{
+ struct ufs_hba *hba = ufs->hba;
+ u32 val = ufs->drv_data->uic_attr->pa_dbg_option_suite;
+ int i;
+
+ exynos_ufs_enable_ov_tm(hba);
+ for_each_ufs_tx_lane(ufs, i)
+ ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(0x297, i), 0x17);
+ for_each_ufs_rx_lane(ufs, i) {
+ ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(0x362, i), 0xff);
+ ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(0x363, i), 0x00);
+ }
+ exynos_ufs_disable_ov_tm(hba);
+
+ for_each_ufs_tx_lane(ufs, i)
+ ufshcd_dme_set(hba,
+ UIC_ARG_MIB_SEL(TX_HIBERN8_CONTROL, i), 0x0);
+ ufshcd_dme_set(hba, UIC_ARG_MIB(PA_DBG_TXPHY_CFGUPDT), 0x1);
+ udelay(1);
+ ufshcd_dme_set(hba, UIC_ARG_MIB(PA_DBG_OPTION_SUITE), val | (1 << 12));
+ ufshcd_dme_set(hba, UIC_ARG_MIB(PA_DBG_SKIP_RESET_PHY), 0x1);
+ ufshcd_dme_set(hba, UIC_ARG_MIB(PA_DBG_SKIP_LINE_RESET), 0x1);
+ ufshcd_dme_set(hba, UIC_ARG_MIB(PA_DBG_LINE_RESET_REQ), 0x1);
+ udelay(1600);
+ ufshcd_dme_set(hba, UIC_ARG_MIB(PA_DBG_OPTION_SUITE), val);
+
+ return 0;
+}
+
+static int exynos7_ufs_post_link(struct exynos_ufs *ufs)
+{
+ struct ufs_hba *hba = ufs->hba;
+ int i;
+
+ exynos_ufs_enable_ov_tm(hba);
+ for_each_ufs_tx_lane(ufs, i) {
+ ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(0x28b, i), 0x83);
+ ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(0x29a, i), 0x07);
+ ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(0x277, i),
+ TX_LINERESET_N(exynos_ufs_calc_time_cntr(ufs, 200000)));
+ }
+ exynos_ufs_disable_ov_tm(hba);
+
+ exynos_ufs_enable_dbg_mode(hba);
+ ufshcd_dme_set(hba, UIC_ARG_MIB(PA_SAVECONFIGTIME), 0xbb8);
+ exynos_ufs_disable_dbg_mode(hba);
+
+ return 0;
+}
+
+static int exynos7_ufs_pre_pwr_change(struct exynos_ufs *ufs,
+ struct ufs_pa_layer_attr *pwr)
+{
+ unipro_writel(ufs, 0x22, UNIPRO_DBG_FORCE_DME_CTRL_STATE);
+
+ return 0;
+}
+
+static int exynos7_ufs_post_pwr_change(struct exynos_ufs *ufs,
+ struct ufs_pa_layer_attr *pwr)
+{
+ struct ufs_hba *hba = ufs->hba;
+ int lanes = max_t(u32, pwr->lane_rx, pwr->lane_tx);
+
+ ufshcd_dme_set(hba, UIC_ARG_MIB(PA_DBG_RXPHY_CFGUPDT), 0x1);
+
+ if (lanes == 1) {
+ exynos_ufs_enable_dbg_mode(hba);
+ ufshcd_dme_set(hba, UIC_ARG_MIB(PA_CONNECTEDTXDATALANES), 0x1);
+ exynos_ufs_disable_dbg_mode(hba);
+ }
+
+ return 0;
+}
+
+/*
+ * exynos_ufs_auto_ctrl_hcc - HCI core clock control by h/w
+ * Control should be disabled in the below cases
+ * - Before host controller S/W reset
+ * - Access to UFS protector's register
+ */
+static void exynos_ufs_auto_ctrl_hcc(struct exynos_ufs *ufs, bool en)
+{
+ u32 misc = hci_readl(ufs, HCI_MISC);
+
+ if (en)
+ hci_writel(ufs, misc | HCI_CORECLK_CTRL_EN, HCI_MISC);
+ else
+ hci_writel(ufs, misc & ~HCI_CORECLK_CTRL_EN, HCI_MISC);
+}
+
+static void exynos_ufs_ctrl_clkstop(struct exynos_ufs *ufs, bool en)
+{
+ u32 ctrl = hci_readl(ufs, HCI_CLKSTOP_CTRL);
+ u32 misc = hci_readl(ufs, HCI_MISC);
+
+ if (en) {
+ hci_writel(ufs, misc | CLK_CTRL_EN_MASK, HCI_MISC);
+ hci_writel(ufs, ctrl | CLK_STOP_MASK, HCI_CLKSTOP_CTRL);
+ } else {
+ hci_writel(ufs, ctrl & ~CLK_STOP_MASK, HCI_CLKSTOP_CTRL);
+ hci_writel(ufs, misc & ~CLK_CTRL_EN_MASK, HCI_MISC);
+ }
+}
+
+static int exynos_ufs_get_clk_info(struct exynos_ufs *ufs)
+{
+ struct ufs_hba *hba = ufs->hba;
+ struct list_head *head = &hba->clk_list_head;
+ struct ufs_clk_info *clki;
+ u32 pclk_rate;
+ u32 f_min, f_max;
+ u8 div = 0;
+ int ret = 0;
+
+ if (list_empty(head))
+ goto out;
+
+ list_for_each_entry(clki, head, list) {
+ if (!IS_ERR(clki->clk)) {
+ if (!strcmp(clki->name, "core_clk"))
+ ufs->clk_hci_core = clki->clk;
+ else if (!strcmp(clki->name, "sclk_unipro_main"))
+ ufs->clk_unipro_main = clki->clk;
+ }
+ }
+
+ if (!ufs->clk_hci_core || !ufs->clk_unipro_main) {
+ dev_err(hba->dev, "failed to get clk info\n");
+ ret = -EINVAL;
+ goto out;
+ }
+
+ ufs->mclk_rate = clk_get_rate(ufs->clk_unipro_main);
+ pclk_rate = clk_get_rate(ufs->clk_hci_core);
+ f_min = ufs->pclk_avail_min;
+ f_max = ufs->pclk_avail_max;
+
+ if (ufs->opts & EXYNOS_UFS_OPT_HAS_APB_CLK_CTRL) {
+ do {
+ pclk_rate /= (div + 1);
+
+ if (pclk_rate <= f_max)
+ break;
+ div++;
+ } while (pclk_rate >= f_min);
+ }
+
+ if (unlikely(pclk_rate < f_min || pclk_rate > f_max)) {
+ dev_err(hba->dev, "not available pclk range %d\n", pclk_rate);
+ ret = -EINVAL;
+ goto out;
+ }
+
+ ufs->pclk_rate = pclk_rate;
+ ufs->pclk_div = div;
+
+out:
+ return ret;
+}
+
+static void exynos_ufs_set_unipro_pclk_div(struct exynos_ufs *ufs)
+{
+ if (ufs->opts & EXYNOS_UFS_OPT_HAS_APB_CLK_CTRL) {
+ u32 val;
+
+ val = hci_readl(ufs, HCI_UNIPRO_APB_CLK_CTRL);
+ hci_writel(ufs, UNIPRO_APB_CLK(val, ufs->pclk_div),
+ HCI_UNIPRO_APB_CLK_CTRL);
+ }
+}
+
+static void exynos_ufs_set_pwm_clk_div(struct exynos_ufs *ufs)
+{
+ struct ufs_hba *hba = ufs->hba;
+ struct exynos_ufs_uic_attr *attr = ufs->drv_data->uic_attr;
+
+ ufshcd_dme_set(hba,
+ UIC_ARG_MIB(CMN_PWM_CLK_CTRL), attr->cmn_pwm_clk_ctrl);
+}
+
+static void exynos_ufs_calc_pwm_clk_div(struct exynos_ufs *ufs)
+{
+ struct ufs_hba *hba = ufs->hba;
+ struct exynos_ufs_uic_attr *attr = ufs->drv_data->uic_attr;
+ const unsigned int div = 30, mult = 20;
+ const unsigned long pwm_min = 3 * 1000 * 1000;
+ const unsigned long pwm_max = 9 * 1000 * 1000;
+ const int divs[] = {32, 16, 8, 4};
+ unsigned long clk = 0, _clk, clk_period;
+ int i = 0, clk_idx = -1;
+
+ clk_period = UNIPRO_PCLK_PERIOD(ufs);
+ for (i = 0; i < ARRAY_SIZE(divs); i++) {
+ _clk = NSEC_PER_SEC * mult / (clk_period * divs[i] * div);
+ if (_clk >= pwm_min && _clk <= pwm_max) {
+ if (_clk > clk) {
+ clk_idx = i;
+ clk = _clk;
+ }
+ }
+ }
+
+ if (clk_idx == -1) {
+ ufshcd_dme_get(hba, UIC_ARG_MIB(CMN_PWM_CLK_CTRL), &clk_idx);
+ dev_err(hba->dev,
+ "failed to decide pwm clock divider, will not change\n");
+ }
+
+ attr->cmn_pwm_clk_ctrl = clk_idx & PWM_CLK_CTRL_MASK;
+}
+
+long exynos_ufs_calc_time_cntr(struct exynos_ufs *ufs, long period)
+{
+ const int precise = 10;
+ long pclk_rate = ufs->pclk_rate;
+ long clk_period, fraction;
+
+ clk_period = UNIPRO_PCLK_PERIOD(ufs);
+ fraction = ((NSEC_PER_SEC % pclk_rate) * precise) / pclk_rate;
+
+ return (period * precise) / ((clk_period * precise) + fraction);
+}
+
+static void exynos_ufs_specify_phy_time_attr(struct exynos_ufs *ufs)
+{
+ struct exynos_ufs_uic_attr *attr = ufs->drv_data->uic_attr;
+ struct ufs_phy_time_cfg *t_cfg = &ufs->t_cfg;
+
+ t_cfg->tx_linereset_p =
+ exynos_ufs_calc_time_cntr(ufs, attr->tx_dif_p_nsec);
+ t_cfg->tx_linereset_n =
+ exynos_ufs_calc_time_cntr(ufs, attr->tx_dif_n_nsec);
+ t_cfg->tx_high_z_cnt =
+ exynos_ufs_calc_time_cntr(ufs, attr->tx_high_z_cnt_nsec);
+ t_cfg->tx_base_n_val =
+ exynos_ufs_calc_time_cntr(ufs, attr->tx_base_unit_nsec);
+ t_cfg->tx_gran_n_val =
+ exynos_ufs_calc_time_cntr(ufs, attr->tx_gran_unit_nsec);
+ t_cfg->tx_sleep_cnt =
+ exynos_ufs_calc_time_cntr(ufs, attr->tx_sleep_cnt);
+
+ t_cfg->rx_linereset =
+ exynos_ufs_calc_time_cntr(ufs, attr->rx_dif_p_nsec);
+ t_cfg->rx_hibern8_wait =
+ exynos_ufs_calc_time_cntr(ufs, attr->rx_hibern8_wait_nsec);
+ t_cfg->rx_base_n_val =
+ exynos_ufs_calc_time_cntr(ufs, attr->rx_base_unit_nsec);
+ t_cfg->rx_gran_n_val =
+ exynos_ufs_calc_time_cntr(ufs, attr->rx_gran_unit_nsec);
+ t_cfg->rx_sleep_cnt =
+ exynos_ufs_calc_time_cntr(ufs, attr->rx_sleep_cnt);
+ t_cfg->rx_stall_cnt =
+ exynos_ufs_calc_time_cntr(ufs, attr->rx_stall_cnt);
+}
+
+static void exynos_ufs_config_phy_time_attr(struct exynos_ufs *ufs)
+{
+ struct ufs_hba *hba = ufs->hba;
+ struct ufs_phy_time_cfg *t_cfg = &ufs->t_cfg;
+ int i;
+
+ exynos_ufs_set_pwm_clk_div(ufs);
+
+ exynos_ufs_enable_ov_tm(hba);
+
+ for_each_ufs_rx_lane(ufs, i) {
+ ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(RX_FILLER_ENABLE, i),
+ ufs->drv_data->uic_attr->rx_filler_enable);
+ ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(RX_LINERESET_VAL, i),
+ RX_LINERESET(t_cfg->rx_linereset));
+ ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(RX_BASE_NVAL_07_00, i),
+ RX_BASE_NVAL_L(t_cfg->rx_base_n_val));
+ ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(RX_BASE_NVAL_15_08, i),
+ RX_BASE_NVAL_H(t_cfg->rx_base_n_val));
+ ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(RX_GRAN_NVAL_07_00, i),
+ RX_GRAN_NVAL_L(t_cfg->rx_gran_n_val));
+ ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(RX_GRAN_NVAL_10_08, i),
+ RX_GRAN_NVAL_H(t_cfg->rx_gran_n_val));
+ ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(RX_OV_SLEEP_CNT_TIMER, i),
+ RX_OV_SLEEP_CNT(t_cfg->rx_sleep_cnt));
+ ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(RX_OV_STALL_CNT_TIMER, i),
+ RX_OV_STALL_CNT(t_cfg->rx_stall_cnt));
+ }
+
+ for_each_ufs_tx_lane(ufs, i) {
+ ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(TX_LINERESET_P_VAL, i),
+ TX_LINERESET_P(t_cfg->tx_linereset_p));
+ ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(TX_HIGH_Z_CNT_07_00, i),
+ TX_HIGH_Z_CNT_L(t_cfg->tx_high_z_cnt));
+ ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(TX_HIGH_Z_CNT_11_08, i),
+ TX_HIGH_Z_CNT_H(t_cfg->tx_high_z_cnt));
+ ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(TX_BASE_NVAL_07_00, i),
+ TX_BASE_NVAL_L(t_cfg->tx_base_n_val));
+ ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(TX_BASE_NVAL_15_08, i),
+ TX_BASE_NVAL_H(t_cfg->tx_base_n_val));
+ ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(TX_GRAN_NVAL_07_00, i),
+ TX_GRAN_NVAL_L(t_cfg->tx_gran_n_val));
+ ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(TX_GRAN_NVAL_10_08, i),
+ TX_GRAN_NVAL_H(t_cfg->tx_gran_n_val));
+ ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(TX_OV_SLEEP_CNT_TIMER, i),
+ TX_OV_H8_ENTER_EN |
+ TX_OV_SLEEP_CNT(t_cfg->tx_sleep_cnt));
+ ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(TX_MIN_ACTIVATETIME, i),
+ ufs->drv_data->uic_attr->tx_min_activatetime);
+ }
+
+ exynos_ufs_disable_ov_tm(hba);
+}
+
+static void exynos_ufs_config_phy_cap_attr(struct exynos_ufs *ufs)
+{
+ struct ufs_hba *hba = ufs->hba;
+ struct exynos_ufs_uic_attr *attr = ufs->drv_data->uic_attr;
+ int i;
+
+ exynos_ufs_enable_ov_tm(hba);
+
+ for_each_ufs_rx_lane(ufs, i) {
+ ufshcd_dme_set(hba,
+ UIC_ARG_MIB_SEL(RX_HS_G1_SYNC_LENGTH_CAP, i),
+ attr->rx_hs_g1_sync_len_cap);
+ ufshcd_dme_set(hba,
+ UIC_ARG_MIB_SEL(RX_HS_G2_SYNC_LENGTH_CAP, i),
+ attr->rx_hs_g2_sync_len_cap);
+ ufshcd_dme_set(hba,
+ UIC_ARG_MIB_SEL(RX_HS_G3_SYNC_LENGTH_CAP, i),
+ attr->rx_hs_g3_sync_len_cap);
+ ufshcd_dme_set(hba,
+ UIC_ARG_MIB_SEL(RX_HS_G1_PREP_LENGTH_CAP, i),
+ attr->rx_hs_g1_prep_sync_len_cap);
+ ufshcd_dme_set(hba,
+ UIC_ARG_MIB_SEL(RX_HS_G2_PREP_LENGTH_CAP, i),
+ attr->rx_hs_g2_prep_sync_len_cap);
+ ufshcd_dme_set(hba,
+ UIC_ARG_MIB_SEL(RX_HS_G3_PREP_LENGTH_CAP, i),
+ attr->rx_hs_g3_prep_sync_len_cap);
+ }
+
+ if (attr->rx_adv_fine_gran_sup_en == 0) {
+ for_each_ufs_rx_lane(ufs, i) {
+ ufshcd_dme_set(hba,
+ UIC_ARG_MIB_SEL(RX_ADV_GRANULARITY_CAP, i), 0);
+
+ if (attr->rx_min_actv_time_cap)
+ ufshcd_dme_set(hba,
+ UIC_ARG_MIB_SEL(RX_MIN_ACTIVATETIME_CAP,
+ i), attr->rx_min_actv_time_cap);
+
+ if (attr->rx_hibern8_time_cap)
+ ufshcd_dme_set(hba,
+ UIC_ARG_MIB_SEL(RX_HIBERN8TIME_CAP, i),
+ attr->rx_hibern8_time_cap);
+ }
+ } else if (attr->rx_adv_fine_gran_sup_en == 1) {
+ for_each_ufs_rx_lane(ufs, i) {
+ if (attr->rx_adv_fine_gran_step)
+ ufshcd_dme_set(hba,
+ UIC_ARG_MIB_SEL(RX_ADV_GRANULARITY_CAP,
+ i), RX_ADV_FINE_GRAN_STEP(
+ attr->rx_adv_fine_gran_step));
+
+ if (attr->rx_adv_min_actv_time_cap)
+ ufshcd_dme_set(hba,
+ UIC_ARG_MIB_SEL(
+ RX_ADV_MIN_ACTIVATETIME_CAP, i),
+ attr->rx_adv_min_actv_time_cap);
+
+ if (attr->rx_adv_hibern8_time_cap)
+ ufshcd_dme_set(hba,
+ UIC_ARG_MIB_SEL(RX_ADV_HIBERN8TIME_CAP,
+ i),
+ attr->rx_adv_hibern8_time_cap);
+ }
+ }
+
+ exynos_ufs_disable_ov_tm(hba);
+}
+
+static void exynos_ufs_establish_connt(struct exynos_ufs *ufs)
+{
+ struct ufs_hba *hba = ufs->hba;
+ enum {
+ DEV_ID = 0x00,
+ PEER_DEV_ID = 0x01,
+ PEER_CPORT_ID = 0x00,
+ TRAFFIC_CLASS = 0x00,
+ };
+
+ /* allow cport attributes to be set */
+ ufshcd_dme_set(hba, UIC_ARG_MIB(T_CONNECTIONSTATE), CPORT_IDLE);
+
+ /* local unipro attributes */
+ ufshcd_dme_set(hba, UIC_ARG_MIB(N_DEVICEID), DEV_ID);
+ ufshcd_dme_set(hba, UIC_ARG_MIB(N_DEVICEID_VALID), TRUE);
+ ufshcd_dme_set(hba, UIC_ARG_MIB(T_PEERDEVICEID), PEER_DEV_ID);
+ ufshcd_dme_set(hba, UIC_ARG_MIB(T_PEERCPORTID), PEER_CPORT_ID);
+ ufshcd_dme_set(hba, UIC_ARG_MIB(T_CPORTFLAGS), CPORT_DEF_FLAGS);
+ ufshcd_dme_set(hba, UIC_ARG_MIB(T_TRAFFICCLASS), TRAFFIC_CLASS);
+ ufshcd_dme_set(hba, UIC_ARG_MIB(T_CONNECTIONSTATE), CPORT_CONNECTED);
+}
+
+static void exynos_ufs_config_smu(struct exynos_ufs *ufs)
+{
+ u32 reg, val;
+
+ exynos_ufs_disable_auto_ctrl_hcc_save(ufs, &val);
+
+ /* make encryption disabled by default */
+ reg = ufsp_readl(ufs, UFSPRSECURITY);
+ ufsp_writel(ufs, reg | NSSMU, UFSPRSECURITY);
+ ufsp_writel(ufs, 0x0, UFSPSBEGIN0);
+ ufsp_writel(ufs, 0xffffffff, UFSPSEND0);
+ ufsp_writel(ufs, 0xff, UFSPSLUN0);
+ ufsp_writel(ufs, 0xf1, UFSPSCTRL0);
+
+ exynos_ufs_auto_ctrl_hcc_restore(ufs, &val);
+}
+
+static void exynos_ufs_config_sync_pattern_mask(struct exynos_ufs *ufs,
+ struct ufs_pa_layer_attr *pwr)
+{
+ struct ufs_hba *hba = ufs->hba;
+ u8 g = max_t(u32, pwr->gear_rx, pwr->gear_tx);
+ u32 mask, sync_len;
+ enum {
+ SYNC_LEN_G1 = 80 * 1000, /* 80us */
+ SYNC_LEN_G2 = 40 * 1000, /* 44us */
+ SYNC_LEN_G3 = 20 * 1000, /* 20us */
+ };
+ int i;
+
+ if (g == 1)
+ sync_len = SYNC_LEN_G1;
+ else if (g == 2)
+ sync_len = SYNC_LEN_G2;
+ else if (g == 3)
+ sync_len = SYNC_LEN_G3;
+ else
+ return;
+
+ mask = exynos_ufs_calc_time_cntr(ufs, sync_len);
+ mask = (mask >> 8) & 0xff;
+
+ exynos_ufs_enable_ov_tm(hba);
+
+ for_each_ufs_rx_lane(ufs, i)
+ ufshcd_dme_set(hba,
+ UIC_ARG_MIB_SEL(RX_SYNC_MASK_LENGTH, i), mask);
+
+ exynos_ufs_disable_ov_tm(hba);
+}
+
+static int exynos_ufs_pre_pwr_mode(struct ufs_hba *hba,
+ struct ufs_pa_layer_attr *dev_max_params,
+ struct ufs_pa_layer_attr *dev_req_params)
+{
+ struct exynos_ufs *ufs = ufshcd_get_variant(hba);
+ struct phy *generic_phy = ufs->phy;
+ struct ufs_dev_params ufs_exynos_cap;
+ int ret;
+
+ if (!dev_req_params) {
+ pr_err("%s: incoming dev_req_params is NULL\n", __func__);
+ ret = -EINVAL;
+ goto out;
+ }
+
+
+ ufs_exynos_cap.tx_lanes = UFS_EXYNOS_LIMIT_NUM_LANES_TX;
+ ufs_exynos_cap.rx_lanes = UFS_EXYNOS_LIMIT_NUM_LANES_RX;
+ ufs_exynos_cap.hs_rx_gear = UFS_EXYNOS_LIMIT_HSGEAR_RX;
+ ufs_exynos_cap.hs_tx_gear = UFS_EXYNOS_LIMIT_HSGEAR_TX;
+ ufs_exynos_cap.pwm_rx_gear = UFS_EXYNOS_LIMIT_PWMGEAR_RX;
+ ufs_exynos_cap.pwm_tx_gear = UFS_EXYNOS_LIMIT_PWMGEAR_TX;
+ ufs_exynos_cap.rx_pwr_pwm = UFS_EXYNOS_LIMIT_RX_PWR_PWM;
+ ufs_exynos_cap.tx_pwr_pwm = UFS_EXYNOS_LIMIT_TX_PWR_PWM;
+ ufs_exynos_cap.rx_pwr_hs = UFS_EXYNOS_LIMIT_RX_PWR_HS;
+ ufs_exynos_cap.tx_pwr_hs = UFS_EXYNOS_LIMIT_TX_PWR_HS;
+ ufs_exynos_cap.hs_rate = UFS_EXYNOS_LIMIT_HS_RATE;
+ ufs_exynos_cap.desired_working_mode =
+ UFS_EXYNOS_LIMIT_DESIRED_MODE;
+
+ ret = ufshcd_get_pwr_dev_param(&ufs_exynos_cap,
+ dev_max_params, dev_req_params);
+ if (ret) {
+ pr_err("%s: failed to determine capabilities\n", __func__);
+ goto out;
+ }
+
+ if (ufs->drv_data->pre_pwr_change)
+ ufs->drv_data->pre_pwr_change(ufs, dev_req_params);
+
+ if (ufshcd_is_hs_mode(dev_req_params)) {
+ exynos_ufs_config_sync_pattern_mask(ufs, dev_req_params);
+
+ switch (dev_req_params->hs_rate) {
+ case PA_HS_MODE_A:
+ case PA_HS_MODE_B:
+ phy_calibrate(generic_phy);
+ break;
+ }
+ }
+
+ return 0;
+out:
+ return ret;
+}
+
+#define PWR_MODE_STR_LEN 64
+static int exynos_ufs_post_pwr_mode(struct ufs_hba *hba,
+ struct ufs_pa_layer_attr *pwr_max,
+ struct ufs_pa_layer_attr *pwr_req)
+{
+ struct exynos_ufs *ufs = ufshcd_get_variant(hba);
+ struct phy *generic_phy = ufs->phy;
+ int gear = max_t(u32, pwr_req->gear_rx, pwr_req->gear_tx);
+ int lanes = max_t(u32, pwr_req->lane_rx, pwr_req->lane_tx);
+ char pwr_str[PWR_MODE_STR_LEN] = "";
+
+ /* let default be PWM Gear 1, Lane 1 */
+ if (!gear)
+ gear = 1;
+
+ if (!lanes)
+ lanes = 1;
+
+ if (ufs->drv_data->post_pwr_change)
+ ufs->drv_data->post_pwr_change(ufs, pwr_req);
+
+ if ((ufshcd_is_hs_mode(pwr_req))) {
+ switch (pwr_req->hs_rate) {
+ case PA_HS_MODE_A:
+ case PA_HS_MODE_B:
+ phy_calibrate(generic_phy);
+ break;
+ }
+
+ snprintf(pwr_str, PWR_MODE_STR_LEN, "%s series_%s G_%d L_%d",
+ "FAST", pwr_req->hs_rate == PA_HS_MODE_A ? "A" : "B",
+ gear, lanes);
+ } else {
+ snprintf(pwr_str, PWR_MODE_STR_LEN, "%s G_%d L_%d",
+ "SLOW", gear, lanes);
+ }
+
+ dev_info(hba->dev, "Power mode changed to : %s\n", pwr_str);
+
+ return 0;
+}
+
+static void exynos_ufs_specify_nexus_t_xfer_req(struct ufs_hba *hba,
+ int tag, bool op)
+{
+ struct exynos_ufs *ufs = ufshcd_get_variant(hba);
+ u32 type;
+
+ type = hci_readl(ufs, HCI_UTRL_NEXUS_TYPE);
+
+ if (op)
+ hci_writel(ufs, type | (1 << tag), HCI_UTRL_NEXUS_TYPE);
+ else
+ hci_writel(ufs, type & ~(1 << tag), HCI_UTRL_NEXUS_TYPE);
+}
+
+static void exynos_ufs_specify_nexus_t_tm_req(struct ufs_hba *hba,
+ int tag, u8 func)
+{
+ struct exynos_ufs *ufs = ufshcd_get_variant(hba);
+ u32 type;
+
+ type = hci_readl(ufs, HCI_UTMRL_NEXUS_TYPE);
+
+ switch (func) {
+ case UFS_ABORT_TASK:
+ case UFS_QUERY_TASK:
+ hci_writel(ufs, type | (1 << tag), HCI_UTMRL_NEXUS_TYPE);
+ break;
+ case UFS_ABORT_TASK_SET:
+ case UFS_CLEAR_TASK_SET:
+ case UFS_LOGICAL_RESET:
+ case UFS_QUERY_TASK_SET:
+ hci_writel(ufs, type & ~(1 << tag), HCI_UTMRL_NEXUS_TYPE);
+ break;
+ }
+}
+
+static int exynos_ufs_phy_init(struct exynos_ufs *ufs)
+{
+ struct ufs_hba *hba = ufs->hba;
+ struct phy *generic_phy = ufs->phy;
+ int ret = 0;
+
+ if (ufs->avail_ln_rx == 0 || ufs->avail_ln_tx == 0) {
+ ufshcd_dme_get(hba, UIC_ARG_MIB(PA_AVAILRXDATALANES),
+ &ufs->avail_ln_rx);
+ ufshcd_dme_get(hba, UIC_ARG_MIB(PA_AVAILTXDATALANES),
+ &ufs->avail_ln_tx);
+ WARN(ufs->avail_ln_rx != ufs->avail_ln_tx,
+ "available data lane is not equal(rx:%d, tx:%d)\n",
+ ufs->avail_ln_rx, ufs->avail_ln_tx);
+ }
+
+ phy_set_bus_width(generic_phy, ufs->avail_ln_rx);
+ ret = phy_init(generic_phy);
+ if (ret) {
+ dev_err(hba->dev, "%s: phy init failed, ret = %d\n",
+ __func__, ret);
+ goto out_exit_phy;
+ }
+
+ return 0;
+
+out_exit_phy:
+ phy_exit(generic_phy);
+
+ return ret;
+}
+
+static void exynos_ufs_config_unipro(struct exynos_ufs *ufs)
+{
+ struct ufs_hba *hba = ufs->hba;
+
+ ufshcd_dme_set(hba, UIC_ARG_MIB(PA_DBG_CLK_PERIOD),
+ DIV_ROUND_UP(NSEC_PER_SEC, ufs->mclk_rate));
+ ufshcd_dme_set(hba, UIC_ARG_MIB(PA_TXTRAILINGCLOCKS),
+ ufs->drv_data->uic_attr->tx_trailingclks);
+ ufshcd_dme_set(hba, UIC_ARG_MIB(PA_DBG_OPTION_SUITE),
+ ufs->drv_data->uic_attr->pa_dbg_option_suite);
+}
+
+static void exynos_ufs_config_intr(struct exynos_ufs *ufs, u32 errs, u8 index)
+{
+ switch (index) {
+ case UNIPRO_L1_5:
+ hci_writel(ufs, DFES_ERR_EN | errs, HCI_ERR_EN_PA_LAYER);
+ break;
+ case UNIPRO_L2:
+ hci_writel(ufs, DFES_ERR_EN | errs, HCI_ERR_EN_DL_LAYER);
+ break;
+ case UNIPRO_L3:
+ hci_writel(ufs, DFES_ERR_EN | errs, HCI_ERR_EN_N_LAYER);
+ break;
+ case UNIPRO_L4:
+ hci_writel(ufs, DFES_ERR_EN | errs, HCI_ERR_EN_T_LAYER);
+ break;
+ case UNIPRO_DME:
+ hci_writel(ufs, DFES_ERR_EN | errs, HCI_ERR_EN_DME_LAYER);
+ break;
+ }
+}
+
+static int exynos_ufs_pre_link(struct ufs_hba *hba)
+{
+ struct exynos_ufs *ufs = ufshcd_get_variant(hba);
+
+ /* hci */
+ exynos_ufs_config_intr(ufs, DFES_DEF_L2_ERRS, UNIPRO_L2);
+ exynos_ufs_config_intr(ufs, DFES_DEF_L3_ERRS, UNIPRO_L3);
+ exynos_ufs_config_intr(ufs, DFES_DEF_L4_ERRS, UNIPRO_L4);
+ exynos_ufs_set_unipro_pclk_div(ufs);
+
+ /* unipro */
+ exynos_ufs_config_unipro(ufs);
+
+ /* m-phy */
+ exynos_ufs_phy_init(ufs);
+ exynos_ufs_config_phy_time_attr(ufs);
+ exynos_ufs_config_phy_cap_attr(ufs);
+
+ if (ufs->drv_data->pre_link)
+ ufs->drv_data->pre_link(ufs);
+
+ return 0;
+}
+
+static void exynos_ufs_fit_aggr_timeout(struct exynos_ufs *ufs)
+{
+ u32 val;
+
+ val = exynos_ufs_calc_time_cntr(ufs, IATOVAL_NSEC / CNTR_DIV_VAL);
+ hci_writel(ufs, val & CNT_VAL_1US_MASK, HCI_1US_TO_CNT_VAL);
+}
+
+static int exynos_ufs_post_link(struct ufs_hba *hba)
+{
+ struct exynos_ufs *ufs = ufshcd_get_variant(hba);
+ struct phy *generic_phy = ufs->phy;
+ struct exynos_ufs_uic_attr *attr = ufs->drv_data->uic_attr;
+
+ exynos_ufs_establish_connt(ufs);
+ exynos_ufs_fit_aggr_timeout(ufs);
+
+ hci_writel(ufs, 0xa, HCI_DATA_REORDER);
+ hci_writel(ufs, PRDT_SET_SIZE(12), HCI_TXPRDT_ENTRY_SIZE);
+ hci_writel(ufs, PRDT_SET_SIZE(12), HCI_RXPRDT_ENTRY_SIZE);
+ hci_writel(ufs, (1 << hba->nutrs) - 1, HCI_UTRL_NEXUS_TYPE);
+ hci_writel(ufs, (1 << hba->nutmrs) - 1, HCI_UTMRL_NEXUS_TYPE);
+ hci_writel(ufs, 0xf, HCI_AXIDMA_RWDATA_BURST_LEN);
+
+ if (ufs->opts & EXYNOS_UFS_OPT_SKIP_CONNECTION_ESTAB)
+ ufshcd_dme_set(hba,
+ UIC_ARG_MIB(T_DBG_SKIP_INIT_HIBERN8_EXIT), TRUE);
+
+ if (attr->pa_granularity) {
+ exynos_ufs_enable_dbg_mode(hba);
+ ufshcd_dme_set(hba, UIC_ARG_MIB(PA_GRANULARITY),
+ attr->pa_granularity);
+ exynos_ufs_disable_dbg_mode(hba);
+
+ if (attr->pa_tactivate)
+ ufshcd_dme_set(hba, UIC_ARG_MIB(PA_TACTIVATE),
+ attr->pa_tactivate);
+ if (attr->pa_hibern8time &&
+ !(ufs->opts & EXYNOS_UFS_OPT_USE_SW_HIBERN8_TIMER))
+ ufshcd_dme_set(hba, UIC_ARG_MIB(PA_HIBERN8TIME),
+ attr->pa_hibern8time);
+ }
+
+ if (ufs->opts & EXYNOS_UFS_OPT_USE_SW_HIBERN8_TIMER) {
+ if (!attr->pa_granularity)
+ ufshcd_dme_get(hba, UIC_ARG_MIB(PA_GRANULARITY),
+ &attr->pa_granularity);
+ if (!attr->pa_hibern8time)
+ ufshcd_dme_get(hba, UIC_ARG_MIB(PA_HIBERN8TIME),
+ &attr->pa_hibern8time);
+ /*
+ * not wait for HIBERN8 time to exit hibernation
+ */
+ ufshcd_dme_set(hba, UIC_ARG_MIB(PA_HIBERN8TIME), 0);
+
+ if (attr->pa_granularity < 1 || attr->pa_granularity > 6) {
+ /* Valid range for granularity: 1 ~ 6 */
+ dev_warn(hba->dev,
+ "%s: pa_granularity %d is invalid, assuming backwards compatibility\n",
+ __func__,
+ attr->pa_granularity);
+ attr->pa_granularity = 6;
+ }
+ }
+
+ phy_calibrate(generic_phy);
+
+ if (ufs->drv_data->post_link)
+ ufs->drv_data->post_link(ufs);
+
+ return 0;
+}
+
+static int exynos_ufs_parse_dt(struct device *dev, struct exynos_ufs *ufs)
+{
+ struct device_node *np = dev->of_node;
+ struct exynos_ufs_drv_data *drv_data = &exynos_ufs_drvs;
+ struct exynos_ufs_uic_attr *attr;
+ int ret = 0;
+
+ while (drv_data->compatible) {
+ if (of_device_is_compatible(np, drv_data->compatible)) {
+ ufs->drv_data = drv_data;
+ break;
+ }
+ drv_data++;
+ }
+
+ if (ufs->drv_data && ufs->drv_data->uic_attr) {
+ attr = ufs->drv_data->uic_attr;
+ } else {
+ dev_err(dev, "failed to get uic attributes\n");
+ ret = -EINVAL;
+ goto out;
+ }
+
+ ufs->pclk_avail_min = PCLK_AVAIL_MIN;
+ ufs->pclk_avail_max = PCLK_AVAIL_MAX;
+
+ attr->rx_adv_fine_gran_sup_en = RX_ADV_FINE_GRAN_SUP_EN;
+ attr->rx_adv_fine_gran_step = RX_ADV_FINE_GRAN_STEP_VAL;
+ attr->rx_adv_min_actv_time_cap = RX_ADV_MIN_ACTV_TIME_CAP;
+ attr->pa_granularity = PA_GRANULARITY_VAL;
+ attr->pa_tactivate = PA_TACTIVATE_VAL;
+ attr->pa_hibern8time = PA_HIBERN8TIME_VAL;
+
+out:
+ return ret;
+}
+
+static int exynos_ufs_init(struct ufs_hba *hba)
+{
+ struct device *dev = hba->dev;
+ struct platform_device *pdev = to_platform_device(dev);
+ struct exynos_ufs *ufs;
+ struct resource *res;
+ int ret;
+
+ ufs = devm_kzalloc(dev, sizeof(*ufs), GFP_KERNEL);
+ if (!ufs)
+ return -ENOMEM;
+
+ /* exynos-specific hci */
+ res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "vs_hci");
+ ufs->reg_hci = devm_ioremap_resource(dev, res);
+ if (IS_ERR(ufs->reg_hci)) {
+ dev_err(dev, "cannot ioremap for hci vendor register\n");
+ return PTR_ERR(ufs->reg_hci);
+ }
+
+ /* unipro */
+ res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "unipro");
+ ufs->reg_unipro = devm_ioremap_resource(dev, res);
+ if (IS_ERR(ufs->reg_unipro)) {
+ dev_err(dev, "cannot ioremap for unipro register\n");
+ return PTR_ERR(ufs->reg_unipro);
+ }
+
+ /* ufs protector */
+ res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "ufsp");
+ ufs->reg_ufsp = devm_ioremap_resource(dev, res);
+ if (IS_ERR(ufs->reg_ufsp)) {
+ dev_err(dev, "cannot ioremap for ufs protector register\n");
+ return PTR_ERR(ufs->reg_ufsp);
+ }
+
+ ret = exynos_ufs_parse_dt(dev, ufs);
+ if (ret) {
+ dev_err(dev, "failed to get dt info.\n");
+ goto out;
+ }
+
+ ufs->phy = devm_phy_get(dev, "ufs-phy");
+ if (IS_ERR(ufs->phy)) {
+ ret = PTR_ERR(ufs->phy);
+ dev_err(dev, "failed to get ufs-phy\n");
+ goto out;
+ }
+
+ ret = phy_power_on(ufs->phy);
+ if (ret)
+ goto phy_off;
+
+ ufs->hba = hba;
+ ufs->opts = ufs->drv_data->opts;
+ ufs->rx_sel_idx = PA_MAXDATALANES;
+ if (ufs->opts & EXYNOS_UFS_OPT_BROKEN_RX_SEL_IDX)
+ ufs->rx_sel_idx = 0;
+ hba->priv = (void *)ufs;
+ hba->quirks = ufs->drv_data->quirks;
+ if (ufs->drv_data->drv_init) {
+ ret = ufs->drv_data->drv_init(dev, ufs);
+ if (ret) {
+ dev_err(dev, "failed to init drv-data\n");
+ goto out;
+ }
+ }
+
+ ret = exynos_ufs_get_clk_info(ufs);
+ if (ret)
+ goto out;
+ exynos_ufs_specify_phy_time_attr(ufs);
+ exynos_ufs_config_smu(ufs);
+ return 0;
+
+phy_off:
+ phy_power_off(ufs->phy);
+out:
+ hba->priv = NULL;
+ return ret;
+}
+
+static int exynos_ufs_host_reset(struct ufs_hba *hba)
+{
+ struct exynos_ufs *ufs = ufshcd_get_variant(hba);
+ unsigned long timeout = jiffies + msecs_to_jiffies(1);
+ u32 val;
+ int ret = 0;
+
+ exynos_ufs_disable_auto_ctrl_hcc_save(ufs, &val);
+
+ hci_writel(ufs, UFS_SW_RST_MASK, HCI_SW_RST);
+
+ do {
+ if (!(hci_readl(ufs, HCI_SW_RST) & UFS_SW_RST_MASK))
+ goto out;
+ } while (time_before(jiffies, timeout));
+
+ dev_err(hba->dev, "timeout host sw-reset\n");
+ ret = -ETIMEDOUT;
+
+out:
+ exynos_ufs_auto_ctrl_hcc_restore(ufs, &val);
+ return ret;
+}
+
+static void exynos_ufs_dev_hw_reset(struct ufs_hba *hba)
+{
+ struct exynos_ufs *ufs = ufshcd_get_variant(hba);
+
+ hci_writel(ufs, 0 << 0, HCI_GPIO_OUT);
+ udelay(5);
+ hci_writel(ufs, 1 << 0, HCI_GPIO_OUT);
+}
+
+static void exynos_ufs_pre_hibern8(struct ufs_hba *hba, u8 enter)
+{
+ struct exynos_ufs *ufs = ufshcd_get_variant(hba);
+ struct exynos_ufs_uic_attr *attr = ufs->drv_data->uic_attr;
+
+ if (!enter) {
+ if (ufs->opts & EXYNOS_UFS_OPT_BROKEN_AUTO_CLK_CTRL)
+ exynos_ufs_disable_auto_ctrl_hcc(ufs);
+ exynos_ufs_ungate_clks(ufs);
+
+ if (ufs->opts & EXYNOS_UFS_OPT_USE_SW_HIBERN8_TIMER) {
+ const unsigned int granularity_tbl[] = {
+ 1, 4, 8, 16, 32, 100
+ };
+ int h8_time = attr->pa_hibern8time *
+ granularity_tbl[attr->pa_granularity - 1];
+ unsigned long us;
+ s64 delta;
+
+ do {
+ delta = h8_time - ktime_us_delta(ktime_get(),
+ ufs->entry_hibern8_t);
+ if (delta <= 0)
+ break;
+
+ us = min_t(s64, delta, USEC_PER_MSEC);
+ if (us >= 10)
+ usleep_range(us, us + 10);
+ } while (1);
+ }
+ }
+}
+
+static void exynos_ufs_post_hibern8(struct ufs_hba *hba, u8 enter)
+{
+ struct exynos_ufs *ufs = ufshcd_get_variant(hba);
+
+ if (!enter) {
+ u32 cur_mode = 0;
+ u32 pwrmode;
+
+ if (ufshcd_is_hs_mode(&ufs->dev_req_params))
+ pwrmode = FAST_MODE;
+ else
+ pwrmode = SLOW_MODE;
+
+ ufshcd_dme_get(hba, UIC_ARG_MIB(PA_PWRMODE), &cur_mode);
+ if (cur_mode != (pwrmode << 4 | pwrmode)) {
+ dev_warn(hba->dev, "%s: power mode change\n", __func__);
+ hba->pwr_info.pwr_rx = (cur_mode >> 4) & 0xf;
+ hba->pwr_info.pwr_tx = cur_mode & 0xf;
+ ufshcd_config_pwr_mode(hba, &hba->max_pwr_info.info);
+ }
+
+ if (!(ufs->opts & EXYNOS_UFS_OPT_SKIP_CONNECTION_ESTAB))
+ exynos_ufs_establish_connt(ufs);
+ } else {
+ ufs->entry_hibern8_t = ktime_get();
+ exynos_ufs_gate_clks(ufs);
+ if (ufs->opts & EXYNOS_UFS_OPT_BROKEN_AUTO_CLK_CTRL)
+ exynos_ufs_enable_auto_ctrl_hcc(ufs);
+ }
+}
+
+static int exynos_ufs_hce_enable_notify(struct ufs_hba *hba,
+ enum ufs_notify_change_status status)
+{
+ struct exynos_ufs *ufs = ufshcd_get_variant(hba);
+ int ret = 0;
+
+ switch (status) {
+ case PRE_CHANGE:
+ ret = exynos_ufs_host_reset(hba);
+ if (ret)
+ return ret;
+ exynos_ufs_dev_hw_reset(hba);
+ break;
+ case POST_CHANGE:
+ exynos_ufs_calc_pwm_clk_div(ufs);
+ if (!(ufs->opts & EXYNOS_UFS_OPT_BROKEN_AUTO_CLK_CTRL))
+ exynos_ufs_enable_auto_ctrl_hcc(ufs);
+ break;
+ }
+
+ return ret;
+}
+
+static int exynos_ufs_link_startup_notify(struct ufs_hba *hba,
+ enum ufs_notify_change_status status)
+{
+ int ret = 0;
+
+ switch (status) {
+ case PRE_CHANGE:
+ ret = exynos_ufs_pre_link(hba);
+ break;
+ case POST_CHANGE:
+ ret = exynos_ufs_post_link(hba);
+ break;
+ }
+
+ return ret;
+}
+
+static int exynos_ufs_pwr_change_notify(struct ufs_hba *hba,
+ enum ufs_notify_change_status status,
+ struct ufs_pa_layer_attr *dev_max_params,
+ struct ufs_pa_layer_attr *dev_req_params)
+{
+ int ret = 0;
+
+ switch (status) {
+ case PRE_CHANGE:
+ ret = exynos_ufs_pre_pwr_mode(hba, dev_max_params,
+ dev_req_params);
+ break;
+ case POST_CHANGE:
+ ret = exynos_ufs_post_pwr_mode(hba, NULL, dev_req_params);
+ break;
+ }
+
+ return ret;
+}
+
+static void exynos_ufs_hibern8_notify(struct ufs_hba *hba,
+ enum uic_cmd_dme enter,
+ enum ufs_notify_change_status notify)
+{
+ switch ((u8)notify) {
+ case PRE_CHANGE:
+ exynos_ufs_pre_hibern8(hba, enter);
+ break;
+ case POST_CHANGE:
+ exynos_ufs_post_hibern8(hba, enter);
+ break;
+ }
+}
+
+static int exynos_ufs_suspend(struct ufs_hba *hba, enum ufs_pm_op pm_op)
+{
+ struct exynos_ufs *ufs = ufshcd_get_variant(hba);
+
+ if (!ufshcd_is_link_active(hba))
+ phy_power_off(ufs->phy);
+
+ return 0;
+}
+
+static int exynos_ufs_resume(struct ufs_hba *hba, enum ufs_pm_op pm_op)
+{
+ struct exynos_ufs *ufs = ufshcd_get_variant(hba);
+
+ if (!ufshcd_is_link_active(hba))
+ phy_power_on(ufs->phy);
+
+ exynos_ufs_config_smu(ufs);
+
+ return 0;
+}
+
+static struct ufs_hba_variant_ops ufs_hba_exynos_ops = {
+ .name = "exynos_ufs",
+ .init = exynos_ufs_init,
+ .hce_enable_notify = exynos_ufs_hce_enable_notify,
+ .link_startup_notify = exynos_ufs_link_startup_notify,
+ .pwr_change_notify = exynos_ufs_pwr_change_notify,
+ .setup_xfer_req = exynos_ufs_specify_nexus_t_xfer_req,
+ .setup_task_mgmt = exynos_ufs_specify_nexus_t_tm_req,
+ .hibern8_notify = exynos_ufs_hibern8_notify,
+ .suspend = exynos_ufs_suspend,
+ .resume = exynos_ufs_resume,
+};
+
+static int exynos_ufs_probe(struct platform_device *pdev)
+{
+ int err;
+ struct device *dev = &pdev->dev;
+
+ err = ufshcd_pltfrm_init(pdev, &ufs_hba_exynos_ops);
+ if (err)
+ dev_err(dev, "ufshcd_pltfrm_init() failed %d\n", err);
+
+ return err;
+}
+
+static int exynos_ufs_remove(struct platform_device *pdev)
+{
+ struct ufs_hba *hba = platform_get_drvdata(pdev);
+
+ pm_runtime_get_sync(&(pdev)->dev);
+ ufshcd_remove(hba);
+ return 0;
+}
+
+struct exynos_ufs_drv_data exynos_ufs_drvs = {
+
+ .compatible = "samsung,exynos7-ufs",
+ .uic_attr = &exynos7_uic_attr,
+ .quirks = UFSHCD_QUIRK_PRDT_BYTE_GRAN |
+ UFSHCI_QUIRK_BROKEN_REQ_LIST_CLR |
+ UFSHCI_QUIRK_BROKEN_HCE |
+ UFSHCI_QUIRK_SKIP_RESET_INTR_AGGR |
+ UFSHCD_QUIRK_BROKEN_OCS_FATAL_ERROR,
+ .opts = EXYNOS_UFS_OPT_HAS_APB_CLK_CTRL |
+ EXYNOS_UFS_OPT_BROKEN_AUTO_CLK_CTRL |
+ EXYNOS_UFS_OPT_BROKEN_RX_SEL_IDX |
+ EXYNOS_UFS_OPT_SKIP_CONNECTION_ESTAB |
+ EXYNOS_UFS_OPT_USE_SW_HIBERN8_TIMER,
+ .drv_init = exynos7_ufs_drv_init,
+ .pre_link = exynos7_ufs_pre_link,
+ .post_link = exynos7_ufs_post_link,
+ .pre_pwr_change = exynos7_ufs_pre_pwr_change,
+ .post_pwr_change = exynos7_ufs_post_pwr_change,
+};
+
+static const struct of_device_id exynos_ufs_of_match[] = {
+ { .compatible = "samsung,exynos7-ufs",
+ .data = &exynos_ufs_drvs },
+ {},
+};
+
+static const struct dev_pm_ops exynos_ufs_pm_ops = {
+ .suspend = ufshcd_pltfrm_suspend,
+ .resume = ufshcd_pltfrm_resume,
+ .runtime_suspend = ufshcd_pltfrm_runtime_suspend,
+ .runtime_resume = ufshcd_pltfrm_runtime_resume,
+ .runtime_idle = ufshcd_pltfrm_runtime_idle,
+};
+
+static struct platform_driver exynos_ufs_pltform = {
+ .probe = exynos_ufs_probe,
+ .remove = exynos_ufs_remove,
+ .shutdown = ufshcd_pltfrm_shutdown,
+ .driver = {
+ .name = "exynos-ufshc",
+ .pm = &exynos_ufs_pm_ops,
+ .of_match_table = of_match_ptr(exynos_ufs_of_match),
+ },
+};
+module_platform_driver(exynos_ufs_pltform);
+
+MODULE_AUTHOR("Alim Akhtar <alim.akhtar@samsung.com>");
+MODULE_AUTHOR("Seungwon Jeon <essuuj@gmail.com>");
+MODULE_DESCRIPTION("Exynos UFS HCI Driver");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/scsi/ufs/ufs-exynos.h b/drivers/scsi/ufs/ufs-exynos.h
new file mode 100644
index 0000000..76d6e39
--- /dev/null
+++ b/drivers/scsi/ufs/ufs-exynos.h
@@ -0,0 +1,287 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*
+ * UFS Host Controller driver for Exynos specific extensions
+ *
+ * Copyright (C) 2014-2015 Samsung Electronics Co., Ltd.
+ *
+ */
+
+#ifndef _UFS_EXYNOS_H_
+#define _UFS_EXYNOS_H_
+
+/*
+ * UNIPRO registers
+ */
+#define UNIPRO_DBG_FORCE_DME_CTRL_STATE 0x150
+
+/*
+ * MIBs for PA debug registers
+ */
+#define PA_DBG_CLK_PERIOD 0x9514
+#define PA_DBG_TXPHY_CFGUPDT 0x9518
+#define PA_DBG_RXPHY_CFGUPDT 0x9519
+#define PA_DBG_MODE 0x9529
+#define PA_DBG_SKIP_RESET_PHY 0x9539
+#define PA_DBG_OV_TM 0x9540
+#define PA_DBG_SKIP_LINE_RESET 0x9541
+#define PA_DBG_LINE_RESET_REQ 0x9543
+#define PA_DBG_OPTION_SUITE 0x9564
+#define PA_DBG_OPTION_SUITE_DYN 0x9565
+
+/*
+ * MIBs for Transport Layer debug registers
+ */
+#define T_DBG_SKIP_INIT_HIBERN8_EXIT 0xc001
+
+/*
+ * Exynos MPHY attributes
+ */
+#define TX_LINERESET_N_VAL 0x0277
+#define TX_LINERESET_N(v) (((v) >> 10) & 0xFF)
+#define TX_LINERESET_P_VAL 0x027D
+#define TX_LINERESET_P(v) (((v) >> 12) & 0xFF)
+#define TX_OV_SLEEP_CNT_TIMER 0x028E
+#define TX_OV_H8_ENTER_EN (1 << 7)
+#define TX_OV_SLEEP_CNT(v) (((v) >> 5) & 0x7F)
+#define TX_HIGH_Z_CNT_11_08 0x028C
+#define TX_HIGH_Z_CNT_H(v) (((v) >> 8) & 0xF)
+#define TX_HIGH_Z_CNT_07_00 0x028D
+#define TX_HIGH_Z_CNT_L(v) ((v) & 0xFF)
+#define TX_BASE_NVAL_07_00 0x0293
+#define TX_BASE_NVAL_L(v) ((v) & 0xFF)
+#define TX_BASE_NVAL_15_08 0x0294
+#define TX_BASE_NVAL_H(v) (((v) >> 8) & 0xFF)
+#define TX_GRAN_NVAL_07_00 0x0295
+#define TX_GRAN_NVAL_L(v) ((v) & 0xFF)
+#define TX_GRAN_NVAL_10_08 0x0296
+#define TX_GRAN_NVAL_H(v) (((v) >> 8) & 0x3)
+
+#define RX_FILLER_ENABLE 0x0316
+#define RX_FILLER_EN (1 << 1)
+#define RX_LINERESET_VAL 0x0317
+#define RX_LINERESET(v) (((v) >> 12) & 0xFF)
+#define RX_LCC_IGNORE 0x0318
+#define RX_SYNC_MASK_LENGTH 0x0321
+#define RX_HIBERN8_WAIT_VAL_BIT_20_16 0x0331
+#define RX_HIBERN8_WAIT_VAL_BIT_15_08 0x0332
+#define RX_HIBERN8_WAIT_VAL_BIT_07_00 0x0333
+#define RX_OV_SLEEP_CNT_TIMER 0x0340
+#define RX_OV_SLEEP_CNT(v) (((v) >> 6) & 0x1F)
+#define RX_OV_STALL_CNT_TIMER 0x0341
+#define RX_OV_STALL_CNT(v) (((v) >> 4) & 0xFF)
+#define RX_BASE_NVAL_07_00 0x0355
+#define RX_BASE_NVAL_L(v) ((v) & 0xFF)
+#define RX_BASE_NVAL_15_08 0x0354
+#define RX_BASE_NVAL_H(v) (((v) >> 8) & 0xFF)
+#define RX_GRAN_NVAL_07_00 0x0353
+#define RX_GRAN_NVAL_L(v) ((v) & 0xFF)
+#define RX_GRAN_NVAL_10_08 0x0352
+#define RX_GRAN_NVAL_H(v) (((v) >> 8) & 0x3)
+
+#define CMN_PWM_CLK_CTRL 0x0402
+#define PWM_CLK_CTRL_MASK 0x3
+
+#define IATOVAL_NSEC 20000 /* unit: ns */
+#define UNIPRO_PCLK_PERIOD(ufs) (NSEC_PER_SEC / ufs->pclk_rate)
+
+struct exynos_ufs;
+
+/* vendor specific pre-defined parameters */
+#define SLOW 1
+#define FAST 2
+
+#define UFS_EXYNOS_LIMIT_NUM_LANES_RX 2
+#define UFS_EXYNOS_LIMIT_NUM_LANES_TX 2
+#define UFS_EXYNOS_LIMIT_HSGEAR_RX UFS_HS_G3
+#define UFS_EXYNOS_LIMIT_HSGEAR_TX UFS_HS_G3
+#define UFS_EXYNOS_LIMIT_PWMGEAR_RX UFS_PWM_G4
+#define UFS_EXYNOS_LIMIT_PWMGEAR_TX UFS_PWM_G4
+#define UFS_EXYNOS_LIMIT_RX_PWR_PWM SLOW_MODE
+#define UFS_EXYNOS_LIMIT_TX_PWR_PWM SLOW_MODE
+#define UFS_EXYNOS_LIMIT_RX_PWR_HS FAST_MODE
+#define UFS_EXYNOS_LIMIT_TX_PWR_HS FAST_MODE
+#define UFS_EXYNOS_LIMIT_HS_RATE PA_HS_MODE_B
+#define UFS_EXYNOS_LIMIT_DESIRED_MODE FAST
+
+#define RX_ADV_FINE_GRAN_SUP_EN 0x1
+#define RX_ADV_FINE_GRAN_STEP_VAL 0x3
+#define RX_ADV_MIN_ACTV_TIME_CAP 0x9
+
+#define PA_GRANULARITY_VAL 0x6
+#define PA_TACTIVATE_VAL 0x3
+#define PA_HIBERN8TIME_VAL 0x20
+
+#define PCLK_AVAIL_MIN 70000000
+#define PCLK_AVAIL_MAX 133000000
+
+struct exynos_ufs_uic_attr {
+ /* TX Attributes */
+ unsigned int tx_trailingclks;
+ unsigned int tx_dif_p_nsec;
+ unsigned int tx_dif_n_nsec;
+ unsigned int tx_high_z_cnt_nsec;
+ unsigned int tx_base_unit_nsec;
+ unsigned int tx_gran_unit_nsec;
+ unsigned int tx_sleep_cnt;
+ unsigned int tx_min_activatetime;
+ /* RX Attributes */
+ unsigned int rx_filler_enable;
+ unsigned int rx_dif_p_nsec;
+ unsigned int rx_hibern8_wait_nsec;
+ unsigned int rx_base_unit_nsec;
+ unsigned int rx_gran_unit_nsec;
+ unsigned int rx_sleep_cnt;
+ unsigned int rx_stall_cnt;
+ unsigned int rx_hs_g1_sync_len_cap;
+ unsigned int rx_hs_g2_sync_len_cap;
+ unsigned int rx_hs_g3_sync_len_cap;
+ unsigned int rx_hs_g1_prep_sync_len_cap;
+ unsigned int rx_hs_g2_prep_sync_len_cap;
+ unsigned int rx_hs_g3_prep_sync_len_cap;
+ /* Common Attributes */
+ unsigned int cmn_pwm_clk_ctrl;
+ /* Internal Attributes */
+ unsigned int pa_dbg_option_suite;
+ /* Changeable Attributes */
+ unsigned int rx_adv_fine_gran_sup_en;
+ unsigned int rx_adv_fine_gran_step;
+ unsigned int rx_min_actv_time_cap;
+ unsigned int rx_hibern8_time_cap;
+ unsigned int rx_adv_min_actv_time_cap;
+ unsigned int rx_adv_hibern8_time_cap;
+ unsigned int pa_granularity;
+ unsigned int pa_tactivate;
+ unsigned int pa_hibern8time;
+};
+
+struct exynos_ufs_drv_data {
+ char *compatible;
+ struct exynos_ufs_uic_attr *uic_attr;
+ unsigned int quirks;
+ unsigned int opts;
+ /* SoC's specific operations */
+ int (*drv_init)(struct device *dev, struct exynos_ufs *ufs);
+ int (*pre_link)(struct exynos_ufs *ufs);
+ int (*post_link)(struct exynos_ufs *ufs);
+ int (*pre_pwr_change)(struct exynos_ufs *ufs,
+ struct ufs_pa_layer_attr *pwr);
+ int (*post_pwr_change)(struct exynos_ufs *ufs,
+ struct ufs_pa_layer_attr *pwr);
+};
+
+struct ufs_phy_time_cfg {
+ u32 tx_linereset_p;
+ u32 tx_linereset_n;
+ u32 tx_high_z_cnt;
+ u32 tx_base_n_val;
+ u32 tx_gran_n_val;
+ u32 tx_sleep_cnt;
+ u32 rx_linereset;
+ u32 rx_hibern8_wait;
+ u32 rx_base_n_val;
+ u32 rx_gran_n_val;
+ u32 rx_sleep_cnt;
+ u32 rx_stall_cnt;
+};
+
+struct exynos_ufs {
+ struct ufs_hba *hba;
+ struct phy *phy;
+ void __iomem *reg_hci;
+ void __iomem *reg_unipro;
+ void __iomem *reg_ufsp;
+ struct clk *clk_hci_core;
+ struct clk *clk_unipro_main;
+ struct clk *clk_apb;
+ u32 pclk_rate;
+ u32 pclk_div;
+ u32 pclk_avail_min;
+ u32 pclk_avail_max;
+ u32 mclk_rate;
+ int avail_ln_rx;
+ int avail_ln_tx;
+ int rx_sel_idx;
+ struct ufs_pa_layer_attr dev_req_params;
+ struct ufs_phy_time_cfg t_cfg;
+ ktime_t entry_hibern8_t;
+ struct exynos_ufs_drv_data *drv_data;
+
+ u32 opts;
+#define EXYNOS_UFS_OPT_HAS_APB_CLK_CTRL BIT(0)
+#define EXYNOS_UFS_OPT_SKIP_CONNECTION_ESTAB BIT(1)
+#define EXYNOS_UFS_OPT_BROKEN_AUTO_CLK_CTRL BIT(2)
+#define EXYNOS_UFS_OPT_BROKEN_RX_SEL_IDX BIT(3)
+#define EXYNOS_UFS_OPT_USE_SW_HIBERN8_TIMER BIT(4)
+};
+
+#define for_each_ufs_rx_lane(ufs, i) \
+ for (i = (ufs)->rx_sel_idx; \
+ i < (ufs)->rx_sel_idx + (ufs)->avail_ln_rx; i++)
+#define for_each_ufs_tx_lane(ufs, i) \
+ for (i = 0; i < (ufs)->avail_ln_tx; i++)
+
+#define EXYNOS_UFS_MMIO_FUNC(name) \
+static inline void name##_writel(struct exynos_ufs *ufs, u32 val, u32 reg)\
+{ \
+ writel(val, ufs->reg_##name + reg); \
+} \
+ \
+static inline u32 name##_readl(struct exynos_ufs *ufs, u32 reg) \
+{ \
+ return readl(ufs->reg_##name + reg); \
+}
+
+EXYNOS_UFS_MMIO_FUNC(hci);
+EXYNOS_UFS_MMIO_FUNC(unipro);
+EXYNOS_UFS_MMIO_FUNC(ufsp);
+#undef EXYNOS_UFS_MMIO_FUNC
+
+long exynos_ufs_calc_time_cntr(struct exynos_ufs *, long);
+
+static inline void exynos_ufs_enable_ov_tm(struct ufs_hba *hba)
+{
+ ufshcd_dme_set(hba, UIC_ARG_MIB(PA_DBG_OV_TM), TRUE);
+}
+
+static inline void exynos_ufs_disable_ov_tm(struct ufs_hba *hba)
+{
+ ufshcd_dme_set(hba, UIC_ARG_MIB(PA_DBG_OV_TM), FALSE);
+}
+
+static inline void exynos_ufs_enable_dbg_mode(struct ufs_hba *hba)
+{
+ ufshcd_dme_set(hba, UIC_ARG_MIB(PA_DBG_MODE), TRUE);
+}
+
+static inline void exynos_ufs_disable_dbg_mode(struct ufs_hba *hba)
+{
+ ufshcd_dme_set(hba, UIC_ARG_MIB(PA_DBG_MODE), FALSE);
+}
+
+struct exynos_ufs_drv_data exynos_ufs_drvs;
+
+struct exynos_ufs_uic_attr exynos7_uic_attr = {
+ .tx_trailingclks = 0x10,
+ .tx_dif_p_nsec = 3000000, /* unit: ns */
+ .tx_dif_n_nsec = 1000000, /* unit: ns */
+ .tx_high_z_cnt_nsec = 20000, /* unit: ns */
+ .tx_base_unit_nsec = 100000, /* unit: ns */
+ .tx_gran_unit_nsec = 4000, /* unit: ns */
+ .tx_sleep_cnt = 1000, /* unit: ns */
+ .tx_min_activatetime = 0xa,
+ .rx_filler_enable = 0x2,
+ .rx_dif_p_nsec = 1000000, /* unit: ns */
+ .rx_hibern8_wait_nsec = 4000000, /* unit: ns */
+ .rx_base_unit_nsec = 100000, /* unit: ns */
+ .rx_gran_unit_nsec = 4000, /* unit: ns */
+ .rx_sleep_cnt = 1280, /* unit: ns */
+ .rx_stall_cnt = 320, /* unit: ns */
+ .rx_hs_g1_sync_len_cap = SYNC_LEN_COARSE(0xf),
+ .rx_hs_g2_sync_len_cap = SYNC_LEN_COARSE(0xf),
+ .rx_hs_g3_sync_len_cap = SYNC_LEN_COARSE(0xf),
+ .rx_hs_g1_prep_sync_len_cap = PREP_LEN(0xf),
+ .rx_hs_g2_prep_sync_len_cap = PREP_LEN(0xf),
+ .rx_hs_g3_prep_sync_len_cap = PREP_LEN(0xf),
+ .pa_dbg_option_suite = 0x30103,
+};
+#endif /* _UFS_EXYNOS_H_ */
diff --git a/drivers/scsi/ufs/ufs-mediatek.c b/drivers/scsi/ufs/ufs-mediatek.c
index d56ce8d..29cd017 100644
--- a/drivers/scsi/ufs/ufs-mediatek.c
+++ b/drivers/scsi/ufs/ufs-mediatek.c
@@ -12,9 +12,11 @@
#include <linux/of_address.h>
#include <linux/phy/phy.h>
#include <linux/platform_device.h>
+#include <linux/regulator/consumer.h>
#include <linux/soc/mediatek/mtk_sip_svc.h>
#include "ufshcd.h"
+#include "ufshcd-crypto.h"
#include "ufshcd-pltfrm.h"
#include "ufs_quirks.h"
#include "unipro.h"
@@ -24,6 +26,9 @@
arm_smccc_smc(MTK_SIP_UFS_CONTROL, \
cmd, val, 0, 0, 0, 0, 0, &(res))
+#define ufs_mtk_crypto_ctrl(res, enable) \
+ ufs_mtk_smc(UFS_MTK_SIP_CRYPTO_CTRL, enable, res)
+
#define ufs_mtk_ref_clk_notify(on, res) \
ufs_mtk_smc(UFS_MTK_SIP_REF_CLK_NOTIFICATION, on, res)
@@ -31,6 +36,8 @@
ufs_mtk_smc(UFS_MTK_SIP_DEVICE_RESET, high, res)
static struct ufs_dev_fix ufs_mtk_dev_fixups[] = {
+ UFS_FIX(UFS_VENDOR_MICRON, UFS_ANY_MODEL,
+ UFS_DEVICE_QUIRK_DELAY_AFTER_LPM),
UFS_FIX(UFS_VENDOR_SKHYNIX, "H9HQ21AFAMZDAR",
UFS_DEVICE_QUIRK_SUPPORT_EXTENDED_FEATURES),
END_FIX
@@ -72,6 +79,18 @@
}
}
+static void ufs_mtk_crypto_enable(struct ufs_hba *hba)
+{
+ struct arm_smccc_res res;
+
+ ufs_mtk_crypto_ctrl(res, 1);
+ if (res.a0) {
+ dev_info(hba->dev, "%s: crypto enable failed, err: %lu\n",
+ __func__, res.a0);
+ hba->caps &= ~UFSHCD_CAP_CRYPTO;
+ }
+}
+
static int ufs_mtk_hce_enable_notify(struct ufs_hba *hba,
enum ufs_notify_change_status status)
{
@@ -82,6 +101,9 @@
hba->vps->hba_enable_delay_us = 0;
else
hba->vps->hba_enable_delay_us = 600;
+
+ if (hba->caps & UFSHCD_CAP_CRYPTO)
+ ufs_mtk_crypto_enable(hba);
}
return 0;
@@ -112,6 +134,12 @@
if (err)
host->mphy = NULL;
+ /*
+ * Allow unbound mphy because not every platform needs specific
+ * mphy control.
+ */
+ if (err == -ENODEV)
+ err = 0;
return err;
}
@@ -120,7 +148,7 @@
{
struct ufs_mtk_host *host = ufshcd_get_variant(hba);
struct arm_smccc_res res;
- unsigned long timeout;
+ ktime_t timeout, time_checked;
u32 value;
if (host->ref_clk_enabled == on)
@@ -135,8 +163,9 @@
}
/* Wait for ack */
- timeout = jiffies + msecs_to_jiffies(REFCLK_REQ_TIMEOUT_MS);
+ timeout = ktime_add_us(ktime_get(), REFCLK_REQ_TIMEOUT_US);
do {
+ time_checked = ktime_get();
value = ufshcd_readl(hba, REG_UFS_REFCLK_CTRL);
/* Wait until ack bit equals to req bit */
@@ -144,7 +173,7 @@
goto out;
usleep_range(100, 200);
- } while (time_before(jiffies, timeout));
+ } while (ktime_before(time_checked, timeout));
dev_err(hba->dev, "missing ack of refclk req, reg: 0x%x\n", value);
@@ -177,15 +206,47 @@
host->ref_clk_ungating_wait_us = ungating_us;
}
-static u32 ufs_mtk_link_get_state(struct ufs_hba *hba)
+static int ufs_mtk_wait_link_state(struct ufs_hba *hba, u32 state,
+ unsigned long max_wait_ms)
{
+ ktime_t timeout, time_checked;
u32 val;
- ufshcd_writel(hba, 0x20, REG_UFS_DEBUG_SEL);
- val = ufshcd_readl(hba, REG_UFS_PROBE);
- val = val >> 28;
+ timeout = ktime_add_us(ktime_get(), ms_to_ktime(max_wait_ms));
+ do {
+ time_checked = ktime_get();
+ ufshcd_writel(hba, 0x20, REG_UFS_DEBUG_SEL);
+ val = ufshcd_readl(hba, REG_UFS_PROBE);
+ val = val >> 28;
- return val;
+ if (val == state)
+ return 0;
+
+ /* Sleep for max. 200us */
+ usleep_range(100, 200);
+ } while (ktime_before(time_checked, timeout));
+
+ if (val == state)
+ return 0;
+
+ return -ETIMEDOUT;
+}
+
+static void ufs_mtk_mphy_power_on(struct ufs_hba *hba, bool on)
+{
+ struct ufs_mtk_host *host = ufshcd_get_variant(hba);
+ struct phy *mphy = host->mphy;
+
+ if (!mphy)
+ return;
+
+ if (on && !host->mphy_powered_on)
+ phy_power_on(mphy);
+ else if (!on && host->mphy_powered_on)
+ phy_power_off(mphy);
+ else
+ return;
+ host->mphy_powered_on = on;
}
/**
@@ -201,6 +262,7 @@
{
struct ufs_mtk_host *host = ufshcd_get_variant(hba);
int ret = 0;
+ bool clk_pwr_off = false;
/*
* In case ufs_mtk_init() is not yet done, simply ignore.
@@ -211,22 +273,29 @@
return 0;
if (!on && status == PRE_CHANGE) {
- if (!ufshcd_is_link_active(hba)) {
- ufs_mtk_setup_ref_clk(hba, on);
- ret = phy_power_off(host->mphy);
- } else {
+ if (ufshcd_is_link_off(hba)) {
+ clk_pwr_off = true;
+ } else if (ufshcd_is_link_hibern8(hba) ||
+ (!ufshcd_can_hibern8_during_gating(hba) &&
+ ufshcd_is_auto_hibern8_enabled(hba))) {
/*
- * Gate ref-clk if link state is in Hibern8
- * triggered by Auto-Hibern8.
+ * Gate ref-clk and poweroff mphy if link state is in
+ * OFF or Hibern8 by either Auto-Hibern8 or
+ * ufshcd_link_state_transition().
*/
- if (!ufshcd_can_hibern8_during_gating(hba) &&
- ufshcd_is_auto_hibern8_enabled(hba) &&
- ufs_mtk_link_get_state(hba) ==
- VS_LINK_HIBERN8)
- ufs_mtk_setup_ref_clk(hba, on);
+ ret = ufs_mtk_wait_link_state(hba,
+ VS_LINK_HIBERN8,
+ 15);
+ if (!ret)
+ clk_pwr_off = true;
+ }
+
+ if (clk_pwr_off) {
+ ufs_mtk_setup_ref_clk(hba, on);
+ ufs_mtk_mphy_power_on(hba, on);
}
} else if (on && status == POST_CHANGE) {
- ret = phy_power_on(host->mphy);
+ ufs_mtk_mphy_power_on(hba, on);
ufs_mtk_setup_ref_clk(hba, on);
}
@@ -269,6 +338,9 @@
/* Enable clock-gating */
hba->caps |= UFSHCD_CAP_CLK_GATING;
+ /* Enable inline encryption */
+ hba->caps |= UFSHCD_CAP_CRYPTO;
+
/* Enable WriteBooster */
hba->caps |= UFSHCD_CAP_WB_EN;
hba->vps->wb_flush_threshold = UFS_WB_BUF_REMAIN_PERCENT(80);
@@ -502,10 +574,22 @@
return 0;
}
+static void ufs_mtk_vreg_set_lpm(struct ufs_hba *hba, bool lpm)
+{
+ if (!hba->vreg_info.vccq2 || !hba->vreg_info.vcc)
+ return;
+
+ if (lpm & !hba->vreg_info.vcc->enabled)
+ regulator_set_mode(hba->vreg_info.vccq2->reg,
+ REGULATOR_MODE_IDLE);
+ else if (!lpm)
+ regulator_set_mode(hba->vreg_info.vccq2->reg,
+ REGULATOR_MODE_NORMAL);
+}
+
static int ufs_mtk_suspend(struct ufs_hba *hba, enum ufs_pm_op pm_op)
{
int err;
- struct ufs_mtk_host *host = ufshcd_get_variant(hba);
if (ufshcd_is_link_hibern8(hba)) {
err = ufs_mtk_link_set_lpm(hba);
@@ -518,23 +602,23 @@
ufshcd_set_link_off(hba);
return -EAGAIN;
}
+ /*
+ * Make sure no error will be returned to prevent
+ * ufshcd_suspend() re-enabling regulators while vreg is still
+ * in low-power mode.
+ */
+ ufs_mtk_vreg_set_lpm(hba, true);
}
- if (!ufshcd_is_link_active(hba))
- phy_power_off(host->mphy);
-
return 0;
}
static int ufs_mtk_resume(struct ufs_hba *hba, enum ufs_pm_op pm_op)
{
- struct ufs_mtk_host *host = ufshcd_get_variant(hba);
int err;
- if (!ufshcd_is_link_active(hba))
- phy_power_on(host->mphy);
-
if (ufshcd_is_link_hibern8(hba)) {
+ ufs_mtk_vreg_set_lpm(hba, false);
err = ufs_mtk_link_set_hpm(hba);
if (err) {
err = ufshcd_link_recovery(hba);
diff --git a/drivers/scsi/ufs/ufs-mediatek.h b/drivers/scsi/ufs/ufs-mediatek.h
index 5bbd3e9..8ed24d5 100644
--- a/drivers/scsi/ufs/ufs-mediatek.h
+++ b/drivers/scsi/ufs/ufs-mediatek.h
@@ -28,7 +28,7 @@
#define REFCLK_REQUEST BIT(0)
#define REFCLK_ACK BIT(1)
-#define REFCLK_REQ_TIMEOUT_MS 3
+#define REFCLK_REQ_TIMEOUT_US 3000
/*
* Vendor specific pre-defined parameters
@@ -70,6 +70,7 @@
*/
#define MTK_SIP_UFS_CONTROL MTK_SIP_SMC_CMD(0x276)
#define UFS_MTK_SIP_DEVICE_RESET BIT(1)
+#define UFS_MTK_SIP_CRYPTO_CTRL BIT(2)
#define UFS_MTK_SIP_REF_CLK_NOTIFICATION BIT(3)
/*
@@ -91,6 +92,7 @@
struct ufs_mtk_host {
struct ufs_hba *hba;
struct phy *mphy;
+ bool mphy_powered_on;
bool unipro_lpm;
bool ref_clk_enabled;
u16 ref_clk_ungating_wait_us;
diff --git a/drivers/scsi/ufs/ufs-qcom-ice.c b/drivers/scsi/ufs/ufs-qcom-ice.c
new file mode 100644
index 0000000..bbb0ad7
--- /dev/null
+++ b/drivers/scsi/ufs/ufs-qcom-ice.c
@@ -0,0 +1,245 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Qualcomm ICE (Inline Crypto Engine) support.
+ *
+ * Copyright (c) 2014-2019, The Linux Foundation. All rights reserved.
+ * Copyright 2019 Google LLC
+ */
+
+#include <linux/platform_device.h>
+#include <linux/qcom_scm.h>
+
+#include "ufshcd-crypto.h"
+#include "ufs-qcom.h"
+
+#define AES_256_XTS_KEY_SIZE 64
+
+/* QCOM ICE registers */
+
+#define QCOM_ICE_REG_CONTROL 0x0000
+#define QCOM_ICE_REG_RESET 0x0004
+#define QCOM_ICE_REG_VERSION 0x0008
+#define QCOM_ICE_REG_FUSE_SETTING 0x0010
+#define QCOM_ICE_REG_PARAMETERS_1 0x0014
+#define QCOM_ICE_REG_PARAMETERS_2 0x0018
+#define QCOM_ICE_REG_PARAMETERS_3 0x001C
+#define QCOM_ICE_REG_PARAMETERS_4 0x0020
+#define QCOM_ICE_REG_PARAMETERS_5 0x0024
+
+/* QCOM ICE v3.X only */
+#define QCOM_ICE_GENERAL_ERR_STTS 0x0040
+#define QCOM_ICE_INVALID_CCFG_ERR_STTS 0x0030
+#define QCOM_ICE_GENERAL_ERR_MASK 0x0044
+
+/* QCOM ICE v2.X only */
+#define QCOM_ICE_REG_NON_SEC_IRQ_STTS 0x0040
+#define QCOM_ICE_REG_NON_SEC_IRQ_MASK 0x0044
+
+#define QCOM_ICE_REG_NON_SEC_IRQ_CLR 0x0048
+#define QCOM_ICE_REG_STREAM1_ERROR_SYNDROME1 0x0050
+#define QCOM_ICE_REG_STREAM1_ERROR_SYNDROME2 0x0054
+#define QCOM_ICE_REG_STREAM2_ERROR_SYNDROME1 0x0058
+#define QCOM_ICE_REG_STREAM2_ERROR_SYNDROME2 0x005C
+#define QCOM_ICE_REG_STREAM1_BIST_ERROR_VEC 0x0060
+#define QCOM_ICE_REG_STREAM2_BIST_ERROR_VEC 0x0064
+#define QCOM_ICE_REG_STREAM1_BIST_FINISH_VEC 0x0068
+#define QCOM_ICE_REG_STREAM2_BIST_FINISH_VEC 0x006C
+#define QCOM_ICE_REG_BIST_STATUS 0x0070
+#define QCOM_ICE_REG_BYPASS_STATUS 0x0074
+#define QCOM_ICE_REG_ADVANCED_CONTROL 0x1000
+#define QCOM_ICE_REG_ENDIAN_SWAP 0x1004
+#define QCOM_ICE_REG_TEST_BUS_CONTROL 0x1010
+#define QCOM_ICE_REG_TEST_BUS_REG 0x1014
+
+/* BIST ("built-in self-test"?) status flags */
+#define QCOM_ICE_BIST_STATUS_MASK 0xF0000000
+
+#define QCOM_ICE_FUSE_SETTING_MASK 0x1
+#define QCOM_ICE_FORCE_HW_KEY0_SETTING_MASK 0x2
+#define QCOM_ICE_FORCE_HW_KEY1_SETTING_MASK 0x4
+
+#define qcom_ice_writel(host, val, reg) \
+ writel((val), (host)->ice_mmio + (reg))
+#define qcom_ice_readl(host, reg) \
+ readl((host)->ice_mmio + (reg))
+
+static bool qcom_ice_supported(struct ufs_qcom_host *host)
+{
+ struct device *dev = host->hba->dev;
+ u32 regval = qcom_ice_readl(host, QCOM_ICE_REG_VERSION);
+ int major = regval >> 24;
+ int minor = (regval >> 16) & 0xFF;
+ int step = regval & 0xFFFF;
+
+ /* For now this driver only supports ICE version 3. */
+ if (major != 3) {
+ dev_warn(dev, "Unsupported ICE version: v%d.%d.%d\n",
+ major, minor, step);
+ return false;
+ }
+
+ dev_info(dev, "Found QC Inline Crypto Engine (ICE) v%d.%d.%d\n",
+ major, minor, step);
+
+ /* If fuses are blown, ICE might not work in the standard way. */
+ regval = qcom_ice_readl(host, QCOM_ICE_REG_FUSE_SETTING);
+ if (regval & (QCOM_ICE_FUSE_SETTING_MASK |
+ QCOM_ICE_FORCE_HW_KEY0_SETTING_MASK |
+ QCOM_ICE_FORCE_HW_KEY1_SETTING_MASK)) {
+ dev_warn(dev, "Fuses are blown; ICE is unusable!\n");
+ return false;
+ }
+ return true;
+}
+
+int ufs_qcom_ice_init(struct ufs_qcom_host *host)
+{
+ struct ufs_hba *hba = host->hba;
+ struct device *dev = hba->dev;
+ struct platform_device *pdev = to_platform_device(dev);
+ struct resource *res;
+ int err;
+
+ if (!(ufshcd_readl(hba, REG_CONTROLLER_CAPABILITIES) &
+ MASK_CRYPTO_SUPPORT))
+ return 0;
+
+ res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "ice");
+ if (!res) {
+ dev_warn(dev, "ICE registers not found\n");
+ goto disable;
+ }
+
+ if (!qcom_scm_ice_available()) {
+ dev_warn(dev, "ICE SCM interface not found\n");
+ goto disable;
+ }
+
+ host->ice_mmio = devm_ioremap_resource(dev, res);
+ if (IS_ERR(host->ice_mmio)) {
+ err = PTR_ERR(host->ice_mmio);
+ dev_err(dev, "Failed to map ICE registers; err=%d\n", err);
+ return err;
+ }
+
+ if (!qcom_ice_supported(host))
+ goto disable;
+
+ return 0;
+
+disable:
+ dev_warn(dev, "Disabling inline encryption support\n");
+ hba->caps &= ~UFSHCD_CAP_CRYPTO;
+ return 0;
+}
+
+static void qcom_ice_low_power_mode_enable(struct ufs_qcom_host *host)
+{
+ u32 regval;
+
+ regval = qcom_ice_readl(host, QCOM_ICE_REG_ADVANCED_CONTROL);
+ /*
+ * Enable low power mode sequence
+ * [0]-0, [1]-0, [2]-0, [3]-E, [4]-0, [5]-0, [6]-0, [7]-0
+ */
+ regval |= 0x7000;
+ qcom_ice_writel(host, regval, QCOM_ICE_REG_ADVANCED_CONTROL);
+}
+
+static void qcom_ice_optimization_enable(struct ufs_qcom_host *host)
+{
+ u32 regval;
+
+ /* ICE Optimizations Enable Sequence */
+ regval = qcom_ice_readl(host, QCOM_ICE_REG_ADVANCED_CONTROL);
+ regval |= 0xD807100;
+ /* ICE HPG requires delay before writing */
+ udelay(5);
+ qcom_ice_writel(host, regval, QCOM_ICE_REG_ADVANCED_CONTROL);
+ udelay(5);
+}
+
+int ufs_qcom_ice_enable(struct ufs_qcom_host *host)
+{
+ if (!(host->hba->caps & UFSHCD_CAP_CRYPTO))
+ return 0;
+ qcom_ice_low_power_mode_enable(host);
+ qcom_ice_optimization_enable(host);
+ return ufs_qcom_ice_resume(host);
+}
+
+/* Poll until all BIST bits are reset */
+static int qcom_ice_wait_bist_status(struct ufs_qcom_host *host)
+{
+ int count;
+ u32 reg;
+
+ for (count = 0; count < 100; count++) {
+ reg = qcom_ice_readl(host, QCOM_ICE_REG_BIST_STATUS);
+ if (!(reg & QCOM_ICE_BIST_STATUS_MASK))
+ break;
+ udelay(50);
+ }
+ if (reg)
+ return -ETIMEDOUT;
+ return 0;
+}
+
+int ufs_qcom_ice_resume(struct ufs_qcom_host *host)
+{
+ int err;
+
+ if (!(host->hba->caps & UFSHCD_CAP_CRYPTO))
+ return 0;
+
+ err = qcom_ice_wait_bist_status(host);
+ if (err) {
+ dev_err(host->hba->dev, "BIST status error (%d)\n", err);
+ return err;
+ }
+ return 0;
+}
+
+/*
+ * Program a key into a QC ICE keyslot, or evict a keyslot. QC ICE requires
+ * vendor-specific SCM calls for this; it doesn't support the standard way.
+ */
+int ufs_qcom_ice_program_key(struct ufs_hba *hba,
+ const union ufs_crypto_cfg_entry *cfg, int slot)
+{
+ union ufs_crypto_cap_entry cap;
+ union {
+ u8 bytes[AES_256_XTS_KEY_SIZE];
+ u32 words[AES_256_XTS_KEY_SIZE / sizeof(u32)];
+ } key;
+ int i;
+ int err;
+
+ if (!(cfg->config_enable & UFS_CRYPTO_CONFIGURATION_ENABLE))
+ return qcom_scm_ice_invalidate_key(slot);
+
+ /* Only AES-256-XTS has been tested so far. */
+ cap = hba->crypto_cap_array[cfg->crypto_cap_idx];
+ if (cap.algorithm_id != UFS_CRYPTO_ALG_AES_XTS ||
+ cap.key_size != UFS_CRYPTO_KEY_SIZE_256) {
+ dev_err_ratelimited(hba->dev,
+ "Unhandled crypto capability; algorithm_id=%d, key_size=%d\n",
+ cap.algorithm_id, cap.key_size);
+ return -EINVAL;
+ }
+
+ memcpy(key.bytes, cfg->crypto_key, AES_256_XTS_KEY_SIZE);
+
+ /*
+ * The SCM call byte-swaps the 32-bit words of the key. So we have to
+ * do the same, in order for the final key be correct.
+ */
+ for (i = 0; i < ARRAY_SIZE(key.words); i++)
+ __cpu_to_be32s(&key.words[i]);
+
+ err = qcom_scm_ice_set_key(slot, key.bytes, AES_256_XTS_KEY_SIZE,
+ QCOM_SCM_ICE_CIPHER_AES_256_XTS,
+ cfg->data_unit_size);
+ memzero_explicit(&key, sizeof(key));
+ return err;
+}
diff --git a/drivers/scsi/ufs/ufs-qcom.c b/drivers/scsi/ufs/ufs-qcom.c
index 2e6ddb5..d0d7552 100644
--- a/drivers/scsi/ufs/ufs-qcom.c
+++ b/drivers/scsi/ufs/ufs-qcom.c
@@ -246,7 +246,7 @@
mb();
}
-/**
+/*
* ufs_qcom_host_reset - reset host controller and PHY
*/
static int ufs_qcom_host_reset(struct ufs_hba *hba)
@@ -365,7 +365,7 @@
/* check if UFS PHY moved from DISABLED to HIBERN8 */
err = ufs_qcom_check_hibern8(hba);
ufs_qcom_enable_hw_clk_gating(hba);
-
+ ufs_qcom_ice_enable(host);
break;
default:
dev_err(hba->dev, "%s: invalid status %d\n", __func__, status);
@@ -375,7 +375,7 @@
return err;
}
-/**
+/*
* Returns zero for success and non-zero in case of a failure
*/
static int ufs_qcom_cfg_timers(struct ufs_hba *hba, u32 gear,
@@ -613,6 +613,10 @@
return err;
}
+ err = ufs_qcom_ice_resume(host);
+ if (err)
+ return err;
+
hba->is_sys_suspended = false;
return 0;
}
@@ -1071,6 +1075,7 @@
hba->caps |= UFSHCD_CAP_CLK_SCALING;
hba->caps |= UFSHCD_CAP_AUTO_BKOPS_SUSPEND;
hba->caps |= UFSHCD_CAP_WB_EN;
+ hba->caps |= UFSHCD_CAP_CRYPTO;
if (host->hw_ver.major >= 0x2) {
host->caps = UFS_QCOM_CAP_QUNIPRO |
@@ -1275,7 +1280,8 @@
host->dev_ref_clk_en_mask = BIT(26);
} else {
/* "dev_ref_clk_ctrl_mem" is optional resource */
- res = platform_get_resource(pdev, IORESOURCE_MEM, 1);
+ res = platform_get_resource_byname(pdev, IORESOURCE_MEM,
+ "dev_ref_clk_ctrl_mem");
if (res) {
host->dev_ref_clk_ctrl_mmio =
devm_ioremap_resource(dev, res);
@@ -1297,6 +1303,10 @@
ufs_qcom_set_caps(hba);
ufs_qcom_advertise_quirks(hba);
+ err = ufs_qcom_ice_init(host);
+ if (err)
+ goto out_variant_clear;
+
ufs_qcom_set_bus_vote(hba, true);
ufs_qcom_setup_clocks(hba, true, POST_CHANGE);
@@ -1713,7 +1723,7 @@
}
#endif
-/**
+/*
* struct ufs_hba_qcom_vops - UFS QCOM specific variant operations
*
* The variant operations configure the necessary controller and PHY
@@ -1735,6 +1745,7 @@
.dbg_register_dump = ufs_qcom_dump_dbg_regs,
.device_reset = ufs_qcom_device_reset,
.config_scaling_param = ufs_qcom_config_scaling_param,
+ .program_key = ufs_qcom_ice_program_key,
};
/**
diff --git a/drivers/scsi/ufs/ufs-qcom.h b/drivers/scsi/ufs/ufs-qcom.h
index 2d95e7c..97247d1 100644
--- a/drivers/scsi/ufs/ufs-qcom.h
+++ b/drivers/scsi/ufs/ufs-qcom.h
@@ -227,6 +227,9 @@
void __iomem *dev_ref_clk_ctrl_mmio;
bool is_dev_ref_clk_enabled;
struct ufs_hw_version hw_ver;
+#ifdef CONFIG_SCSI_UFS_CRYPTO
+ void __iomem *ice_mmio;
+#endif
u32 dev_ref_clk_en_mask;
@@ -264,4 +267,28 @@
return false;
}
+/* ufs-qcom-ice.c */
+
+#ifdef CONFIG_SCSI_UFS_CRYPTO
+int ufs_qcom_ice_init(struct ufs_qcom_host *host);
+int ufs_qcom_ice_enable(struct ufs_qcom_host *host);
+int ufs_qcom_ice_resume(struct ufs_qcom_host *host);
+int ufs_qcom_ice_program_key(struct ufs_hba *hba,
+ const union ufs_crypto_cfg_entry *cfg, int slot);
+#else
+static inline int ufs_qcom_ice_init(struct ufs_qcom_host *host)
+{
+ return 0;
+}
+static inline int ufs_qcom_ice_enable(struct ufs_qcom_host *host)
+{
+ return 0;
+}
+static inline int ufs_qcom_ice_resume(struct ufs_qcom_host *host)
+{
+ return 0;
+}
+#define ufs_qcom_ice_program_key NULL
+#endif /* !CONFIG_SCSI_UFS_CRYPTO */
+
#endif /* UFS_QCOM_H_ */
diff --git a/drivers/scsi/ufs/ufs.h b/drivers/scsi/ufs/ufs.h
index c70845d..f8ab16f 100644
--- a/drivers/scsi/ufs/ufs.h
+++ b/drivers/scsi/ufs/ufs.h
@@ -1,36 +1,11 @@
+/* SPDX-License-Identifier: GPL-2.0-or-later */
/*
* Universal Flash Storage Host controller driver
- *
- * This code is based on drivers/scsi/ufs/ufs.h
* Copyright (C) 2011-2013 Samsung India Software Operations
*
* Authors:
* Santosh Yaraganavi <santosh.sy@samsung.com>
* Vinayak Holikatti <h.vinayak@samsung.com>
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License
- * as published by the Free Software Foundation; either version 2
- * of the License, or (at your option) any later version.
- * See the COPYING file in the top-level directory or visit
- * <http://www.gnu.org/licenses/gpl-2.0.html>
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU General Public License for more details.
- *
- * This program is provided "AS IS" and "WITH ALL FAULTS" and
- * without warranty of any kind. You are solely responsible for
- * determining the appropriateness of using and distributing
- * the program and assume all risks associated with your exercise
- * of rights with respect to the program, including but not limited
- * to infringement of third party rights, the risks and costs of
- * program errors, damage to or loss of data, programs or equipment,
- * and unavailability or interruption of operations. Under no
- * circumstances will the contributor of this Program be liable for
- * any damages of any kind arising from your use or distribution of
- * this program.
*/
#ifndef _UFS_H
@@ -63,6 +38,7 @@
#define UFS_UPIU_MAX_UNIT_NUM_ID 0x7F
#define UFS_MAX_LUNS (SCSI_W_LUN_BASE + UFS_UPIU_MAX_UNIT_NUM_ID)
#define UFS_UPIU_WLUN_ID (1 << 7)
+#define UFS_RPMB_UNIT 0xC4
/* WriteBooster buffer is available only for the logical unit from 0 to 7 */
#define UFS_UPIU_MAX_WB_LUN_ID 8
@@ -200,16 +176,6 @@
QUERY_DESC_DESC_TYPE_OFFSET = 0x01,
};
-enum ufs_desc_def_size {
- QUERY_DESC_DEVICE_DEF_SIZE = 0x59,
- QUERY_DESC_CONFIGURATION_DEF_SIZE = 0x90,
- QUERY_DESC_UNIT_DEF_SIZE = 0x2D,
- QUERY_DESC_INTERCONNECT_DEF_SIZE = 0x06,
- QUERY_DESC_GEOMETRY_DEF_SIZE = 0x48,
- QUERY_DESC_POWER_DEF_SIZE = 0x62,
- QUERY_DESC_HEALTH_DEF_SIZE = 0x25,
-};
-
/* Unit descriptor parameters offsets in bytes*/
enum unit_desc_param {
UNIT_DESC_PARAM_LEN = 0x0,
diff --git a/drivers/scsi/ufs/ufs_bsg.c b/drivers/scsi/ufs/ufs_bsg.c
index 516a7f5..bcfbbd0 100644
--- a/drivers/scsi/ufs/ufs_bsg.c
+++ b/drivers/scsi/ufs/ufs_bsg.c
@@ -11,13 +11,12 @@
{
int desc_size = be16_to_cpu(qr->length);
int desc_id = qr->idn;
- int ret;
if (desc_size <= 0)
return -EINVAL;
- ret = ufshcd_map_desc_id_to_length(hba, desc_id, desc_len);
- if (ret || !*desc_len)
+ ufshcd_map_desc_id_to_length(hba, desc_id, desc_len);
+ if (!*desc_len)
return -EINVAL;
*desc_len = min_t(int, *desc_len, desc_size);
diff --git a/drivers/scsi/ufs/ufs_quirks.h b/drivers/scsi/ufs/ufs_quirks.h
index e3175a6..07f559a 100644
--- a/drivers/scsi/ufs/ufs_quirks.h
+++ b/drivers/scsi/ufs/ufs_quirks.h
@@ -12,9 +12,10 @@
#define UFS_ANY_VENDOR 0xFFFF
#define UFS_ANY_MODEL "ANY_MODEL"
-#define UFS_VENDOR_TOSHIBA 0x198
+#define UFS_VENDOR_MICRON 0x12C
#define UFS_VENDOR_SAMSUNG 0x1CE
#define UFS_VENDOR_SKHYNIX 0x1AD
+#define UFS_VENDOR_TOSHIBA 0x198
#define UFS_VENDOR_WDC 0x145
/**
@@ -108,4 +109,11 @@
*/
#define UFS_DEVICE_QUIRK_SUPPORT_EXTENDED_FEATURES (1 << 10)
+/*
+ * Some UFS devices require delay after VCC power rail is turned-off.
+ * Enable this quirk to introduce 5ms delays after VCC power-off during
+ * suspend flow.
+ */
+#define UFS_DEVICE_QUIRK_DELAY_AFTER_LPM (1 << 11)
+
#endif /* UFS_QUIRKS_H_ */
diff --git a/drivers/scsi/ufs/ufshcd-crypto.c b/drivers/scsi/ufs/ufshcd-crypto.c
new file mode 100644
index 0000000..d2edbd9
--- /dev/null
+++ b/drivers/scsi/ufs/ufshcd-crypto.c
@@ -0,0 +1,245 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright 2019 Google LLC
+ */
+
+#include "ufshcd.h"
+#include "ufshcd-crypto.h"
+
+/* Blk-crypto modes supported by UFS crypto */
+static const struct ufs_crypto_alg_entry {
+ enum ufs_crypto_alg ufs_alg;
+ enum ufs_crypto_key_size ufs_key_size;
+} ufs_crypto_algs[BLK_ENCRYPTION_MODE_MAX] = {
+ [BLK_ENCRYPTION_MODE_AES_256_XTS] = {
+ .ufs_alg = UFS_CRYPTO_ALG_AES_XTS,
+ .ufs_key_size = UFS_CRYPTO_KEY_SIZE_256,
+ },
+};
+
+static int ufshcd_program_key(struct ufs_hba *hba,
+ const union ufs_crypto_cfg_entry *cfg, int slot)
+{
+ int i;
+ u32 slot_offset = hba->crypto_cfg_register + slot * sizeof(*cfg);
+ int err = 0;
+
+ ufshcd_hold(hba, false);
+
+ if (hba->vops && hba->vops->program_key) {
+ err = hba->vops->program_key(hba, cfg, slot);
+ goto out;
+ }
+
+ /* Ensure that CFGE is cleared before programming the key */
+ ufshcd_writel(hba, 0, slot_offset + 16 * sizeof(cfg->reg_val[0]));
+ for (i = 0; i < 16; i++) {
+ ufshcd_writel(hba, le32_to_cpu(cfg->reg_val[i]),
+ slot_offset + i * sizeof(cfg->reg_val[0]));
+ }
+ /* Write dword 17 */
+ ufshcd_writel(hba, le32_to_cpu(cfg->reg_val[17]),
+ slot_offset + 17 * sizeof(cfg->reg_val[0]));
+ /* Dword 16 must be written last */
+ ufshcd_writel(hba, le32_to_cpu(cfg->reg_val[16]),
+ slot_offset + 16 * sizeof(cfg->reg_val[0]));
+out:
+ ufshcd_release(hba);
+ return err;
+}
+
+static int ufshcd_crypto_keyslot_program(struct blk_keyslot_manager *ksm,
+ const struct blk_crypto_key *key,
+ unsigned int slot)
+{
+ struct ufs_hba *hba = container_of(ksm, struct ufs_hba, ksm);
+ const union ufs_crypto_cap_entry *ccap_array = hba->crypto_cap_array;
+ const struct ufs_crypto_alg_entry *alg =
+ &ufs_crypto_algs[key->crypto_cfg.crypto_mode];
+ u8 data_unit_mask = key->crypto_cfg.data_unit_size / 512;
+ int i;
+ int cap_idx = -1;
+ union ufs_crypto_cfg_entry cfg = { 0 };
+ int err;
+
+ BUILD_BUG_ON(UFS_CRYPTO_KEY_SIZE_INVALID != 0);
+ for (i = 0; i < hba->crypto_capabilities.num_crypto_cap; i++) {
+ if (ccap_array[i].algorithm_id == alg->ufs_alg &&
+ ccap_array[i].key_size == alg->ufs_key_size &&
+ (ccap_array[i].sdus_mask & data_unit_mask)) {
+ cap_idx = i;
+ break;
+ }
+ }
+
+ if (WARN_ON(cap_idx < 0))
+ return -EOPNOTSUPP;
+
+ cfg.data_unit_size = data_unit_mask;
+ cfg.crypto_cap_idx = cap_idx;
+ cfg.config_enable = UFS_CRYPTO_CONFIGURATION_ENABLE;
+
+ if (ccap_array[cap_idx].algorithm_id == UFS_CRYPTO_ALG_AES_XTS) {
+ /* In XTS mode, the blk_crypto_key's size is already doubled */
+ memcpy(cfg.crypto_key, key->raw, key->size/2);
+ memcpy(cfg.crypto_key + UFS_CRYPTO_KEY_MAX_SIZE/2,
+ key->raw + key->size/2, key->size/2);
+ } else {
+ memcpy(cfg.crypto_key, key->raw, key->size);
+ }
+
+ err = ufshcd_program_key(hba, &cfg, slot);
+
+ memzero_explicit(&cfg, sizeof(cfg));
+ return err;
+}
+
+static int ufshcd_clear_keyslot(struct ufs_hba *hba, int slot)
+{
+ /*
+ * Clear the crypto cfg on the device. Clearing CFGE
+ * might not be sufficient, so just clear the entire cfg.
+ */
+ union ufs_crypto_cfg_entry cfg = { 0 };
+
+ return ufshcd_program_key(hba, &cfg, slot);
+}
+
+static int ufshcd_crypto_keyslot_evict(struct blk_keyslot_manager *ksm,
+ const struct blk_crypto_key *key,
+ unsigned int slot)
+{
+ struct ufs_hba *hba = container_of(ksm, struct ufs_hba, ksm);
+
+ return ufshcd_clear_keyslot(hba, slot);
+}
+
+bool ufshcd_crypto_enable(struct ufs_hba *hba)
+{
+ if (!(hba->caps & UFSHCD_CAP_CRYPTO))
+ return false;
+
+ /* Reset might clear all keys, so reprogram all the keys. */
+ blk_ksm_reprogram_all_keys(&hba->ksm);
+ return true;
+}
+
+static const struct blk_ksm_ll_ops ufshcd_ksm_ops = {
+ .keyslot_program = ufshcd_crypto_keyslot_program,
+ .keyslot_evict = ufshcd_crypto_keyslot_evict,
+};
+
+static enum blk_crypto_mode_num
+ufshcd_find_blk_crypto_mode(union ufs_crypto_cap_entry cap)
+{
+ int i;
+
+ for (i = 0; i < ARRAY_SIZE(ufs_crypto_algs); i++) {
+ BUILD_BUG_ON(UFS_CRYPTO_KEY_SIZE_INVALID != 0);
+ if (ufs_crypto_algs[i].ufs_alg == cap.algorithm_id &&
+ ufs_crypto_algs[i].ufs_key_size == cap.key_size) {
+ return i;
+ }
+ }
+ return BLK_ENCRYPTION_MODE_INVALID;
+}
+
+/**
+ * ufshcd_hba_init_crypto_capabilities - Read crypto capabilities, init crypto
+ * fields in hba
+ * @hba: Per adapter instance
+ *
+ * Return: 0 if crypto was initialized or is not supported, else a -errno value.
+ */
+int ufshcd_hba_init_crypto_capabilities(struct ufs_hba *hba)
+{
+ int cap_idx;
+ int err = 0;
+ enum blk_crypto_mode_num blk_mode_num;
+
+ /*
+ * Don't use crypto if either the hardware doesn't advertise the
+ * standard crypto capability bit *or* if the vendor specific driver
+ * hasn't advertised that crypto is supported.
+ */
+ if (!(hba->capabilities & MASK_CRYPTO_SUPPORT) ||
+ !(hba->caps & UFSHCD_CAP_CRYPTO))
+ goto out;
+
+ hba->crypto_capabilities.reg_val =
+ cpu_to_le32(ufshcd_readl(hba, REG_UFS_CCAP));
+ hba->crypto_cfg_register =
+ (u32)hba->crypto_capabilities.config_array_ptr * 0x100;
+ hba->crypto_cap_array =
+ devm_kcalloc(hba->dev, hba->crypto_capabilities.num_crypto_cap,
+ sizeof(hba->crypto_cap_array[0]), GFP_KERNEL);
+ if (!hba->crypto_cap_array) {
+ err = -ENOMEM;
+ goto out;
+ }
+
+ /* The actual number of configurations supported is (CFGC+1) */
+ err = blk_ksm_init(&hba->ksm,
+ hba->crypto_capabilities.config_count + 1);
+ if (err)
+ goto out_free_caps;
+
+ hba->ksm.ksm_ll_ops = ufshcd_ksm_ops;
+ /* UFS only supports 8 bytes for any DUN */
+ hba->ksm.max_dun_bytes_supported = 8;
+ hba->ksm.dev = hba->dev;
+
+ /*
+ * Cache all the UFS crypto capabilities and advertise the supported
+ * crypto modes and data unit sizes to the block layer.
+ */
+ for (cap_idx = 0; cap_idx < hba->crypto_capabilities.num_crypto_cap;
+ cap_idx++) {
+ hba->crypto_cap_array[cap_idx].reg_val =
+ cpu_to_le32(ufshcd_readl(hba,
+ REG_UFS_CRYPTOCAP +
+ cap_idx * sizeof(__le32)));
+ blk_mode_num = ufshcd_find_blk_crypto_mode(
+ hba->crypto_cap_array[cap_idx]);
+ if (blk_mode_num != BLK_ENCRYPTION_MODE_INVALID)
+ hba->ksm.crypto_modes_supported[blk_mode_num] |=
+ hba->crypto_cap_array[cap_idx].sdus_mask * 512;
+ }
+
+ return 0;
+
+out_free_caps:
+ devm_kfree(hba->dev, hba->crypto_cap_array);
+out:
+ /* Indicate that init failed by clearing UFSHCD_CAP_CRYPTO */
+ hba->caps &= ~UFSHCD_CAP_CRYPTO;
+ return err;
+}
+
+/**
+ * ufshcd_init_crypto - Initialize crypto hardware
+ * @hba: Per adapter instance
+ */
+void ufshcd_init_crypto(struct ufs_hba *hba)
+{
+ int slot;
+
+ if (!(hba->caps & UFSHCD_CAP_CRYPTO))
+ return;
+
+ /* Clear all keyslots - the number of keyslots is (CFGC + 1) */
+ for (slot = 0; slot < hba->crypto_capabilities.config_count + 1; slot++)
+ ufshcd_clear_keyslot(hba, slot);
+}
+
+void ufshcd_crypto_setup_rq_keyslot_manager(struct ufs_hba *hba,
+ struct request_queue *q)
+{
+ if (hba->caps & UFSHCD_CAP_CRYPTO)
+ blk_ksm_register(&hba->ksm, q);
+}
+
+void ufshcd_crypto_destroy_keyslot_manager(struct ufs_hba *hba)
+{
+ blk_ksm_destroy(&hba->ksm);
+}
diff --git a/drivers/scsi/ufs/ufshcd-crypto.h b/drivers/scsi/ufs/ufshcd-crypto.h
new file mode 100644
index 0000000..d53851b
--- /dev/null
+++ b/drivers/scsi/ufs/ufshcd-crypto.h
@@ -0,0 +1,77 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright 2019 Google LLC
+ */
+
+#ifndef _UFSHCD_CRYPTO_H
+#define _UFSHCD_CRYPTO_H
+
+#ifdef CONFIG_SCSI_UFS_CRYPTO
+#include "ufshcd.h"
+#include "ufshci.h"
+
+static inline void ufshcd_prepare_lrbp_crypto(struct request *rq,
+ struct ufshcd_lrb *lrbp)
+{
+ if (!rq || !rq->crypt_keyslot) {
+ lrbp->crypto_key_slot = -1;
+ return;
+ }
+
+ lrbp->crypto_key_slot = blk_ksm_get_slot_idx(rq->crypt_keyslot);
+ lrbp->data_unit_num = rq->crypt_ctx->bc_dun[0];
+}
+
+static inline void
+ufshcd_prepare_req_desc_hdr_crypto(struct ufshcd_lrb *lrbp, u32 *dword_0,
+ u32 *dword_1, u32 *dword_3)
+{
+ if (lrbp->crypto_key_slot >= 0) {
+ *dword_0 |= UTP_REQ_DESC_CRYPTO_ENABLE_CMD;
+ *dword_0 |= lrbp->crypto_key_slot;
+ *dword_1 = lower_32_bits(lrbp->data_unit_num);
+ *dword_3 = upper_32_bits(lrbp->data_unit_num);
+ }
+}
+
+bool ufshcd_crypto_enable(struct ufs_hba *hba);
+
+int ufshcd_hba_init_crypto_capabilities(struct ufs_hba *hba);
+
+void ufshcd_init_crypto(struct ufs_hba *hba);
+
+void ufshcd_crypto_setup_rq_keyslot_manager(struct ufs_hba *hba,
+ struct request_queue *q);
+
+void ufshcd_crypto_destroy_keyslot_manager(struct ufs_hba *hba);
+
+#else /* CONFIG_SCSI_UFS_CRYPTO */
+
+static inline void ufshcd_prepare_lrbp_crypto(struct request *rq,
+ struct ufshcd_lrb *lrbp) { }
+
+static inline void
+ufshcd_prepare_req_desc_hdr_crypto(struct ufshcd_lrb *lrbp, u32 *dword_0,
+ u32 *dword_1, u32 *dword_3) { }
+
+static inline bool ufshcd_crypto_enable(struct ufs_hba *hba)
+{
+ return false;
+}
+
+static inline int ufshcd_hba_init_crypto_capabilities(struct ufs_hba *hba)
+{
+ return 0;
+}
+
+static inline void ufshcd_init_crypto(struct ufs_hba *hba) { }
+
+static inline void ufshcd_crypto_setup_rq_keyslot_manager(struct ufs_hba *hba,
+ struct request_queue *q) { }
+
+static inline void ufshcd_crypto_destroy_keyslot_manager(struct ufs_hba *hba)
+{ }
+
+#endif /* CONFIG_SCSI_UFS_CRYPTO */
+
+#endif /* _UFSHCD_CRYPTO_H */
diff --git a/drivers/scsi/ufs/ufshcd-pci.c b/drivers/scsi/ufs/ufshcd-pci.c
index 8f78a81..f407b13 100644
--- a/drivers/scsi/ufs/ufshcd-pci.c
+++ b/drivers/scsi/ufs/ufshcd-pci.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
/*
* Universal Flash Storage Host controller PCI glue driver
*
@@ -7,30 +8,6 @@
* Authors:
* Santosh Yaraganavi <santosh.sy@samsung.com>
* Vinayak Holikatti <h.vinayak@samsung.com>
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License
- * as published by the Free Software Foundation; either version 2
- * of the License, or (at your option) any later version.
- * See the COPYING file in the top-level directory or visit
- * <http://www.gnu.org/licenses/gpl-2.0.html>
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU General Public License for more details.
- *
- * This program is provided "AS IS" and "WITH ALL FAULTS" and
- * without warranty of any kind. You are solely responsible for
- * determining the appropriateness of using and distributing
- * the program and assume all risks associated with your exercise
- * of rights with respect to the program, including but not limited
- * to infringement of third party rights, the risks and costs of
- * program errors, damage to or loss of data, programs or equipment,
- * and unavailability or interruption of operations. Under no
- * circumstances will the contributor of this Program be liable for
- * any damages of any kind arising from your use or distribution of
- * this program.
*/
#include "ufshcd.h"
diff --git a/drivers/scsi/ufs/ufshcd-pltfrm.c b/drivers/scsi/ufs/ufshcd-pltfrm.c
index 76f9be7..3db0af6 100644
--- a/drivers/scsi/ufs/ufshcd-pltfrm.c
+++ b/drivers/scsi/ufs/ufshcd-pltfrm.c
@@ -1,36 +1,11 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
/*
* Universal Flash Storage Host controller Platform bus based glue driver
- *
- * This code is based on drivers/scsi/ufs/ufshcd-pltfrm.c
* Copyright (C) 2011-2013 Samsung India Software Operations
*
* Authors:
* Santosh Yaraganavi <santosh.sy@samsung.com>
* Vinayak Holikatti <h.vinayak@samsung.com>
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License
- * as published by the Free Software Foundation; either version 2
- * of the License, or (at your option) any later version.
- * See the COPYING file in the top-level directory or visit
- * <http://www.gnu.org/licenses/gpl-2.0.html>
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU General Public License for more details.
- *
- * This program is provided "AS IS" and "WITH ALL FAULTS" and
- * without warranty of any kind. You are solely responsible for
- * determining the appropriateness of using and distributing
- * the program and assume all risks associated with your exercise
- * of rights with respect to the program, including but not limited
- * to infringement of third party rights, the risks and costs of
- * program errors, damage to or loss of data, programs or equipment,
- * and unavailability or interruption of operations. Under no
- * circumstances will the contributor of this Program be liable for
- * any damages of any kind arising from your use or distribution of
- * this program.
*/
#include <linux/platform_device.h>
diff --git a/drivers/scsi/ufs/ufshcd.c b/drivers/scsi/ufs/ufshcd.c
index ad4fc82..3076222 100644
--- a/drivers/scsi/ufs/ufshcd.c
+++ b/drivers/scsi/ufs/ufshcd.c
@@ -1,40 +1,12 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
/*
* Universal Flash Storage Host controller driver Core
- *
- * This code is based on drivers/scsi/ufs/ufshcd.c
* Copyright (C) 2011-2013 Samsung India Software Operations
* Copyright (c) 2013-2016, The Linux Foundation. All rights reserved.
*
* Authors:
* Santosh Yaraganavi <santosh.sy@samsung.com>
* Vinayak Holikatti <h.vinayak@samsung.com>
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License
- * as published by the Free Software Foundation; either version 2
- * of the License, or (at your option) any later version.
- * See the COPYING file in the top-level directory or visit
- * <http://www.gnu.org/licenses/gpl-2.0.html>
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU General Public License for more details.
- *
- * This program is provided "AS IS" and "WITH ALL FAULTS" and
- * without warranty of any kind. You are solely responsible for
- * determining the appropriateness of using and distributing
- * the program and assume all risks associated with your exercise
- * of rights with respect to the program, including but not limited
- * to infringement of third party rights, the risks and costs of
- * program errors, damage to or loss of data, programs or equipment,
- * and unavailability or interruption of operations. Under no
- * circumstances will the contributor of this Program be liable for
- * any damages of any kind arising from your use or distribution of
- * this program.
- *
- * The Linux Foundation chooses to take subject only to the GPLv2
- * license terms, and distributes only under these terms.
*/
#include <linux/async.h>
@@ -48,6 +20,7 @@
#include "unipro.h"
#include "ufs-sysfs.h"
#include "ufs_bsg.h"
+#include "ufshcd-crypto.h"
#include <asm/unaligned.h>
#include <linux/blkdev.h>
@@ -216,23 +189,22 @@
static struct ufs_dev_fix ufs_fixups[] = {
/* UFS cards deviations table */
- UFS_FIX(UFS_VENDOR_SAMSUNG, UFS_ANY_MODEL,
+ UFS_FIX(UFS_VENDOR_MICRON, UFS_ANY_MODEL,
UFS_DEVICE_QUIRK_DELAY_BEFORE_LPM),
UFS_FIX(UFS_VENDOR_SAMSUNG, UFS_ANY_MODEL,
+ UFS_DEVICE_QUIRK_DELAY_BEFORE_LPM |
+ UFS_DEVICE_QUIRK_HOST_PA_TACTIVATE |
UFS_DEVICE_QUIRK_RECOVERY_FROM_DL_NAC_ERRORS),
- UFS_FIX(UFS_VENDOR_SAMSUNG, UFS_ANY_MODEL,
- UFS_DEVICE_QUIRK_HOST_PA_TACTIVATE),
+ UFS_FIX(UFS_VENDOR_SKHYNIX, UFS_ANY_MODEL,
+ UFS_DEVICE_QUIRK_HOST_PA_SAVECONFIGTIME),
+ UFS_FIX(UFS_VENDOR_SKHYNIX, "hB8aL1" /*H28U62301AMR*/,
+ UFS_DEVICE_QUIRK_HOST_VS_DEBUGSAVECONFIGTIME),
UFS_FIX(UFS_VENDOR_TOSHIBA, UFS_ANY_MODEL,
UFS_DEVICE_QUIRK_DELAY_BEFORE_LPM),
UFS_FIX(UFS_VENDOR_TOSHIBA, "THGLF2G9C8KBADG",
UFS_DEVICE_QUIRK_PA_TACTIVATE),
UFS_FIX(UFS_VENDOR_TOSHIBA, "THGLF2G9D8KBADG",
UFS_DEVICE_QUIRK_PA_TACTIVATE),
- UFS_FIX(UFS_VENDOR_SKHYNIX, UFS_ANY_MODEL,
- UFS_DEVICE_QUIRK_HOST_PA_SAVECONFIGTIME),
- UFS_FIX(UFS_VENDOR_SKHYNIX, "hB8aL1" /*H28U62301AMR*/,
- UFS_DEVICE_QUIRK_HOST_VS_DEBUGSAVECONFIGTIME),
-
END_FIX
};
@@ -340,6 +312,26 @@
&descp->input_param1);
}
+static void ufshcd_add_uic_command_trace(struct ufs_hba *hba,
+ struct uic_command *ucmd,
+ const char *str)
+{
+ u32 cmd;
+
+ if (!trace_ufshcd_uic_command_enabled())
+ return;
+
+ if (!strcmp(str, "send"))
+ cmd = ucmd->command;
+ else
+ cmd = ufshcd_readl(hba, REG_UIC_COMMAND);
+
+ trace_ufshcd_uic_command(dev_name(hba->dev), str, cmd,
+ ufshcd_readl(hba, REG_UIC_COMMAND_ARG_1),
+ ufshcd_readl(hba, REG_UIC_COMMAND_ARG_2),
+ ufshcd_readl(hba, REG_UIC_COMMAND_ARG_3));
+}
+
static void ufshcd_add_command_trace(struct ufs_hba *hba,
unsigned int tag, const char *str)
{
@@ -672,7 +664,11 @@
*/
static inline void ufshcd_utrl_clear(struct ufs_hba *hba, u32 pos)
{
- ufshcd_writel(hba, ~(1 << pos), REG_UTP_TRANSFER_REQ_LIST_CLEAR);
+ if (hba->quirks & UFSHCI_QUIRK_BROKEN_REQ_LIST_CLR)
+ ufshcd_writel(hba, (1 << pos), REG_UTP_TRANSFER_REQ_LIST_CLEAR);
+ else
+ ufshcd_writel(hba, ~(1 << pos),
+ REG_UTP_TRANSFER_REQ_LIST_CLEAR);
}
/**
@@ -682,7 +678,10 @@
*/
static inline void ufshcd_utmrl_clear(struct ufs_hba *hba, u32 pos)
{
- ufshcd_writel(hba, ~(1 << pos), REG_UTP_TASK_REQ_LIST_CLEAR);
+ if (hba->quirks & UFSHCI_QUIRK_BROKEN_REQ_LIST_CLR)
+ ufshcd_writel(hba, (1 << pos), REG_UTP_TASK_REQ_LIST_CLEAR);
+ else
+ ufshcd_writel(hba, ~(1 << pos), REG_UTP_TASK_REQ_LIST_CLEAR);
}
/**
@@ -839,7 +838,12 @@
*/
static inline void ufshcd_hba_start(struct ufs_hba *hba)
{
- ufshcd_writel(hba, CONTROLLER_ENABLE, REG_CONTROLLER_ENABLE);
+ u32 val = CONTROLLER_ENABLE;
+
+ if (ufshcd_crypto_enable(hba))
+ val |= CRYPTO_GENERAL_ENABLE;
+
+ ufshcd_writel(hba, val, REG_CONTROLLER_ENABLE);
}
/**
@@ -1314,6 +1318,7 @@
unsigned long flags;
struct list_head *clk_list = &hba->clk_list_head;
struct ufs_clk_info *clki;
+ ktime_t curr_t;
if (!ufshcd_is_clkscaling_supported(hba))
return -EINVAL;
@@ -1321,6 +1326,7 @@
memset(stat, 0, sizeof(*stat));
spin_lock_irqsave(hba->host->host_lock, flags);
+ curr_t = ktime_get();
if (!scaling->window_start_t)
goto start_window;
@@ -1332,18 +1338,17 @@
*/
stat->current_frequency = clki->curr_freq;
if (scaling->is_busy_started)
- scaling->tot_busy_t += ktime_to_us(ktime_sub(ktime_get(),
- scaling->busy_start_t));
+ scaling->tot_busy_t += ktime_us_delta(curr_t,
+ scaling->busy_start_t);
- stat->total_time = jiffies_to_usecs((long)jiffies -
- (long)scaling->window_start_t);
+ stat->total_time = ktime_us_delta(curr_t, scaling->window_start_t);
stat->busy_time = scaling->tot_busy_t;
start_window:
- scaling->window_start_t = jiffies;
+ scaling->window_start_t = curr_t;
scaling->tot_busy_t = 0;
if (hba->outstanding_reqs) {
- scaling->busy_start_t = ktime_get();
+ scaling->busy_start_t = curr_t;
scaling->is_busy_started = true;
} else {
scaling->busy_start_t = 0;
@@ -1877,6 +1882,7 @@
static void ufshcd_clk_scaling_start_busy(struct ufs_hba *hba)
{
bool queue_resume_work = false;
+ ktime_t curr_t = ktime_get();
if (!ufshcd_is_clkscaling_supported(hba))
return;
@@ -1892,13 +1898,13 @@
&hba->clk_scaling.resume_work);
if (!hba->clk_scaling.window_start_t) {
- hba->clk_scaling.window_start_t = jiffies;
+ hba->clk_scaling.window_start_t = curr_t;
hba->clk_scaling.tot_busy_t = 0;
hba->clk_scaling.is_busy_started = false;
}
if (!hba->clk_scaling.is_busy_started) {
- hba->clk_scaling.busy_start_t = ktime_get();
+ hba->clk_scaling.busy_start_t = curr_t;
hba->clk_scaling.is_busy_started = true;
}
}
@@ -1925,8 +1931,11 @@
static inline
void ufshcd_send_command(struct ufs_hba *hba, unsigned int task_tag)
{
- hba->lrb[task_tag].issue_time_stamp = ktime_get();
- hba->lrb[task_tag].compl_time_stamp = ktime_set(0, 0);
+ struct ufshcd_lrb *lrbp = &hba->lrb[task_tag];
+
+ lrbp->issue_time_stamp = ktime_get();
+ lrbp->compl_time_stamp = ktime_set(0, 0);
+ ufshcd_vops_setup_xfer_req(hba, task_tag, (lrbp->cmd ? true : false));
ufshcd_add_command_trace(hba, task_tag, "send");
ufshcd_clk_scaling_start_busy(hba);
__set_bit(task_tag, &hba->outstanding_reqs);
@@ -1996,15 +2005,26 @@
/**
* ufshcd_hba_capabilities - Read controller capabilities
* @hba: per adapter instance
+ *
+ * Return: 0 on success, negative on error.
*/
-static inline void ufshcd_hba_capabilities(struct ufs_hba *hba)
+static inline int ufshcd_hba_capabilities(struct ufs_hba *hba)
{
+ int err;
+
hba->capabilities = ufshcd_readl(hba, REG_CONTROLLER_CAPABILITIES);
/* nutrs and nutmrs are 0 based values */
hba->nutrs = (hba->capabilities & MASK_TRANSFER_REQUESTS_SLOTS) + 1;
hba->nutmrs =
((hba->capabilities & MASK_TASK_MANAGEMENT_REQUEST_SLOTS) >> 16) + 1;
+
+ /* Read crypto capabilities */
+ err = ufshcd_hba_init_crypto_capabilities(hba);
+ if (err)
+ dev_err(hba->dev, "crypto setup failed\n");
+
+ return err;
}
/**
@@ -2052,6 +2072,8 @@
ufshcd_writel(hba, uic_cmd->argument2, REG_UIC_COMMAND_ARG_2);
ufshcd_writel(hba, uic_cmd->argument3, REG_UIC_COMMAND_ARG_3);
+ ufshcd_add_uic_command_trace(hba, uic_cmd, "send");
+
/* Write UIC Cmd */
ufshcd_writel(hba, uic_cmd->command & COMMAND_OPCODE_MASK,
REG_UIC_COMMAND);
@@ -2161,8 +2183,14 @@
return sg_segments;
if (sg_segments) {
- lrbp->utr_descriptor_ptr->prd_table_length =
- cpu_to_le16((u16)sg_segments);
+
+ if (hba->quirks & UFSHCD_QUIRK_PRDT_BYTE_GRAN)
+ lrbp->utr_descriptor_ptr->prd_table_length =
+ cpu_to_le16((sg_segments *
+ sizeof(struct ufshcd_sg_entry)));
+ else
+ lrbp->utr_descriptor_ptr->prd_table_length =
+ cpu_to_le16((u16) (sg_segments));
prd_table = (struct ufshcd_sg_entry *)lrbp->ucd_prdt_ptr;
@@ -2232,11 +2260,13 @@
* @cmd_dir: requests data direction
*/
static void ufshcd_prepare_req_desc_hdr(struct ufshcd_lrb *lrbp,
- u32 *upiu_flags, enum dma_data_direction cmd_dir)
+ u8 *upiu_flags, enum dma_data_direction cmd_dir)
{
struct utp_transfer_req_desc *req_desc = lrbp->utr_descriptor_ptr;
u32 data_direction;
u32 dword_0;
+ u32 dword_1 = 0;
+ u32 dword_3 = 0;
if (cmd_dir == DMA_FROM_DEVICE) {
data_direction = UTP_DEVICE_TO_HOST;
@@ -2254,10 +2284,12 @@
if (lrbp->intr_cmd)
dword_0 |= UTP_REQ_DESC_INT_CMD;
+ /* Prepare crypto related dwords */
+ ufshcd_prepare_req_desc_hdr_crypto(lrbp, &dword_0, &dword_1, &dword_3);
+
/* Transfer request descriptor header fields */
req_desc->header.dword_0 = cpu_to_le32(dword_0);
- /* dword_1 is reserved, hence it is set to 0 */
- req_desc->header.dword_1 = 0;
+ req_desc->header.dword_1 = cpu_to_le32(dword_1);
/*
* assigning invalid value for command status. Controller
* updates OCS on command completion, with the command
@@ -2265,8 +2297,7 @@
*/
req_desc->header.dword_2 =
cpu_to_le32(OCS_INVALID_COMMAND_STATUS);
- /* dword_3 is reserved, hence it is set to 0 */
- req_desc->header.dword_3 = 0;
+ req_desc->header.dword_3 = cpu_to_le32(dword_3);
req_desc->prd_table_length = 0;
}
@@ -2278,7 +2309,7 @@
* @upiu_flags: flags
*/
static
-void ufshcd_prepare_utp_scsi_cmd_upiu(struct ufshcd_lrb *lrbp, u32 upiu_flags)
+void ufshcd_prepare_utp_scsi_cmd_upiu(struct ufshcd_lrb *lrbp, u8 upiu_flags)
{
struct scsi_cmnd *cmd = lrbp->cmd;
struct utp_upiu_req *ucd_req_ptr = lrbp->ucd_req_ptr;
@@ -2311,7 +2342,7 @@
* @upiu_flags: flags
*/
static void ufshcd_prepare_utp_query_req_upiu(struct ufs_hba *hba,
- struct ufshcd_lrb *lrbp, u32 upiu_flags)
+ struct ufshcd_lrb *lrbp, u8 upiu_flags)
{
struct utp_upiu_req *ucd_req_ptr = lrbp->ucd_req_ptr;
struct ufs_query *query = &hba->dev_cmd.query;
@@ -2367,7 +2398,7 @@
*/
static int ufshcd_comp_devman_upiu(struct ufs_hba *hba, struct ufshcd_lrb *lrbp)
{
- u32 upiu_flags;
+ u8 upiu_flags;
int ret = 0;
if ((hba->ufs_version == UFSHCI_VERSION_10) ||
@@ -2395,7 +2426,7 @@
*/
static int ufshcd_comp_scsi_upiu(struct ufs_hba *hba, struct ufshcd_lrb *lrbp)
{
- u32 upiu_flags;
+ u8 upiu_flags;
int ret = 0;
if ((hba->ufs_version == UFSHCI_VERSION_10) ||
@@ -2521,6 +2552,9 @@
lrbp->task_tag = tag;
lrbp->lun = ufshcd_scsi_to_upiu_lun(cmd->device->lun);
lrbp->intr_cmd = !ufshcd_is_intr_aggr_allowed(hba) ? true : false;
+
+ ufshcd_prepare_lrbp_crypto(cmd->request, lrbp);
+
lrbp->req_abort_skip = false;
ufshcd_comp_scsi_upiu(hba, lrbp);
@@ -2536,7 +2570,6 @@
/* issue command to the controller */
spin_lock_irqsave(hba->host->host_lock, flags);
- ufshcd_vops_setup_xfer_req(hba, tag, true);
ufshcd_send_command(hba, tag);
out_unlock:
spin_unlock_irqrestore(hba->host->host_lock, flags);
@@ -2554,6 +2587,7 @@
lrbp->task_tag = tag;
lrbp->lun = 0; /* device management cmd is not specific to any LUN */
lrbp->intr_cmd = true; /* No interrupt aggregation */
+ ufshcd_prepare_lrbp_crypto(NULL, lrbp);
hba->dev_cmd.type = cmd_type;
return ufshcd_comp_devman_upiu(hba, lrbp);
@@ -2723,7 +2757,6 @@
/* Make sure descriptors are ready before ringing the doorbell */
wmb();
spin_lock_irqsave(hba->host->host_lock, flags);
- ufshcd_vops_setup_xfer_req(hba, tag, false);
ufshcd_send_command(hba, tag);
spin_unlock_irqrestore(hba->host->host_lock, flags);
@@ -3053,94 +3086,36 @@
}
/**
- * ufshcd_read_desc_length - read the specified descriptor length from header
- * @hba: Pointer to adapter instance
- * @desc_id: descriptor idn value
- * @desc_index: descriptor index
- * @desc_length: pointer to variable to read the length of descriptor
- *
- * Return 0 in case of success, non-zero otherwise
- */
-static int ufshcd_read_desc_length(struct ufs_hba *hba,
- enum desc_idn desc_id,
- int desc_index,
- int *desc_length)
-{
- int ret;
- u8 header[QUERY_DESC_HDR_SIZE];
- int header_len = QUERY_DESC_HDR_SIZE;
-
- if (desc_id >= QUERY_DESC_IDN_MAX)
- return -EINVAL;
-
- ret = ufshcd_query_descriptor_retry(hba, UPIU_QUERY_OPCODE_READ_DESC,
- desc_id, desc_index, 0, header,
- &header_len);
-
- if (ret) {
- dev_err(hba->dev, "%s: Failed to get descriptor header id %d",
- __func__, desc_id);
- return ret;
- } else if (desc_id != header[QUERY_DESC_DESC_TYPE_OFFSET]) {
- dev_warn(hba->dev, "%s: descriptor header id %d and desc_id %d mismatch",
- __func__, header[QUERY_DESC_DESC_TYPE_OFFSET],
- desc_id);
- ret = -EINVAL;
- }
-
- *desc_length = header[QUERY_DESC_LENGTH_OFFSET];
- return ret;
-
-}
-
-/**
* ufshcd_map_desc_id_to_length - map descriptor IDN to its length
* @hba: Pointer to adapter instance
* @desc_id: descriptor idn value
* @desc_len: mapped desc length (out)
- *
- * Return 0 in case of success, non-zero otherwise
*/
-int ufshcd_map_desc_id_to_length(struct ufs_hba *hba,
- enum desc_idn desc_id, int *desc_len)
+void ufshcd_map_desc_id_to_length(struct ufs_hba *hba, enum desc_idn desc_id,
+ int *desc_len)
{
- switch (desc_id) {
- case QUERY_DESC_IDN_DEVICE:
- *desc_len = hba->desc_size.dev_desc;
- break;
- case QUERY_DESC_IDN_POWER:
- *desc_len = hba->desc_size.pwr_desc;
- break;
- case QUERY_DESC_IDN_GEOMETRY:
- *desc_len = hba->desc_size.geom_desc;
- break;
- case QUERY_DESC_IDN_CONFIGURATION:
- *desc_len = hba->desc_size.conf_desc;
- break;
- case QUERY_DESC_IDN_UNIT:
- *desc_len = hba->desc_size.unit_desc;
- break;
- case QUERY_DESC_IDN_INTERCONNECT:
- *desc_len = hba->desc_size.interc_desc;
- break;
- case QUERY_DESC_IDN_STRING:
- *desc_len = QUERY_DESC_MAX_SIZE;
- break;
- case QUERY_DESC_IDN_HEALTH:
- *desc_len = hba->desc_size.hlth_desc;
- break;
- case QUERY_DESC_IDN_RFU_0:
- case QUERY_DESC_IDN_RFU_1:
+ if (desc_id >= QUERY_DESC_IDN_MAX || desc_id == QUERY_DESC_IDN_RFU_0 ||
+ desc_id == QUERY_DESC_IDN_RFU_1)
*desc_len = 0;
- break;
- default:
- *desc_len = 0;
- return -EINVAL;
- }
- return 0;
+ else
+ *desc_len = hba->desc_size[desc_id];
}
EXPORT_SYMBOL(ufshcd_map_desc_id_to_length);
+static void ufshcd_update_desc_length(struct ufs_hba *hba,
+ enum desc_idn desc_id, int desc_index,
+ unsigned char desc_len)
+{
+ if (hba->desc_size[desc_id] == QUERY_DESC_MAX_SIZE &&
+ desc_id != QUERY_DESC_IDN_STRING && desc_index != UFS_RPMB_UNIT)
+ /* For UFS 3.1, the normal unit descriptor is 10 bytes larger
+ * than the RPMB unit, however, both descriptors share the same
+ * desc_idn, to cover both unit descriptors with one length, we
+ * choose the normal unit descriptor length by desc_index.
+ */
+ hba->desc_size[desc_id] = desc_len;
+}
+
/**
* ufshcd_read_desc_param - read the specified descriptor parameter
* @hba: Pointer to adapter instance
@@ -3168,16 +3143,11 @@
if (desc_id >= QUERY_DESC_IDN_MAX || !param_size)
return -EINVAL;
- /* Get the max length of descriptor from structure filled up at probe
- * time.
- */
- ret = ufshcd_map_desc_id_to_length(hba, desc_id, &buff_len);
-
- /* Sanity checks */
- if (ret || !buff_len) {
- dev_err(hba->dev, "%s: Failed to get full descriptor length",
- __func__);
- return ret;
+ /* Get the length of descriptor */
+ ufshcd_map_desc_id_to_length(hba, desc_id, &buff_len);
+ if (!buff_len) {
+ dev_err(hba->dev, "%s: Failed to get desc length", __func__);
+ return -EINVAL;
}
/* Check whether we need temp memory */
@@ -3209,9 +3179,13 @@
goto out;
}
+ /* Update descriptor length */
+ buff_len = desc_buf[QUERY_DESC_LENGTH_OFFSET];
+ ufshcd_update_desc_length(hba, desc_id, desc_index, buff_len);
+
/* Check wherher we will not copy more data, than available */
- if (is_kmalloc && param_size > buff_len)
- param_size = buff_len;
+ if (is_kmalloc && (param_offset + param_size) > buff_len)
+ param_size = buff_len - param_offset;
if (is_kmalloc)
memcpy(param_read_buf, &desc_buf[param_offset], param_size);
@@ -3221,16 +3195,6 @@
return ret;
}
-static inline int ufshcd_read_desc(struct ufs_hba *hba,
- enum desc_idn desc_id,
- int desc_index,
- void *buf,
- u32 size)
-{
- return ufshcd_read_desc_param(hba, desc_id, desc_index, 0, buf, size);
-}
-
-
/**
* struct uc_string_id - unicode string
*
@@ -3278,9 +3242,8 @@
if (!uc_str)
return -ENOMEM;
- ret = ufshcd_read_desc(hba, QUERY_DESC_IDN_STRING,
- desc_index, uc_str,
- QUERY_DESC_MAX_SIZE);
+ ret = ufshcd_read_desc_param(hba, QUERY_DESC_IDN_STRING, desc_index, 0,
+ (u8 *)uc_str, QUERY_DESC_MAX_SIZE);
if (ret < 0) {
dev_err(hba->dev, "Reading String Desc failed after %d retries. err = %d\n",
QUERY_REQ_RETRIES, ret);
@@ -3511,11 +3474,21 @@
cpu_to_le32(upper_32_bits(cmd_desc_element_addr));
/* Response upiu and prdt offset should be in double words */
- utrdlp[i].response_upiu_offset =
- cpu_to_le16(response_offset >> 2);
- utrdlp[i].prd_table_offset = cpu_to_le16(prdt_offset >> 2);
- utrdlp[i].response_upiu_length =
- cpu_to_le16(ALIGNED_UPIU_SIZE >> 2);
+ if (hba->quirks & UFSHCD_QUIRK_PRDT_BYTE_GRAN) {
+ utrdlp[i].response_upiu_offset =
+ cpu_to_le16(response_offset);
+ utrdlp[i].prd_table_offset =
+ cpu_to_le16(prdt_offset);
+ utrdlp[i].response_upiu_length =
+ cpu_to_le16(ALIGNED_UPIU_SIZE);
+ } else {
+ utrdlp[i].response_upiu_offset =
+ cpu_to_le16(response_offset >> 2);
+ utrdlp[i].prd_table_offset =
+ cpu_to_le16(prdt_offset >> 2);
+ utrdlp[i].response_upiu_length =
+ cpu_to_le16(ALIGNED_UPIU_SIZE >> 2);
+ }
ufshcd_init_lrb(hba, &hba->lrb[i], i);
}
@@ -3545,6 +3518,52 @@
"dme-link-startup: error code %d\n", ret);
return ret;
}
+/**
+ * ufshcd_dme_reset - UIC command for DME_RESET
+ * @hba: per adapter instance
+ *
+ * DME_RESET command is issued in order to reset UniPro stack.
+ * This function now deals with cold reset.
+ *
+ * Returns 0 on success, non-zero value on failure
+ */
+static int ufshcd_dme_reset(struct ufs_hba *hba)
+{
+ struct uic_command uic_cmd = {0};
+ int ret;
+
+ uic_cmd.command = UIC_CMD_DME_RESET;
+
+ ret = ufshcd_send_uic_cmd(hba, &uic_cmd);
+ if (ret)
+ dev_err(hba->dev,
+ "dme-reset: error code %d\n", ret);
+
+ return ret;
+}
+
+/**
+ * ufshcd_dme_enable - UIC command for DME_ENABLE
+ * @hba: per adapter instance
+ *
+ * DME_ENABLE command is issued in order to enable UniPro stack.
+ *
+ * Returns 0 on success, non-zero value on failure
+ */
+static int ufshcd_dme_enable(struct ufs_hba *hba)
+{
+ struct uic_command uic_cmd = {0};
+ int ret;
+
+ uic_cmd.command = UIC_CMD_DME_ENABLE;
+
+ ret = ufshcd_send_uic_cmd(hba, &uic_cmd);
+ if (ret)
+ dev_err(hba->dev,
+ "dme-reset: error code %d\n", ret);
+
+ return ret;
+}
static inline void ufshcd_add_delay_before_dme_cmd(struct ufs_hba *hba)
{
@@ -4269,7 +4288,7 @@
}
/**
- * ufshcd_hba_enable - initialize the controller
+ * ufshcd_hba_execute_hce - initialize the controller
* @hba: per adapter instance
*
* The controller resets itself and controller firmware initialization
@@ -4278,7 +4297,7 @@
*
* Returns 0 on success, non-zero value on failure
*/
-int ufshcd_hba_enable(struct ufs_hba *hba)
+static int ufshcd_hba_execute_hce(struct ufs_hba *hba)
{
int retry;
@@ -4326,6 +4345,32 @@
return 0;
}
+
+int ufshcd_hba_enable(struct ufs_hba *hba)
+{
+ int ret;
+
+ if (hba->quirks & UFSHCI_QUIRK_BROKEN_HCE) {
+ ufshcd_set_link_off(hba);
+ ufshcd_vops_hce_enable_notify(hba, PRE_CHANGE);
+
+ /* enable UIC related interrupts */
+ ufshcd_enable_intr(hba, UFSHCD_UIC_MASK);
+ ret = ufshcd_dme_reset(hba);
+ if (!ret) {
+ ret = ufshcd_dme_enable(hba);
+ if (!ret)
+ ufshcd_vops_hce_enable_notify(hba, POST_CHANGE);
+ if (ret)
+ dev_err(hba->dev,
+ "Host controller enable failed with non-hce\n");
+ }
+ } else {
+ ret = ufshcd_hba_execute_hce(hba);
+ }
+
+ return ret;
+}
EXPORT_SYMBOL_GPL(ufshcd_hba_enable);
static int ufshcd_disable_tx_lcc(struct ufs_hba *hba, bool peer)
@@ -4650,6 +4695,8 @@
if (ufshcd_is_rpm_autosuspend_allowed(hba))
sdev->rpm_autosuspend = 1;
+ ufshcd_crypto_setup_rq_keyslot_manager(hba, q);
+
return 0;
}
@@ -4724,6 +4771,12 @@
/* overall command status of utrd */
ocs = ufshcd_get_tr_ocs(lrbp);
+ if (hba->quirks & UFSHCD_QUIRK_BROKEN_OCS_FATAL_ERROR) {
+ if (be32_to_cpu(lrbp->ucd_rsp_ptr->header.dword_1) &
+ MASK_RSP_UPIU_RESULT)
+ ocs = OCS_SUCCESS;
+ }
+
switch (ocs) {
case OCS_SUCCESS:
result = ufshcd_get_req_rsp(lrbp->ucd_rsp_ptr);
@@ -4792,6 +4845,9 @@
case OCS_MISMATCH_RESP_UPIU_SIZE:
case OCS_PEER_COMM_FAILURE:
case OCS_FATAL_ERROR:
+ case OCS_DEVICE_FATAL_ERROR:
+ case OCS_INVALID_CRYPTO_CONFIG:
+ case OCS_GENERAL_CRYPTO_ERROR:
default:
result |= DID_ERROR << 16;
dev_err(hba->dev,
@@ -4833,6 +4889,10 @@
complete(hba->uic_async_done);
retval = IRQ_HANDLED;
}
+
+ if (retval == IRQ_HANDLED)
+ ufshcd_add_uic_command_trace(hba, hba->active_uic_cmd,
+ "complete");
return retval;
}
@@ -4851,6 +4911,7 @@
for_each_set_bit(index, &completed_reqs, hba->nutrs) {
lrbp = &hba->lrb[index];
+ lrbp->compl_time_stamp = ktime_get();
cmd = lrbp->cmd;
if (cmd) {
ufshcd_add_command_trace(hba, index, "complete");
@@ -4859,13 +4920,11 @@
cmd->result = result;
/* Mark completed command as NULL in LRB */
lrbp->cmd = NULL;
- lrbp->compl_time_stamp = ktime_get();
/* Do not touch lrbp after scsi done */
cmd->scsi_done(cmd);
__ufshcd_release(hba);
} else if (lrbp->command_type == UTP_CMD_TYPE_DEV_MANAGE ||
lrbp->command_type == UTP_CMD_TYPE_UFS_STORAGE) {
- lrbp->compl_time_stamp = ktime_get();
if (hba->dev_cmd.complete) {
ufshcd_add_command_trace(hba, index,
"dev_complete");
@@ -4902,7 +4961,8 @@
* false interrupt if device completes another request after resetting
* aggregation and before reading the DB.
*/
- if (ufshcd_is_intr_aggr_allowed(hba))
+ if (ufshcd_is_intr_aggr_allowed(hba) &&
+ !(hba->quirks & UFSHCI_QUIRK_SKIP_RESET_INTR_AGGR))
ufshcd_reset_intr_aggr(hba);
tr_doorbell = ufshcd_readl(hba, REG_UTP_TRANSFER_REQ_DOOR_BELL);
@@ -6090,7 +6150,7 @@
int tag;
struct completion wait;
unsigned long flags;
- u32 upiu_flags;
+ u8 upiu_flags;
down_read(&hba->clk_scaling_lock);
@@ -6112,6 +6172,7 @@
lrbp->task_tag = tag;
lrbp->lun = 0;
lrbp->intr_cmd = true;
+ ufshcd_prepare_lrbp_crypto(NULL, lrbp);
hba->dev_cmd.type = cmd_type;
switch (hba->ufs_version) {
@@ -6703,7 +6764,7 @@
static void ufshcd_set_active_icc_lvl(struct ufs_hba *hba)
{
int ret;
- int buff_len = hba->desc_size.pwr_desc;
+ int buff_len = hba->desc_size[QUERY_DESC_IDN_POWER];
u8 *desc_buf;
u32 icc_level;
@@ -6711,8 +6772,8 @@
if (!desc_buf)
return;
- ret = ufshcd_read_desc(hba, QUERY_DESC_IDN_POWER, 0,
- desc_buf, buff_len);
+ ret = ufshcd_read_desc_param(hba, QUERY_DESC_IDN_POWER, 0, 0,
+ desc_buf, buff_len);
if (ret) {
dev_err(hba->dev,
"%s: Failed reading power descriptor.len = %d ret = %d",
@@ -6815,20 +6876,31 @@
static void ufshcd_wb_probe(struct ufs_hba *hba, u8 *desc_buf)
{
+ struct ufs_dev_info *dev_info = &hba->dev_info;
u8 lun;
u32 d_lu_wb_buf_alloc;
if (!ufshcd_is_wb_allowed(hba))
return;
-
- if (hba->desc_size.dev_desc < DEVICE_DESC_PARAM_EXT_UFS_FEATURE_SUP + 4)
+ /*
+ * Probe WB only for UFS-2.2 and UFS-3.1 (and later) devices or
+ * UFS devices with quirk UFS_DEVICE_QUIRK_SUPPORT_EXTENDED_FEATURES
+ * enabled
+ */
+ if (!(dev_info->wspecversion >= 0x310 ||
+ dev_info->wspecversion == 0x220 ||
+ (hba->dev_quirks & UFS_DEVICE_QUIRK_SUPPORT_EXTENDED_FEATURES)))
goto wb_disabled;
- hba->dev_info.d_ext_ufs_feature_sup =
+ if (hba->desc_size[QUERY_DESC_IDN_DEVICE] <
+ DEVICE_DESC_PARAM_EXT_UFS_FEATURE_SUP + 4)
+ goto wb_disabled;
+
+ dev_info->d_ext_ufs_feature_sup =
get_unaligned_be32(desc_buf +
DEVICE_DESC_PARAM_EXT_UFS_FEATURE_SUP);
- if (!(hba->dev_info.d_ext_ufs_feature_sup & UFS_DEV_WRITE_BOOSTER_SUP))
+ if (!(dev_info->d_ext_ufs_feature_sup & UFS_DEV_WRITE_BOOSTER_SUP))
goto wb_disabled;
/*
@@ -6837,17 +6909,17 @@
* a max of 1 lun would have wb buffer configured.
* Now only shared buffer mode is supported.
*/
- hba->dev_info.b_wb_buffer_type =
+ dev_info->b_wb_buffer_type =
desc_buf[DEVICE_DESC_PARAM_WB_TYPE];
- hba->dev_info.b_presrv_uspc_en =
+ dev_info->b_presrv_uspc_en =
desc_buf[DEVICE_DESC_PARAM_WB_PRESRV_USRSPC_EN];
- if (hba->dev_info.b_wb_buffer_type == WB_BUF_MODE_SHARED) {
- hba->dev_info.d_wb_alloc_units =
+ if (dev_info->b_wb_buffer_type == WB_BUF_MODE_SHARED) {
+ dev_info->d_wb_alloc_units =
get_unaligned_be32(desc_buf +
DEVICE_DESC_PARAM_WB_SHARED_ALLOC_UNITS);
- if (!hba->dev_info.d_wb_alloc_units)
+ if (!dev_info->d_wb_alloc_units)
goto wb_disabled;
} else {
for (lun = 0; lun < UFS_UPIU_MAX_WB_LUN_ID; lun++) {
@@ -6858,7 +6930,7 @@
(u8 *)&d_lu_wb_buf_alloc,
sizeof(d_lu_wb_buf_alloc));
if (d_lu_wb_buf_alloc) {
- hba->dev_info.wb_dedicated_lu = lun;
+ dev_info->wb_dedicated_lu = lun;
break;
}
}
@@ -6903,21 +6975,18 @@
static int ufs_get_device_desc(struct ufs_hba *hba)
{
int err;
- size_t buff_len;
u8 model_index;
u8 *desc_buf;
struct ufs_dev_info *dev_info = &hba->dev_info;
- buff_len = max_t(size_t, hba->desc_size.dev_desc,
- QUERY_DESC_MAX_SIZE + 1);
- desc_buf = kmalloc(buff_len, GFP_KERNEL);
+ desc_buf = kmalloc(QUERY_DESC_MAX_SIZE, GFP_KERNEL);
if (!desc_buf) {
err = -ENOMEM;
goto out;
}
- err = ufshcd_read_desc(hba, QUERY_DESC_IDN_DEVICE, 0, desc_buf,
- hba->desc_size.dev_desc);
+ err = ufshcd_read_desc_param(hba, QUERY_DESC_IDN_DEVICE, 0, 0, desc_buf,
+ hba->desc_size[QUERY_DESC_IDN_DEVICE]);
if (err) {
dev_err(hba->dev, "%s: Failed reading Device Desc. err = %d\n",
__func__, err);
@@ -6947,14 +7016,7 @@
ufs_fixup_device_setup(hba);
- /*
- * Probe WB only for UFS-3.1 devices or UFS devices with quirk
- * UFS_DEVICE_QUIRK_SUPPORT_EXTENDED_FEATURES enabled
- */
- if (dev_info->wspecversion >= 0x310 ||
- dev_info->wspecversion == 0x220 ||
- (hba->dev_quirks & UFS_DEVICE_QUIRK_SUPPORT_EXTENDED_FEATURES))
- ufshcd_wb_probe(hba, desc_buf);
+ ufshcd_wb_probe(hba, desc_buf);
/*
* ufshcd_read_string_desc returns size of the string
@@ -7146,61 +7208,21 @@
hba->req_abort_count = 0;
}
-static void ufshcd_init_desc_sizes(struct ufs_hba *hba)
-{
- int err;
-
- err = ufshcd_read_desc_length(hba, QUERY_DESC_IDN_DEVICE, 0,
- &hba->desc_size.dev_desc);
- if (err)
- hba->desc_size.dev_desc = QUERY_DESC_DEVICE_DEF_SIZE;
-
- err = ufshcd_read_desc_length(hba, QUERY_DESC_IDN_POWER, 0,
- &hba->desc_size.pwr_desc);
- if (err)
- hba->desc_size.pwr_desc = QUERY_DESC_POWER_DEF_SIZE;
-
- err = ufshcd_read_desc_length(hba, QUERY_DESC_IDN_INTERCONNECT, 0,
- &hba->desc_size.interc_desc);
- if (err)
- hba->desc_size.interc_desc = QUERY_DESC_INTERCONNECT_DEF_SIZE;
-
- err = ufshcd_read_desc_length(hba, QUERY_DESC_IDN_CONFIGURATION, 0,
- &hba->desc_size.conf_desc);
- if (err)
- hba->desc_size.conf_desc = QUERY_DESC_CONFIGURATION_DEF_SIZE;
-
- err = ufshcd_read_desc_length(hba, QUERY_DESC_IDN_UNIT, 0,
- &hba->desc_size.unit_desc);
- if (err)
- hba->desc_size.unit_desc = QUERY_DESC_UNIT_DEF_SIZE;
-
- err = ufshcd_read_desc_length(hba, QUERY_DESC_IDN_GEOMETRY, 0,
- &hba->desc_size.geom_desc);
- if (err)
- hba->desc_size.geom_desc = QUERY_DESC_GEOMETRY_DEF_SIZE;
-
- err = ufshcd_read_desc_length(hba, QUERY_DESC_IDN_HEALTH, 0,
- &hba->desc_size.hlth_desc);
- if (err)
- hba->desc_size.hlth_desc = QUERY_DESC_HEALTH_DEF_SIZE;
-}
-
static int ufshcd_device_geo_params_init(struct ufs_hba *hba)
{
int err;
size_t buff_len;
u8 *desc_buf;
- buff_len = hba->desc_size.geom_desc;
+ buff_len = hba->desc_size[QUERY_DESC_IDN_GEOMETRY];
desc_buf = kmalloc(buff_len, GFP_KERNEL);
if (!desc_buf) {
err = -ENOMEM;
goto out;
}
- err = ufshcd_read_desc(hba, QUERY_DESC_IDN_GEOMETRY, 0,
- desc_buf, buff_len);
+ err = ufshcd_read_desc_param(hba, QUERY_DESC_IDN_GEOMETRY, 0, 0,
+ desc_buf, buff_len);
if (err) {
dev_err(hba->dev, "%s: Failed reading Geometry Desc. err = %d\n",
__func__, err);
@@ -7288,10 +7310,11 @@
static int ufshcd_device_params_init(struct ufs_hba *hba)
{
bool flag;
- int ret;
+ int ret, i;
- /* Init check for device descriptor sizes */
- ufshcd_init_desc_sizes(hba);
+ /* Init device descriptor sizes */
+ for (i = 0; i < QUERY_DESC_IDN_MAX; i++)
+ hba->desc_size[i] = QUERY_DESC_MAX_SIZE;
/* Init UFS geometry descriptor related parameters */
ret = ufshcd_device_geo_params_init(hba);
@@ -8084,6 +8107,8 @@
static void ufshcd_vreg_set_lpm(struct ufs_hba *hba)
{
+ bool vcc_off = false;
+
/*
* It seems some UFS devices may keep drawing more than sleep current
* (atleast for 500us) from UFS rails (especially from VCCQ rail).
@@ -8112,13 +8137,22 @@
if (ufshcd_is_ufs_dev_poweroff(hba) && ufshcd_is_link_off(hba) &&
!hba->dev_info.is_lu_power_on_wp) {
ufshcd_setup_vreg(hba, false);
+ vcc_off = true;
} else if (!ufshcd_is_ufs_dev_active(hba)) {
ufshcd_toggle_vreg(hba->dev, hba->vreg_info.vcc, false);
+ vcc_off = true;
if (!ufshcd_is_link_active(hba)) {
ufshcd_config_vreg_lpm(hba, hba->vreg_info.vccq);
ufshcd_config_vreg_lpm(hba, hba->vreg_info.vccq2);
}
}
+
+ /*
+ * Some UFS devices require delay after VCC power rail is turned-off.
+ */
+ if (vcc_off && hba->vreg_info.vcc &&
+ hba->dev_quirks & UFS_DEVICE_QUIRK_DELAY_AFTER_LPM)
+ usleep_range(5000, 5100);
}
static int ufshcd_vreg_set_hpm(struct ufs_hba *hba)
@@ -8659,6 +8693,7 @@
*/
void ufshcd_dealloc_host(struct ufs_hba *hba)
{
+ ufshcd_crypto_destroy_keyslot_manager(hba);
scsi_host_put(hba->host);
}
EXPORT_SYMBOL_GPL(ufshcd_dealloc_host);
@@ -8759,7 +8794,9 @@
goto out_error;
/* Read capabilities registers */
- ufshcd_hba_capabilities(hba);
+ err = ufshcd_hba_capabilities(hba);
+ if (err)
+ goto out_disable;
/* Get UFS version supported by the controller */
hba->ufs_version = ufshcd_get_ufs_version(hba);
@@ -8869,6 +8906,8 @@
/* Reset the attached device */
ufshcd_vops_device_reset(hba);
+ ufshcd_init_crypto(hba);
+
/* Host controller enable */
err = ufshcd_hba_enable(hba);
if (err) {
diff --git a/drivers/scsi/ufs/ufshcd.h b/drivers/scsi/ufs/ufshcd.h
index bf97d61..b2ef18f 100644
--- a/drivers/scsi/ufs/ufshcd.h
+++ b/drivers/scsi/ufs/ufshcd.h
@@ -1,37 +1,12 @@
+/* SPDX-License-Identifier: GPL-2.0-or-later */
/*
* Universal Flash Storage Host controller driver
- *
- * This code is based on drivers/scsi/ufs/ufshcd.h
* Copyright (C) 2011-2013 Samsung India Software Operations
* Copyright (c) 2013-2016, The Linux Foundation. All rights reserved.
*
* Authors:
* Santosh Yaraganavi <santosh.sy@samsung.com>
* Vinayak Holikatti <h.vinayak@samsung.com>
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License
- * as published by the Free Software Foundation; either version 2
- * of the License, or (at your option) any later version.
- * See the COPYING file in the top-level directory or visit
- * <http://www.gnu.org/licenses/gpl-2.0.html>
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU General Public License for more details.
- *
- * This program is provided "AS IS" and "WITH ALL FAULTS" and
- * without warranty of any kind. You are solely responsible for
- * determining the appropriateness of using and distributing
- * the program and assume all risks associated with your exercise
- * of rights with respect to the program, including but not limited
- * to infringement of third party rights, the risks and costs of
- * program errors, damage to or loss of data, programs or equipment,
- * and unavailability or interruption of operations. Under no
- * circumstances will the contributor of this Program be liable for
- * any damages of any kind arising from your use or distribution of
- * this program.
*/
#ifndef _UFSHCD_H
@@ -57,6 +32,7 @@
#include <linux/regulator/consumer.h>
#include <linux/bitfield.h>
#include <linux/devfreq.h>
+#include <linux/keyslot-manager.h>
#include "unipro.h"
#include <asm/irq.h>
@@ -88,8 +64,6 @@
* @argument1: UIC command argument 1
* @argument2: UIC command argument 2
* @argument3: UIC command argument 3
- * @cmd_active: Indicate if UIC command is outstanding
- * @result: UIC command result
* @done: UIC command completion
*/
struct uic_command {
@@ -97,8 +71,6 @@
u32 argument1;
u32 argument2;
u32 argument3;
- int cmd_active;
- int result;
struct completion done;
};
@@ -183,6 +155,8 @@
* @intr_cmd: Interrupt command (doesn't participate in interrupt aggregation)
* @issue_time_stamp: time stamp for debug purposes
* @compl_time_stamp: time stamp for statistics
+ * @crypto_key_slot: the key slot to use for inline crypto (-1 if none)
+ * @data_unit_num: the data unit number for the first block for inline crypto
* @req_abort_skip: skip request abort task flag
*/
struct ufshcd_lrb {
@@ -207,6 +181,10 @@
bool intr_cmd;
ktime_t issue_time_stamp;
ktime_t compl_time_stamp;
+#ifdef CONFIG_SCSI_UFS_CRYPTO
+ int crypto_key_slot;
+ u64 data_unit_num;
+#endif
bool req_abort_skip;
};
@@ -236,16 +214,6 @@
struct ufs_query query;
};
-struct ufs_desc_size {
- int dev_desc;
- int pwr_desc;
- int geom_desc;
- int interc_desc;
- int unit_desc;
- int conf_desc;
- int hlth_desc;
-};
-
/**
* struct ufs_clk_info - UFS clock related info
* @list: list headed by hba->clk_list_head
@@ -313,6 +281,7 @@
* @dbg_register_dump: used to dump controller debug information
* @phy_initialization: used to initialize phys
* @device_reset: called to issue a reset pulse on the UFS device
+ * @program_key: program or evict an inline encryption key
*/
struct ufs_hba_variant_ops {
const char *name;
@@ -346,6 +315,8 @@
void (*config_scaling_param)(struct ufs_hba *hba,
struct devfreq_dev_profile *profile,
void *data);
+ int (*program_key)(struct ufs_hba *hba,
+ const union ufs_crypto_cfg_entry *cfg, int slot);
};
/* clock gating state */
@@ -411,7 +382,7 @@
struct ufs_clk_scaling {
int active_reqs;
unsigned long tot_busy_t;
- unsigned long window_start_t;
+ ktime_t window_start_t;
ktime_t busy_start_t;
struct device_attribute enable_attr;
struct ufs_saved_pwr_info saved_pwr_info;
@@ -520,6 +491,35 @@
* ops (get_ufs_hci_version) to get the correct version.
*/
UFSHCD_QUIRK_BROKEN_UFS_HCI_VERSION = 1 << 5,
+
+ /*
+ * Clear handling for transfer/task request list is just opposite.
+ */
+ UFSHCI_QUIRK_BROKEN_REQ_LIST_CLR = 1 << 6,
+
+ /*
+ * This quirk needs to be enabled if host controller doesn't allow
+ * that the interrupt aggregation timer and counter are reset by s/w.
+ */
+ UFSHCI_QUIRK_SKIP_RESET_INTR_AGGR = 1 << 7,
+
+ /*
+ * This quirks needs to be enabled if host controller cannot be
+ * enabled via HCE register.
+ */
+ UFSHCI_QUIRK_BROKEN_HCE = 1 << 8,
+
+ /*
+ * This quirk needs to be enabled if the host controller regards
+ * resolution of the values of PRDTO and PRDTL in UTRD as byte.
+ */
+ UFSHCD_QUIRK_PRDT_BYTE_GRAN = 1 << 9,
+
+ /*
+ * This quirk needs to be enabled if the host controller reports
+ * OCS FATAL ERROR with device error through sense data
+ */
+ UFSHCD_QUIRK_BROKEN_OCS_FATAL_ERROR = 1 << 10,
};
enum ufshcd_caps {
@@ -564,6 +564,12 @@
* provisioned to be used. This would increase the write performance.
*/
UFSHCD_CAP_WB_EN = 1 << 7,
+
+ /*
+ * This capability allows the host controller driver to use the
+ * inline crypto engine, if it is present
+ */
+ UFSHCD_CAP_CRYPTO = 1 << 8,
};
struct ufs_hba_variant_params {
@@ -624,6 +630,10 @@
* @is_urgent_bkops_lvl_checked: keeps track if the urgent bkops level for
* device is known or not.
* @scsi_block_reqs_cnt: reference counting for scsi block requests
+ * @crypto_capabilities: Content of crypto capabilities register (0x100)
+ * @crypto_cap_array: Array of crypto capabilities
+ * @crypto_cfg_register: Start of the crypto cfg array
+ * @ksm: the keyslot manager tied to this hba
*/
struct ufs_hba {
void __iomem *mmio_base;
@@ -738,7 +748,7 @@
bool is_urgent_bkops_lvl_checked;
struct rw_semaphore clk_scaling_lock;
- struct ufs_desc_size desc_size;
+ unsigned char desc_size[QUERY_DESC_IDN_MAX];
atomic_t scsi_block_reqs_cnt;
struct device bsg_dev;
@@ -746,6 +756,13 @@
bool wb_buf_flush_enabled;
bool wb_enabled;
struct delayed_work rpm_dev_flush_recheck_work;
+
+#ifdef CONFIG_SCSI_UFS_CRYPTO
+ union ufs_crypto_capabilities crypto_capabilities;
+ union ufs_crypto_cap_entry *crypto_cap_array;
+ u32 crypto_cfg_register;
+ struct blk_keyslot_manager ksm;
+#endif
};
/* Returns true if clocks can be gated. Otherwise false */
@@ -976,8 +993,8 @@
int ufshcd_hold(struct ufs_hba *hba, bool async);
void ufshcd_release(struct ufs_hba *hba);
-int ufshcd_map_desc_id_to_length(struct ufs_hba *hba, enum desc_idn desc_id,
- int *desc_length);
+void ufshcd_map_desc_id_to_length(struct ufs_hba *hba, enum desc_idn desc_id,
+ int *desc_length);
u32 ufshcd_get_local_unipro_ver(struct ufs_hba *hba);
diff --git a/drivers/scsi/ufs/ufshci.h b/drivers/scsi/ufs/ufshci.h
index c2961d3..ba31b09 100644
--- a/drivers/scsi/ufs/ufshci.h
+++ b/drivers/scsi/ufs/ufshci.h
@@ -1,36 +1,11 @@
+/* SPDX-License-Identifier: GPL-2.0-or-later */
/*
* Universal Flash Storage Host controller driver
- *
- * This code is based on drivers/scsi/ufs/ufshci.h
* Copyright (C) 2011-2013 Samsung India Software Operations
*
* Authors:
* Santosh Yaraganavi <santosh.sy@samsung.com>
* Vinayak Holikatti <h.vinayak@samsung.com>
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License
- * as published by the Free Software Foundation; either version 2
- * of the License, or (at your option) any later version.
- * See the COPYING file in the top-level directory or visit
- * <http://www.gnu.org/licenses/gpl-2.0.html>
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU General Public License for more details.
- *
- * This program is provided "AS IS" and "WITH ALL FAULTS" and
- * without warranty of any kind. You are solely responsible for
- * determining the appropriateness of using and distributing
- * the program and assume all risks associated with your exercise
- * of rights with respect to the program, including but not limited
- * to infringement of third party rights, the risks and costs of
- * program errors, damage to or loss of data, programs or equipment,
- * and unavailability or interruption of operations. Under no
- * circumstances will the contributor of this Program be liable for
- * any damages of any kind arising from your use or distribution of
- * this program.
*/
#ifndef _UFSHCI_H
@@ -90,6 +65,7 @@
MASK_64_ADDRESSING_SUPPORT = 0x01000000,
MASK_OUT_OF_ORDER_DATA_DELIVERY_SUPPORT = 0x02000000,
MASK_UIC_DME_TEST_MODE_SUPPORT = 0x04000000,
+ MASK_CRYPTO_SUPPORT = 0x10000000,
};
#define UFS_MASK(mask, offset) ((mask) << (offset))
@@ -143,6 +119,7 @@
#define DEVICE_FATAL_ERROR 0x800
#define CONTROLLER_FATAL_ERROR 0x10000
#define SYSTEM_BUS_FATAL_ERROR 0x20000
+#define CRYPTO_ENGINE_FATAL_ERROR 0x40000
#define UFSHCD_UIC_HIBERN8_MASK (UIC_HIBERNATE_ENTER |\
UIC_HIBERNATE_EXIT)
@@ -155,11 +132,13 @@
#define UFSHCD_ERROR_MASK (UIC_ERROR |\
DEVICE_FATAL_ERROR |\
CONTROLLER_FATAL_ERROR |\
- SYSTEM_BUS_FATAL_ERROR)
+ SYSTEM_BUS_FATAL_ERROR |\
+ CRYPTO_ENGINE_FATAL_ERROR)
#define INT_FATAL_ERRORS (DEVICE_FATAL_ERROR |\
CONTROLLER_FATAL_ERROR |\
- SYSTEM_BUS_FATAL_ERROR)
+ SYSTEM_BUS_FATAL_ERROR |\
+ CRYPTO_ENGINE_FATAL_ERROR)
/* HCS - Host Controller Status 30h */
#define DEVICE_PRESENT 0x1
@@ -318,6 +297,61 @@
INTERRUPT_MASK_ALL_VER_21 = 0x71FFF,
};
+/* CCAP - Crypto Capability 100h */
+union ufs_crypto_capabilities {
+ __le32 reg_val;
+ struct {
+ u8 num_crypto_cap;
+ u8 config_count;
+ u8 reserved;
+ u8 config_array_ptr;
+ };
+};
+
+enum ufs_crypto_key_size {
+ UFS_CRYPTO_KEY_SIZE_INVALID = 0x0,
+ UFS_CRYPTO_KEY_SIZE_128 = 0x1,
+ UFS_CRYPTO_KEY_SIZE_192 = 0x2,
+ UFS_CRYPTO_KEY_SIZE_256 = 0x3,
+ UFS_CRYPTO_KEY_SIZE_512 = 0x4,
+};
+
+enum ufs_crypto_alg {
+ UFS_CRYPTO_ALG_AES_XTS = 0x0,
+ UFS_CRYPTO_ALG_BITLOCKER_AES_CBC = 0x1,
+ UFS_CRYPTO_ALG_AES_ECB = 0x2,
+ UFS_CRYPTO_ALG_ESSIV_AES_CBC = 0x3,
+};
+
+/* x-CRYPTOCAP - Crypto Capability X */
+union ufs_crypto_cap_entry {
+ __le32 reg_val;
+ struct {
+ u8 algorithm_id;
+ u8 sdus_mask; /* Supported data unit size mask */
+ u8 key_size;
+ u8 reserved;
+ };
+};
+
+#define UFS_CRYPTO_CONFIGURATION_ENABLE (1 << 7)
+#define UFS_CRYPTO_KEY_MAX_SIZE 64
+/* x-CRYPTOCFG - Crypto Configuration X */
+union ufs_crypto_cfg_entry {
+ __le32 reg_val[32];
+ struct {
+ u8 crypto_key[UFS_CRYPTO_KEY_MAX_SIZE];
+ u8 data_unit_size;
+ u8 crypto_cap_idx;
+ u8 reserved_1;
+ u8 config_enable;
+ u8 reserved_multi_host;
+ u8 reserved_2;
+ u8 vsb[2];
+ u8 reserved_3[56];
+ };
+};
+
/*
* Request Descriptor Definitions
*/
@@ -339,6 +373,7 @@
UTP_NATIVE_UFS_COMMAND = 0x10000000,
UTP_DEVICE_MANAGEMENT_FUNCTION = 0x20000000,
UTP_REQ_DESC_INT_CMD = 0x01000000,
+ UTP_REQ_DESC_CRYPTO_ENABLE_CMD = 0x00800000,
};
/* UTP Transfer Request Data Direction (DD) */
@@ -358,6 +393,9 @@
OCS_PEER_COMM_FAILURE = 0x5,
OCS_ABORTED = 0x6,
OCS_FATAL_ERROR = 0x7,
+ OCS_DEVICE_FATAL_ERROR = 0x8,
+ OCS_INVALID_CRYPTO_CONFIG = 0x9,
+ OCS_GENERAL_CRYPTO_ERROR = 0xA,
OCS_INVALID_COMMAND_STATUS = 0x0F,
MASK_OCS = 0x0F,
};
diff --git a/drivers/scsi/ufs/unipro.h b/drivers/scsi/ufs/unipro.h
index 766d551..4ee6478 100644
--- a/drivers/scsi/ufs/unipro.h
+++ b/drivers/scsi/ufs/unipro.h
@@ -64,8 +64,25 @@
#define CFGRXOVR4 0x00E9
#define RXSQCTRL 0x00B5
#define CFGRXOVR6 0x00BF
+#define RX_HS_G1_SYNC_LENGTH_CAP 0x008B
+#define RX_HS_G1_PREP_LENGTH_CAP 0x008C
+#define RX_HS_G2_SYNC_LENGTH_CAP 0x0094
+#define RX_HS_G3_SYNC_LENGTH_CAP 0x0095
+#define RX_HS_G2_PREP_LENGTH_CAP 0x0096
+#define RX_HS_G3_PREP_LENGTH_CAP 0x0097
+#define RX_ADV_GRANULARITY_CAP 0x0098
+#define RX_MIN_ACTIVATETIME_CAP 0x008F
+#define RX_HIBERN8TIME_CAP 0x0092
+#define RX_ADV_HIBERN8TIME_CAP 0x0099
+#define RX_ADV_MIN_ACTIVATETIME_CAP 0x009A
+
#define is_mphy_tx_attr(attr) (attr < RX_MODE)
+#define RX_ADV_FINE_GRAN_STEP(x) ((((x) & 0x3) << 1) | 0x1)
+#define SYNC_LEN_FINE(x) ((x) & 0x3F)
+#define SYNC_LEN_COARSE(x) ((1 << 6) | ((x) & 0x3F))
+#define PREP_LEN(x) ((x) & 0xF)
+
#define RX_MIN_ACTIVATETIME_UNIT_US 100
#define HIBERN8TIME_UNIT_US 100
@@ -124,6 +141,7 @@
#define PA_PACPREQEOBTIMEOUT 0x1591
#define PA_HIBERN8TIME 0x15A7
#define PA_LOCALVERINFO 0x15A9
+#define PA_GRANULARITY 0x15AA
#define PA_TACTIVATE 0x15A8
#define PA_PACPFRAMECOUNT 0x15C0
#define PA_PACPERRORCOUNT 0x15C1
@@ -291,4 +309,19 @@
TRUE,
};
+/* CPort setting */
+#define E2EFC_ON (1 << 0)
+#define E2EFC_OFF (0 << 0)
+#define CSD_N_ON (0 << 1)
+#define CSD_N_OFF (1 << 1)
+#define CSV_N_ON (0 << 2)
+#define CSV_N_OFF (1 << 2)
+#define CPORT_DEF_FLAGS (CSV_N_OFF | CSD_N_OFF | E2EFC_OFF)
+
+/* CPort connection state */
+enum {
+ CPORT_IDLE = 0,
+ CPORT_CONNECTED,
+};
+
#endif /* _UNIPRO_H_ */
diff --git a/drivers/scsi/virtio_scsi.c b/drivers/scsi/virtio_scsi.c
index 0e0910c5..8cc003a 100644
--- a/drivers/scsi/virtio_scsi.c
+++ b/drivers/scsi/virtio_scsi.c
@@ -100,7 +100,7 @@
scsi_set_resid(sc, resid);
}
-/**
+/*
* virtscsi_complete_cmd - finish a scsi_cmd and invoke scsi_done
*
* Called with vq_lock held.
@@ -350,6 +350,14 @@
if (result == 0 && inq_result[0] >> 5) {
/* PQ indicates the LUN is not attached */
scsi_remove_device(sdev);
+ } else if (host_byte(result) == DID_BAD_TARGET) {
+ /*
+ * If all LUNs of a virtio-scsi device are unplugged
+ * it will respond with BAD TARGET on any INQUIRY
+ * command.
+ * Remove the device in this case as well.
+ */
+ scsi_remove_device(sdev);
}
}
@@ -1002,14 +1010,10 @@
return 0;
error:
- if (virtscsi_cmd_pool) {
- mempool_destroy(virtscsi_cmd_pool);
- virtscsi_cmd_pool = NULL;
- }
- if (virtscsi_cmd_cache) {
- kmem_cache_destroy(virtscsi_cmd_cache);
- virtscsi_cmd_cache = NULL;
- }
+ mempool_destroy(virtscsi_cmd_pool);
+ virtscsi_cmd_pool = NULL;
+ kmem_cache_destroy(virtscsi_cmd_cache);
+ virtscsi_cmd_cache = NULL;
return ret;
}
diff --git a/drivers/target/iscsi/cxgbit/cxgbit.h b/drivers/target/iscsi/cxgbit/cxgbit.h
index c04cd08..4069033 100644
--- a/drivers/target/iscsi/cxgbit/cxgbit.h
+++ b/drivers/target/iscsi/cxgbit/cxgbit.h
@@ -207,7 +207,6 @@
/* socket lock */
spinlock_t lock;
wait_queue_head_t waitq;
- wait_queue_head_t ack_waitq;
bool lock_owner;
struct kref kref;
u32 max_iso_npdu;
diff --git a/drivers/target/iscsi/cxgbit/cxgbit_cm.c b/drivers/target/iscsi/cxgbit/cxgbit_cm.c
index a2b5c79..518ded2 100644
--- a/drivers/target/iscsi/cxgbit/cxgbit_cm.c
+++ b/drivers/target/iscsi/cxgbit/cxgbit_cm.c
@@ -1360,7 +1360,6 @@
cxgbit_sock_reset_wr_list(csk);
spin_lock_init(&csk->lock);
init_waitqueue_head(&csk->waitq);
- init_waitqueue_head(&csk->ack_waitq);
csk->lock_owner = false;
if (cxgbit_alloc_csk_skb(csk)) {
@@ -1485,6 +1484,26 @@
return flowclen16;
}
+static int
+cxgbit_send_tcb_skb(struct cxgbit_sock *csk, struct sk_buff *skb)
+{
+ spin_lock_bh(&csk->lock);
+ if (unlikely(csk->com.state != CSK_STATE_ESTABLISHED)) {
+ spin_unlock_bh(&csk->lock);
+ pr_err("%s: csk 0x%p, tid %u, state %u\n",
+ __func__, csk, csk->tid, csk->com.state);
+ __kfree_skb(skb);
+ return -1;
+ }
+
+ cxgbit_get_csk(csk);
+ cxgbit_init_wr_wait(&csk->com.wr_wait);
+ cxgbit_ofld_send(csk->com.cdev, skb);
+ spin_unlock_bh(&csk->lock);
+
+ return 0;
+}
+
int cxgbit_setup_conn_digest(struct cxgbit_sock *csk)
{
struct sk_buff *skb;
@@ -1510,10 +1529,8 @@
(dcrc ? ULP_CRC_DATA : 0)) << 4);
set_wr_txq(skb, CPL_PRIORITY_CONTROL, csk->ctrlq_idx);
- cxgbit_get_csk(csk);
- cxgbit_init_wr_wait(&csk->com.wr_wait);
-
- cxgbit_ofld_send(csk->com.cdev, skb);
+ if (cxgbit_send_tcb_skb(csk, skb))
+ return -1;
ret = cxgbit_wait_for_reply(csk->com.cdev,
&csk->com.wr_wait,
@@ -1545,10 +1562,8 @@
req->val = cpu_to_be64(pg_idx << 8);
set_wr_txq(skb, CPL_PRIORITY_CONTROL, csk->ctrlq_idx);
- cxgbit_get_csk(csk);
- cxgbit_init_wr_wait(&csk->com.wr_wait);
-
- cxgbit_ofld_send(csk->com.cdev, skb);
+ if (cxgbit_send_tcb_skb(csk, skb))
+ return -1;
ret = cxgbit_wait_for_reply(csk->com.cdev,
&csk->com.wr_wait,
@@ -1871,7 +1886,6 @@
if (csk->snd_una != snd_una) {
csk->snd_una = snd_una;
dst_confirm(csk->dst);
- wake_up(&csk->ack_waitq);
}
}
diff --git a/drivers/target/iscsi/cxgbit/cxgbit_target.c b/drivers/target/iscsi/cxgbit/cxgbit_target.c
index fcdc421..9b3eb2e 100644
--- a/drivers/target/iscsi/cxgbit/cxgbit_target.c
+++ b/drivers/target/iscsi/cxgbit/cxgbit_target.c
@@ -284,18 +284,6 @@
}
}
-static bool cxgbit_lock_sock(struct cxgbit_sock *csk)
-{
- spin_lock_bh(&csk->lock);
-
- if (before(csk->write_seq, csk->snd_una + csk->snd_win))
- csk->lock_owner = true;
-
- spin_unlock_bh(&csk->lock);
-
- return csk->lock_owner;
-}
-
static void cxgbit_unlock_sock(struct cxgbit_sock *csk)
{
struct sk_buff_head backlogq;
@@ -325,20 +313,16 @@
{
int ret = 0;
- wait_event_interruptible(csk->ack_waitq, cxgbit_lock_sock(csk));
+ spin_lock_bh(&csk->lock);
+ csk->lock_owner = true;
+ spin_unlock_bh(&csk->lock);
if (unlikely((csk->com.state != CSK_STATE_ESTABLISHED) ||
signal_pending(current))) {
__kfree_skb(skb);
__skb_queue_purge(&csk->ppodq);
ret = -1;
- spin_lock_bh(&csk->lock);
- if (csk->lock_owner) {
- spin_unlock_bh(&csk->lock);
- goto unlock;
- }
- spin_unlock_bh(&csk->lock);
- return ret;
+ goto unlock;
}
csk->write_seq += skb->len +
diff --git a/drivers/target/iscsi/iscsi_target_nego.c b/drivers/target/iscsi/iscsi_target_nego.c
index 685d771..f88a52f 100644
--- a/drivers/target/iscsi/iscsi_target_nego.c
+++ b/drivers/target/iscsi/iscsi_target_nego.c
@@ -481,7 +481,7 @@
{
if (sk->sk_state == TCP_CLOSE_WAIT || sk->sk_state == TCP_CLOSE) {
pr_debug("__iscsi_target_sk_check_close: TCP_CLOSE_WAIT|TCP_CLOSE,"
- "returning FALSE\n");
+ "returning TRUE\n");
return true;
}
return false;
@@ -625,13 +625,37 @@
pr_debug("iscsi_target_do_login_rx after rx_login_io, %p, %s:%d\n",
conn, current->comm, current->pid);
+ /*
+ * LOGIN_FLAGS_READ_ACTIVE is cleared so that sk_data_ready
+ * could be triggered again after this.
+ *
+ * LOGIN_FLAGS_WRITE_ACTIVE is cleared after we successfully
+ * process a login PDU, so that sk_state_chage can do login
+ * cleanup as needed if the socket is closed. If a delayed work is
+ * ongoing (LOGIN_FLAGS_WRITE_ACTIVE or LOGIN_FLAGS_READ_ACTIVE),
+ * sk_state_change will leave the cleanup to the delayed work or
+ * it will schedule a delayed work to do cleanup.
+ */
+ if (conn->sock) {
+ struct sock *sk = conn->sock->sk;
+
+ write_lock_bh(&sk->sk_callback_lock);
+ if (!test_bit(LOGIN_FLAGS_INITIAL_PDU, &conn->login_flags)) {
+ clear_bit(LOGIN_FLAGS_READ_ACTIVE, &conn->login_flags);
+ set_bit(LOGIN_FLAGS_WRITE_ACTIVE, &conn->login_flags);
+ }
+ write_unlock_bh(&sk->sk_callback_lock);
+ }
+
rc = iscsi_target_do_login(conn, login);
if (rc < 0) {
goto err;
} else if (!rc) {
- if (iscsi_target_sk_check_and_clear(conn, LOGIN_FLAGS_READ_ACTIVE))
+ if (iscsi_target_sk_check_and_clear(conn,
+ LOGIN_FLAGS_WRITE_ACTIVE))
goto err;
} else if (rc == 1) {
+ cancel_delayed_work(&conn->login_work);
iscsi_target_nego_release(conn);
iscsi_post_login_handler(np, conn, zero_tsih);
iscsit_deaccess_np(np, tpg, tpg_np);
@@ -640,6 +664,7 @@
err:
iscsi_target_restore_sock_callbacks(conn);
+ cancel_delayed_work(&conn->login_work);
iscsi_target_login_drop(conn, login);
iscsit_deaccess_np(np, tpg, tpg_np);
}
@@ -670,9 +695,10 @@
state = __iscsi_target_sk_check_close(sk);
pr_debug("__iscsi_target_sk_close_change: state: %d\n", state);
- if (test_bit(LOGIN_FLAGS_READ_ACTIVE, &conn->login_flags)) {
- pr_debug("Got LOGIN_FLAGS_READ_ACTIVE=1 sk_state_change"
- " conn: %p\n", conn);
+ if (test_bit(LOGIN_FLAGS_READ_ACTIVE, &conn->login_flags) ||
+ test_bit(LOGIN_FLAGS_WRITE_ACTIVE, &conn->login_flags)) {
+ pr_debug("Got LOGIN_FLAGS_{READ|WRITE}_ACTIVE=1"
+ " sk_state_change conn: %p\n", conn);
if (state)
set_bit(LOGIN_FLAGS_CLOSED, &conn->login_flags);
write_unlock_bh(&sk->sk_callback_lock);
diff --git a/drivers/target/target_core_fabric_lib.c b/drivers/target/target_core_fabric_lib.c
index 1e031d8..6600ae4 100644
--- a/drivers/target/target_core_fabric_lib.c
+++ b/drivers/target/target_core_fabric_lib.c
@@ -132,33 +132,27 @@
unsigned char *buf)
{
u32 off = 4, padding = 0;
+ int isid_len;
u16 len = 0;
spin_lock_irq(&se_nacl->nacl_sess_lock);
/*
- * From spc4r17 Section 7.5.4.6: TransportID for initiator
- * ports using SCSI over iSCSI.
+ * Only null terminate the last field.
*
- * The null-terminated, null-padded (see 4.4.2) ISCSI NAME field
- * shall contain the iSCSI name of an iSCSI initiator node (see
- * RFC 3720). The first ISCSI NAME field byte containing an ASCII
- * null character terminates the ISCSI NAME field without regard for
- * the specified length of the iSCSI TransportID or the contents of
- * the ADDITIONAL LENGTH field.
+ * From spc4r37 section 7.6.4.6: TransportID for initiator ports using
+ * SCSI over iSCSI.
+ *
+ * Table 507 TPID=0 Initiator device TransportID
+ *
+ * The null-terminated, null-padded (see 4.3.2) ISCSI NAME field shall
+ * contain the iSCSI name of an iSCSI initiator node (see RFC 7143).
+ * The first ISCSI NAME field byte containing an ASCII null character
+ * terminates the ISCSI NAME field without regard for the specified
+ * length of the iSCSI TransportID or the contents of the ADDITIONAL
+ * LENGTH field.
*/
len = sprintf(&buf[off], "%s", se_nacl->initiatorname);
- /*
- * Add Extra byte for NULL terminator
- */
- len++;
- /*
- * If there is ISID present with the registration and *format code == 1
- * 1, use iSCSI Initiator port TransportID format.
- *
- * Otherwise use iSCSI Initiator device TransportID format that
- * does not contain the ASCII encoded iSCSI Initiator iSID value
- * provied by the iSCSi Initiator during the iSCSI login process.
- */
+ off += len;
if ((*format_code == 1) && (pr_reg->isid_present_at_reg)) {
/*
* Set FORMAT CODE 01b for iSCSI Initiator port TransportID
@@ -166,8 +160,12 @@
*/
buf[0] |= 0x40;
/*
- * From spc4r17 Section 7.5.4.6: TransportID for initiator
- * ports using SCSI over iSCSI. Table 390
+ * From spc4r37 Section 7.6.4.6
+ *
+ * Table 508 TPID=1 Initiator port TransportID.
+ *
+ * The ISCSI NAME field shall not be null-terminated
+ * (see 4.3.2) and shall not be padded.
*
* The SEPARATOR field shall contain the five ASCII
* characters ",i,0x".
@@ -177,23 +175,24 @@
* (see RFC 3720) in the form of ASCII characters that are the
* hexadecimal digits converted from the binary iSCSI initiator
* session identifier value. The first ISCSI INITIATOR SESSION
- * ID field byte containing an ASCII null character
+ * ID field byte containing an ASCII null character terminates
+ * the ISCSI INITIATOR SESSION ID field without regard for the
+ * specified length of the iSCSI TransportID or the contents
+ * of the ADDITIONAL LENGTH field.
*/
- buf[off+len] = 0x2c; off++; /* ASCII Character: "," */
- buf[off+len] = 0x69; off++; /* ASCII Character: "i" */
- buf[off+len] = 0x2c; off++; /* ASCII Character: "," */
- buf[off+len] = 0x30; off++; /* ASCII Character: "0" */
- buf[off+len] = 0x78; off++; /* ASCII Character: "x" */
+ buf[off++] = 0x2c; /* ASCII Character: "," */
+ buf[off++] = 0x69; /* ASCII Character: "i" */
+ buf[off++] = 0x2c; /* ASCII Character: "," */
+ buf[off++] = 0x30; /* ASCII Character: "0" */
+ buf[off++] = 0x78; /* ASCII Character: "x" */
len += 5;
- buf[off+len] = pr_reg->pr_reg_isid[0]; off++;
- buf[off+len] = pr_reg->pr_reg_isid[1]; off++;
- buf[off+len] = pr_reg->pr_reg_isid[2]; off++;
- buf[off+len] = pr_reg->pr_reg_isid[3]; off++;
- buf[off+len] = pr_reg->pr_reg_isid[4]; off++;
- buf[off+len] = pr_reg->pr_reg_isid[5]; off++;
- buf[off+len] = '\0'; off++;
- len += 7;
+
+ isid_len = sprintf(buf + off, "%s", pr_reg->pr_reg_isid);
+ off += isid_len;
+ len += isid_len;
}
+ buf[off] = '\0';
+ len += 1;
spin_unlock_irq(&se_nacl->nacl_sess_lock);
/*
* The ADDITIONAL LENGTH field specifies the number of bytes that follow
@@ -236,7 +235,7 @@
*/
if (pr_reg->isid_present_at_reg) {
len += 5; /* For ",i,0x" ASCII separator */
- len += 7; /* For iSCSI Initiator Session ID + Null terminator */
+ len += strlen(pr_reg->pr_reg_isid);
*format_code = 1;
} else
*format_code = 0;
@@ -265,9 +264,7 @@
char **port_nexus_ptr)
{
char *p;
- u32 tid_len, padding;
int i;
- u16 add_len;
u8 format_code = (buf[0] & 0xc0);
/*
* Check for FORMAT CODE 00b or 01b from spc4r17, section 7.5.4.6:
@@ -293,23 +290,11 @@
*/
if (out_tid_len) {
/* The shift works thanks to integer promotion rules */
- add_len = get_unaligned_be16(&buf[2]);
-
- tid_len = strlen(&buf[4]);
- tid_len += 4; /* Add four bytes for iSCSI Transport ID header */
- tid_len += 1; /* Add one byte for NULL terminator */
- padding = ((-tid_len) & 3);
- if (padding != 0)
- tid_len += padding;
-
- if ((add_len + 4) != tid_len) {
- pr_debug("LIO-Target Extracted add_len: %hu "
- "does not match calculated tid_len: %u,"
- " using tid_len instead\n", add_len+4, tid_len);
- *out_tid_len = tid_len;
- } else
- *out_tid_len = (add_len + 4);
+ *out_tid_len = get_unaligned_be16(&buf[2]);
+ /* Add four bytes for iSCSI Transport ID header */
+ *out_tid_len += 4;
}
+
/*
* Check for ',i,0x' separator between iSCSI Name and iSCSI Initiator
* Session ID as defined in Table 390 - iSCSI initiator port TransportID
@@ -334,6 +319,16 @@
* iscsi_target.c:lio_sess_get_initiator_sid()
*/
for (i = 0; i < 12; i++) {
+ /*
+ * The first ISCSI INITIATOR SESSION ID field byte
+ * containing an ASCII null character terminates the
+ * ISCSI INITIATOR SESSION ID field without regard for
+ * the specified length of the iSCSI TransportID or the
+ * contents of the ADDITIONAL LENGTH field.
+ */
+ if (*p == '\0')
+ break;
+
if (isdigit(*p)) {
p++;
continue;
diff --git a/drivers/target/target_core_internal.h b/drivers/target/target_core_internal.h
index 8533444..e7b3c6e 100644
--- a/drivers/target/target_core_internal.h
+++ b/drivers/target/target_core_internal.h
@@ -138,6 +138,7 @@
void release_se_kmem_caches(void);
u32 scsi_get_new_index(scsi_index_t);
void transport_subsystem_check_init(void);
+void transport_uninit_session(struct se_session *);
unsigned char *transport_dump_cmd_direction(struct se_cmd *);
void transport_dump_dev_state(struct se_device *, char *, int *);
void transport_dump_dev_info(struct se_device *, struct se_lun *,
diff --git a/drivers/target/target_core_pr.c b/drivers/target/target_core_pr.c
index 91e41cc..8fc8865 100644
--- a/drivers/target/target_core_pr.c
+++ b/drivers/target/target_core_pr.c
@@ -1159,7 +1159,6 @@
{
struct t10_reservation *pr_tmpl = &dev->t10_pr;
struct t10_pr_registration *pr_reg, *pr_reg_tmp;
- struct se_portal_group *tpg;
spin_lock(&pr_tmpl->registration_lock);
list_for_each_entry_safe(pr_reg, pr_reg_tmp,
@@ -1170,21 +1169,11 @@
if (pr_reg->pr_reg_nacl != nacl)
continue;
- tpg = pr_reg->pr_reg_nacl->se_tpg;
/*
* If this registration does NOT contain a fabric provided
* ISID, then we have found a match.
*/
if (!pr_reg->isid_present_at_reg) {
- /*
- * Determine if this SCSI device server requires that
- * SCSI Intiatior TransportID w/ ISIDs is enforced
- * for fabric modules (iSCSI) requiring them.
- */
- if (tpg->se_tpg_tfo->sess_get_initiator_sid != NULL) {
- if (dev->dev_attrib.enforce_pr_isids)
- continue;
- }
atomic_inc_mb(&pr_reg->pr_res_holders);
spin_unlock(&pr_tmpl->registration_lock);
return pr_reg;
@@ -1530,13 +1519,16 @@
kfree(tidh_new);
return TCM_INSUFFICIENT_REGISTRATION_RESOURCES;
}
+
+ if (core_scsi3_lunacl_depend_item(local_pr_reg->pr_reg_deve)) {
+ kfree(tidh_new);
+ kref_put(&local_pr_reg->pr_reg_deve->pr_kref,
+ target_pr_kref_release);
+ kmem_cache_free(t10_pr_reg_cache, local_pr_reg);
+ return TCM_INSUFFICIENT_REGISTRATION_RESOURCES;
+ }
+
tidh_new->dest_pr_reg = local_pr_reg;
- /*
- * The local I_T nexus does not hold any configfs dependances,
- * so we set tidh_new->dest_se_deve to NULL to prevent the
- * configfs_undepend_item() calls in the tid_dest_list loops below.
- */
- tidh_new->dest_se_deve = NULL;
list_add_tail(&tidh_new->dest_list, &tid_dest_list);
if (cmd->data_length < 28) {
@@ -1591,10 +1583,25 @@
continue;
dest_rtpi = tmp_lun->lun_rtpi;
+ iport_ptr = NULL;
i_str = target_parse_pr_out_transport_id(tmp_tpg,
ptr, &tid_len, &iport_ptr);
if (!i_str)
continue;
+ /*
+ * Determine if this SCSI device server requires that
+ * SCSI Intiatior TransportID w/ ISIDs is enforced
+ * for fabric modules (iSCSI) requiring them.
+ */
+ if (tpg->se_tpg_tfo->sess_get_initiator_sid &&
+ dev->dev_attrib.enforce_pr_isids &&
+ !iport_ptr) {
+ pr_warn("SPC-PR: enforce_pr_isids is set but a isid has not been sent in the SPEC_I_PT data for %s.",
+ i_str);
+ ret = TCM_INVALID_PARAMETER_LIST;
+ spin_unlock(&dev->se_port_lock);
+ goto out_unmap;
+ }
atomic_inc_mb(&tmp_tpg->tpg_pr_ref_count);
spin_unlock(&dev->se_port_lock);
@@ -1810,12 +1817,9 @@
dest_node_acl->initiatorname, i_buf, (dest_se_deve) ?
dest_se_deve->mapped_lun : 0);
- if (!dest_se_deve) {
- kref_put(&local_pr_reg->pr_reg_deve->pr_kref,
- target_pr_kref_release);
+ if (dest_pr_reg == local_pr_reg)
continue;
- }
- core_scsi3_lunacl_undepend_item(dest_se_deve);
+
core_scsi3_nodeacl_undepend_item(dest_node_acl);
core_scsi3_tpg_undepend_item(dest_tpg);
}
@@ -1829,11 +1833,16 @@
* including *dest_pr_reg and the configfs dependances..
*/
list_for_each_entry_safe(tidh, tidh_tmp, &tid_dest_list, dest_list) {
+ bool is_local = false;
+
dest_tpg = tidh->dest_tpg;
dest_node_acl = tidh->dest_node_acl;
dest_se_deve = tidh->dest_se_deve;
dest_pr_reg = tidh->dest_pr_reg;
+ if (dest_pr_reg == local_pr_reg)
+ is_local = true;
+
list_del(&tidh->dest_list);
kfree(tidh);
/*
@@ -1849,13 +1858,11 @@
}
kmem_cache_free(t10_pr_reg_cache, dest_pr_reg);
-
- if (!dest_se_deve) {
- kref_put(&local_pr_reg->pr_reg_deve->pr_kref,
- target_pr_kref_release);
- continue;
- }
core_scsi3_lunacl_undepend_item(dest_se_deve);
+
+ if (is_local)
+ continue;
+
core_scsi3_nodeacl_undepend_item(dest_node_acl);
core_scsi3_tpg_undepend_item(dest_tpg);
}
diff --git a/drivers/target/target_core_tmr.c b/drivers/target/target_core_tmr.c
index 89c84d4..e4513ef 100644
--- a/drivers/target/target_core_tmr.c
+++ b/drivers/target/target_core_tmr.c
@@ -116,14 +116,16 @@
struct se_tmr_req *tmr,
struct se_session *se_sess)
{
- struct se_cmd *se_cmd;
+ LIST_HEAD(aborted_list);
+ struct se_cmd *se_cmd, *next;
unsigned long flags;
+ bool rc;
u64 ref_tag;
- spin_lock_irqsave(&se_sess->sess_cmd_lock, flags);
- list_for_each_entry(se_cmd, &se_sess->sess_cmd_list, se_cmd_list) {
+ spin_lock_irqsave(&dev->execute_task_lock, flags);
+ list_for_each_entry_safe(se_cmd, next, &dev->state_list, state_list) {
- if (dev != se_cmd->se_dev)
+ if (se_sess != se_cmd->se_sess)
continue;
/* skip task management functions, including tmr->task_cmd */
@@ -137,11 +139,16 @@
printk("ABORT_TASK: Found referenced %s task_tag: %llu\n",
se_cmd->se_tfo->fabric_name, ref_tag);
- if (!__target_check_io_state(se_cmd, se_sess,
- dev->dev_attrib.emulate_tas))
+ spin_lock(&se_sess->sess_cmd_lock);
+ rc = __target_check_io_state(se_cmd, se_sess, 0);
+ spin_unlock(&se_sess->sess_cmd_lock);
+ if (!rc)
continue;
- spin_unlock_irqrestore(&se_sess->sess_cmd_lock, flags);
+ list_move_tail(&se_cmd->state_list, &aborted_list);
+ se_cmd->state_active = false;
+
+ spin_unlock_irqrestore(&dev->execute_task_lock, flags);
/*
* Ensure that this ABORT request is visible to the LU RESET
@@ -151,6 +158,11 @@
WARN_ON_ONCE(transport_lookup_tmr_lun(tmr->task_cmd) <
0);
+ if (dev->transport->tmr_notify)
+ dev->transport->tmr_notify(dev, TMR_ABORT_TASK,
+ &aborted_list);
+
+ list_del_init(&se_cmd->state_list);
target_put_cmd_and_wait(se_cmd);
printk("ABORT_TASK: Sending TMR_FUNCTION_COMPLETE for"
@@ -159,7 +171,10 @@
atomic_long_inc(&dev->aborts_complete);
return;
}
- spin_unlock_irqrestore(&se_sess->sess_cmd_lock, flags);
+ spin_unlock_irqrestore(&dev->execute_task_lock, flags);
+
+ if (dev->transport->tmr_notify)
+ dev->transport->tmr_notify(dev, TMR_ABORT_TASK, &aborted_list);
printk("ABORT_TASK: Sending TMR_TASK_DOES_NOT_EXIST for ref_tag: %lld\n",
tmr->ref_task_tag);
@@ -312,6 +327,11 @@
}
spin_unlock_irqrestore(&dev->execute_task_lock, flags);
+ if (dev->transport->tmr_notify)
+ dev->transport->tmr_notify(dev, preempt_and_abort_list ?
+ TMR_LUN_RESET_PRO : TMR_LUN_RESET,
+ &drain_task_list);
+
while (!list_empty(&drain_task_list)) {
cmd = list_entry(drain_task_list.next, struct se_cmd, state_list);
list_del_init(&cmd->state_list);
diff --git a/drivers/target/target_core_transport.c b/drivers/target/target_core_transport.c
index 90ecdd7..9fb0be0 100644
--- a/drivers/target/target_core_transport.c
+++ b/drivers/target/target_core_transport.c
@@ -236,6 +236,11 @@
}
EXPORT_SYMBOL(transport_init_session);
+void transport_uninit_session(struct se_session *se_sess)
+{
+ percpu_ref_exit(&se_sess->cmd_count);
+}
+
/**
* transport_alloc_session - allocate a session object and initialize it
* @sup_prot_ops: bitmask that defines which T10-PI modes are supported.
@@ -579,7 +584,7 @@
sbitmap_queue_free(&se_sess->sess_tag_pool);
kvfree(se_sess->sess_cmd_map);
}
- percpu_ref_exit(&se_sess->cmd_count);
+ transport_uninit_session(se_sess);
kmem_cache_free(se_sess_cache, se_sess);
}
EXPORT_SYMBOL(transport_free_session);
@@ -2941,6 +2946,7 @@
case TMR_LUN_RESET: return "LUN_RESET";
case TMR_TARGET_WARM_RESET: return "TARGET_WARM_RESET";
case TMR_TARGET_COLD_RESET: return "TARGET_COLD_RESET";
+ case TMR_LUN_RESET_PRO: return "LUN_RESET_PRO";
case TMR_UNKNOWN: break;
}
return "(?)";
diff --git a/drivers/target/target_core_user.c b/drivers/target/target_core_user.c
index 560bfec..9b75923 100644
--- a/drivers/target/target_core_user.c
+++ b/drivers/target/target_core_user.c
@@ -118,6 +118,7 @@
#define TCMU_DEV_BIT_OPEN 0
#define TCMU_DEV_BIT_BROKEN 1
#define TCMU_DEV_BIT_BLOCKED 2
+#define TCMU_DEV_BIT_TMR_NOTIFY 3
unsigned long flags;
struct uio_info uio_info;
@@ -137,6 +138,7 @@
struct mutex cmdr_lock;
struct list_head qfull_queue;
+ struct list_head tmr_queue;
uint32_t dbi_max;
uint32_t dbi_thresh;
@@ -181,9 +183,17 @@
unsigned long deadline;
#define TCMU_CMD_BIT_EXPIRED 0
-#define TCMU_CMD_BIT_INFLIGHT 1
unsigned long flags;
};
+
+struct tcmu_tmr {
+ struct list_head queue_entry;
+
+ uint8_t tmr_type;
+ uint32_t tmr_cmd_cnt;
+ int16_t tmr_cmd_ids[0];
+};
+
/*
* To avoid dead lock the mutex lock order should always be:
*
@@ -542,6 +552,8 @@
static inline void tcmu_free_cmd(struct tcmu_cmd *tcmu_cmd)
{
+ if (tcmu_cmd->se_cmd)
+ tcmu_cmd->se_cmd->priv = NULL;
kfree(tcmu_cmd->dbi);
kmem_cache_free(tcmu_cmd_cache, tcmu_cmd);
}
@@ -601,7 +613,7 @@
size = round_up(size+offset, PAGE_SIZE);
while (size) {
- flush_dcache_page(virt_to_page(start));
+ flush_dcache_page(vmalloc_to_page(start));
start += PAGE_SIZE;
size -= PAGE_SIZE;
}
@@ -676,8 +688,10 @@
from = kmap_atomic(sg_page(sg)) + sg->offset;
while (sg_remaining > 0) {
if (block_remaining == 0) {
- if (to)
+ if (to) {
+ flush_dcache_page(page);
kunmap_atomic(to);
+ }
block_remaining = DATA_BLOCK_SIZE;
dbi = tcmu_cmd_get_dbi(tcmu_cmd);
@@ -722,7 +736,6 @@
memcpy(to + offset,
from + sg->length - sg_remaining,
copy_bytes);
- tcmu_flush_dcache_range(to, copy_bytes);
}
sg_remaining -= copy_bytes;
@@ -731,8 +744,10 @@
kunmap_atomic(from - sg->offset);
}
- if (to)
+ if (to) {
+ flush_dcache_page(page);
kunmap_atomic(to);
+ }
}
static void gather_data_area(struct tcmu_dev *udev, struct tcmu_cmd *cmd,
@@ -778,13 +793,13 @@
dbi = tcmu_cmd_get_dbi(cmd);
page = tcmu_get_block_page(udev, dbi);
from = kmap_atomic(page);
+ flush_dcache_page(page);
}
copy_bytes = min_t(size_t, sg_remaining,
block_remaining);
if (read_len < copy_bytes)
copy_bytes = read_len;
offset = DATA_BLOCK_SIZE - block_remaining;
- tcmu_flush_dcache_range(from, copy_bytes);
memcpy(to + sg->length - sg_remaining, from + offset,
copy_bytes);
@@ -840,6 +855,9 @@
return false;
}
+ if (!data_needed)
+ return true;
+
/* try to check and get the data blocks as needed */
space = spc_bitmap_free(udev->data_bitmap, udev->dbi_thresh);
if ((space * DATA_BLOCK_SIZE) < data_needed) {
@@ -922,6 +940,34 @@
return 0;
}
+static uint32_t ring_insert_padding(struct tcmu_dev *udev, size_t cmd_size)
+{
+ struct tcmu_cmd_entry_hdr *hdr;
+ struct tcmu_mailbox *mb = udev->mb_addr;
+ uint32_t cmd_head = mb->cmd_head % udev->cmdr_size; /* UAM */
+
+ /* Insert a PAD if end-of-ring space is too small */
+ if (head_to_end(cmd_head, udev->cmdr_size) < cmd_size) {
+ size_t pad_size = head_to_end(cmd_head, udev->cmdr_size);
+
+ hdr = (void *) mb + CMDR_OFF + cmd_head;
+ tcmu_hdr_set_op(&hdr->len_op, TCMU_OP_PAD);
+ tcmu_hdr_set_len(&hdr->len_op, pad_size);
+ hdr->cmd_id = 0; /* not used for PAD */
+ hdr->kflags = 0;
+ hdr->uflags = 0;
+ tcmu_flush_dcache_range(hdr, sizeof(*hdr));
+
+ UPDATE_HEAD(mb->cmd_head, pad_size, udev->cmdr_size);
+ tcmu_flush_dcache_range(mb, sizeof(*mb));
+
+ cmd_head = mb->cmd_head % udev->cmdr_size; /* UAM */
+ WARN_ON(cmd_head != 0);
+ }
+
+ return cmd_head;
+}
+
/**
* queue_cmd_ring - queue cmd to ring or internally
* @tcmu_cmd: cmd to queue
@@ -937,7 +983,7 @@
struct tcmu_dev *udev = tcmu_cmd->tcmu_dev;
struct se_cmd *se_cmd = tcmu_cmd->se_cmd;
size_t base_command_size, command_size;
- struct tcmu_mailbox *mb;
+ struct tcmu_mailbox *mb = udev->mb_addr;
struct tcmu_cmd_entry *entry;
struct iovec *iov;
int iov_cnt, cmd_id;
@@ -976,8 +1022,6 @@
if (!list_empty(&udev->qfull_queue))
goto queue;
- mb = udev->mb_addr;
- cmd_head = mb->cmd_head % udev->cmdr_size; /* UAM */
if ((command_size > (udev->cmdr_size / 2)) ||
data_length > udev->data_size) {
pr_warn("TCMU: Request of size %zu/%zu is too big for %u/%zu "
@@ -997,24 +1041,7 @@
goto queue;
}
- /* Insert a PAD if end-of-ring space is too small */
- if (head_to_end(cmd_head, udev->cmdr_size) < command_size) {
- size_t pad_size = head_to_end(cmd_head, udev->cmdr_size);
-
- entry = (void *) mb + CMDR_OFF + cmd_head;
- tcmu_hdr_set_op(&entry->hdr.len_op, TCMU_OP_PAD);
- tcmu_hdr_set_len(&entry->hdr.len_op, pad_size);
- entry->hdr.cmd_id = 0; /* not used for PAD */
- entry->hdr.kflags = 0;
- entry->hdr.uflags = 0;
- tcmu_flush_dcache_range(entry, sizeof(entry->hdr));
-
- UPDATE_HEAD(mb->cmd_head, pad_size, udev->cmdr_size);
- tcmu_flush_dcache_range(mb, sizeof(*mb));
-
- cmd_head = mb->cmd_head % udev->cmdr_size; /* UAM */
- WARN_ON(cmd_head != 0);
- }
+ cmd_head = ring_insert_padding(udev, command_size);
entry = (void *) mb + CMDR_OFF + cmd_head;
memset(entry, 0, command_size);
@@ -1078,7 +1105,6 @@
tcmu_flush_dcache_range(mb, sizeof(*mb));
list_add_tail(&tcmu_cmd->queue_entry, &udev->inflight_queue);
- set_bit(TCMU_CMD_BIT_INFLIGHT, &tcmu_cmd->flags);
/* TODO: only if FLUSH and FUA? */
uio_event_notify(&udev->uio_info);
@@ -1094,27 +1120,177 @@
return 1;
}
+/**
+ * queue_tmr_ring - queue tmr info to ring or internally
+ * @udev: related tcmu_dev
+ * @tmr: tcmu_tmr containing tmr info to queue
+ *
+ * Returns:
+ * 0 success
+ * 1 internally queued to wait for ring memory to free.
+ */
+static int
+queue_tmr_ring(struct tcmu_dev *udev, struct tcmu_tmr *tmr)
+{
+ struct tcmu_tmr_entry *entry;
+ int cmd_size;
+ int id_list_sz;
+ struct tcmu_mailbox *mb = udev->mb_addr;
+ uint32_t cmd_head;
+
+ if (test_bit(TCMU_DEV_BIT_BROKEN, &udev->flags))
+ goto out_free;
+
+ id_list_sz = sizeof(tmr->tmr_cmd_ids[0]) * tmr->tmr_cmd_cnt;
+ cmd_size = round_up(sizeof(*entry) + id_list_sz, TCMU_OP_ALIGN_SIZE);
+
+ if (!list_empty(&udev->tmr_queue) ||
+ !is_ring_space_avail(udev, NULL, cmd_size, 0)) {
+ list_add_tail(&tmr->queue_entry, &udev->tmr_queue);
+ pr_debug("adding tmr %p on dev %s to TMR ring space wait queue\n",
+ tmr, udev->name);
+ return 1;
+ }
+
+ cmd_head = ring_insert_padding(udev, cmd_size);
+
+ entry = (void *)mb + CMDR_OFF + cmd_head;
+ memset(entry, 0, cmd_size);
+ tcmu_hdr_set_op(&entry->hdr.len_op, TCMU_OP_TMR);
+ tcmu_hdr_set_len(&entry->hdr.len_op, cmd_size);
+ entry->tmr_type = tmr->tmr_type;
+ entry->cmd_cnt = tmr->tmr_cmd_cnt;
+ memcpy(&entry->cmd_ids[0], &tmr->tmr_cmd_ids[0], id_list_sz);
+ tcmu_flush_dcache_range(entry, cmd_size);
+
+ UPDATE_HEAD(mb->cmd_head, cmd_size, udev->cmdr_size);
+ tcmu_flush_dcache_range(mb, sizeof(*mb));
+
+ uio_event_notify(&udev->uio_info);
+
+out_free:
+ kfree(tmr);
+
+ return 0;
+}
+
static sense_reason_t
tcmu_queue_cmd(struct se_cmd *se_cmd)
{
struct se_device *se_dev = se_cmd->se_dev;
struct tcmu_dev *udev = TCMU_DEV(se_dev);
struct tcmu_cmd *tcmu_cmd;
- sense_reason_t scsi_ret;
- int ret;
+ sense_reason_t scsi_ret = TCM_CHECK_CONDITION_ABORT_CMD;
+ int ret = -1;
tcmu_cmd = tcmu_alloc_cmd(se_cmd);
if (!tcmu_cmd)
return TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE;
mutex_lock(&udev->cmdr_lock);
- ret = queue_cmd_ring(tcmu_cmd, &scsi_ret);
- mutex_unlock(&udev->cmdr_lock);
+ se_cmd->priv = tcmu_cmd;
+ if (!(se_cmd->transport_state & CMD_T_ABORTED))
+ ret = queue_cmd_ring(tcmu_cmd, &scsi_ret);
if (ret < 0)
tcmu_free_cmd(tcmu_cmd);
+ mutex_unlock(&udev->cmdr_lock);
return scsi_ret;
}
+static void tcmu_set_next_deadline(struct list_head *queue,
+ struct timer_list *timer)
+{
+ struct tcmu_cmd *cmd;
+
+ if (!list_empty(queue)) {
+ cmd = list_first_entry(queue, struct tcmu_cmd, queue_entry);
+ mod_timer(timer, cmd->deadline);
+ } else
+ del_timer(timer);
+}
+
+static int
+tcmu_tmr_type(enum tcm_tmreq_table tmf)
+{
+ switch (tmf) {
+ case TMR_ABORT_TASK: return TCMU_TMR_ABORT_TASK;
+ case TMR_ABORT_TASK_SET: return TCMU_TMR_ABORT_TASK_SET;
+ case TMR_CLEAR_ACA: return TCMU_TMR_CLEAR_ACA;
+ case TMR_CLEAR_TASK_SET: return TCMU_TMR_CLEAR_TASK_SET;
+ case TMR_LUN_RESET: return TCMU_TMR_LUN_RESET;
+ case TMR_TARGET_WARM_RESET: return TCMU_TMR_TARGET_WARM_RESET;
+ case TMR_TARGET_COLD_RESET: return TCMU_TMR_TARGET_COLD_RESET;
+ case TMR_LUN_RESET_PRO: return TCMU_TMR_LUN_RESET_PRO;
+ default: return TCMU_TMR_UNKNOWN;
+ }
+}
+
+static void
+tcmu_tmr_notify(struct se_device *se_dev, enum tcm_tmreq_table tmf,
+ struct list_head *cmd_list)
+{
+ int i = 0, cmd_cnt = 0;
+ bool unqueued = false;
+ uint16_t *cmd_ids = NULL;
+ struct tcmu_cmd *cmd;
+ struct se_cmd *se_cmd;
+ struct tcmu_tmr *tmr;
+ struct tcmu_dev *udev = TCMU_DEV(se_dev);
+
+ mutex_lock(&udev->cmdr_lock);
+
+ /* First we check for aborted commands in qfull_queue */
+ list_for_each_entry(se_cmd, cmd_list, state_list) {
+ i++;
+ if (!se_cmd->priv)
+ continue;
+ cmd = se_cmd->priv;
+ /* Commands on qfull queue have no id yet */
+ if (cmd->cmd_id) {
+ cmd_cnt++;
+ continue;
+ }
+ pr_debug("Removing aborted command %p from queue on dev %s.\n",
+ cmd, udev->name);
+
+ list_del_init(&cmd->queue_entry);
+ tcmu_free_cmd(cmd);
+ target_complete_cmd(se_cmd, SAM_STAT_TASK_ABORTED);
+ unqueued = true;
+ }
+ if (unqueued)
+ tcmu_set_next_deadline(&udev->qfull_queue, &udev->qfull_timer);
+
+ if (!test_bit(TCMU_DEV_BIT_TMR_NOTIFY, &udev->flags))
+ goto unlock;
+
+ pr_debug("TMR event %d on dev %s, aborted cmds %d, afflicted cmd_ids %d\n",
+ tcmu_tmr_type(tmf), udev->name, i, cmd_cnt);
+
+ tmr = kmalloc(sizeof(*tmr) + cmd_cnt * sizeof(*cmd_ids), GFP_KERNEL);
+ if (!tmr)
+ goto unlock;
+
+ tmr->tmr_type = tcmu_tmr_type(tmf);
+ tmr->tmr_cmd_cnt = cmd_cnt;
+
+ if (cmd_cnt != 0) {
+ cmd_cnt = 0;
+ list_for_each_entry(se_cmd, cmd_list, state_list) {
+ if (!se_cmd->priv)
+ continue;
+ cmd = se_cmd->priv;
+ if (cmd->cmd_id)
+ tmr->tmr_cmd_ids[cmd_cnt++] = cmd->cmd_id;
+ }
+ }
+
+ queue_tmr_ring(udev, tmr);
+
+unlock:
+ mutex_unlock(&udev->cmdr_lock);
+}
+
static void tcmu_handle_completion(struct tcmu_cmd *cmd, struct tcmu_cmd_entry *entry)
{
struct se_cmd *se_cmd = cmd->se_cmd;
@@ -1178,35 +1354,47 @@
target_complete_cmd(cmd->se_cmd, entry->rsp.scsi_status);
out:
- cmd->se_cmd = NULL;
tcmu_cmd_free_data(cmd, cmd->dbi_cnt);
tcmu_free_cmd(cmd);
}
-static void tcmu_set_next_deadline(struct list_head *queue,
- struct timer_list *timer)
+static int tcmu_run_tmr_queue(struct tcmu_dev *udev)
{
- struct tcmu_cmd *tcmu_cmd, *tmp_cmd;
- unsigned long deadline = 0;
+ struct tcmu_tmr *tmr, *tmp;
+ LIST_HEAD(tmrs);
- list_for_each_entry_safe(tcmu_cmd, tmp_cmd, queue, queue_entry) {
- if (!time_after(jiffies, tcmu_cmd->deadline)) {
- deadline = tcmu_cmd->deadline;
- break;
+ if (list_empty(&udev->tmr_queue))
+ return 1;
+
+ pr_debug("running %s's tmr queue\n", udev->name);
+
+ list_splice_init(&udev->tmr_queue, &tmrs);
+
+ list_for_each_entry_safe(tmr, tmp, &tmrs, queue_entry) {
+ list_del_init(&tmr->queue_entry);
+
+ pr_debug("removing tmr %p on dev %s from queue\n",
+ tmr, udev->name);
+
+ if (queue_tmr_ring(udev, tmr)) {
+ pr_debug("ran out of space during tmr queue run\n");
+ /*
+ * tmr was requeued, so just put all tmrs back in
+ * the queue
+ */
+ list_splice_tail(&tmrs, &udev->tmr_queue);
+ return 0;
}
}
- if (deadline)
- mod_timer(timer, deadline);
- else
- del_timer(timer);
+ return 1;
}
static unsigned int tcmu_handle_completions(struct tcmu_dev *udev)
{
struct tcmu_mailbox *mb;
struct tcmu_cmd *cmd;
- int handled = 0;
+ bool free_space = false;
if (test_bit(TCMU_DEV_BIT_BROKEN, &udev->flags)) {
pr_err("ring broken, not handling completions\n");
@@ -1220,9 +1408,19 @@
struct tcmu_cmd_entry *entry = (void *) mb + CMDR_OFF + udev->cmdr_last_cleaned;
- tcmu_flush_dcache_range(entry, sizeof(*entry));
+ /*
+ * Flush max. up to end of cmd ring since current entry might
+ * be a padding that is shorter than sizeof(*entry)
+ */
+ size_t ring_left = head_to_end(udev->cmdr_last_cleaned,
+ udev->cmdr_size);
+ tcmu_flush_dcache_range(entry, ring_left < sizeof(*entry) ?
+ ring_left : sizeof(*entry));
- if (tcmu_hdr_get_op(entry->hdr.len_op) == TCMU_OP_PAD) {
+ free_space = true;
+
+ if (tcmu_hdr_get_op(entry->hdr.len_op) == TCMU_OP_PAD ||
+ tcmu_hdr_get_op(entry->hdr.len_op) == TCMU_OP_TMR) {
UPDATE_HEAD(udev->cmdr_last_cleaned,
tcmu_hdr_get_len(entry->hdr.len_op),
udev->cmdr_size);
@@ -1243,40 +1441,35 @@
UPDATE_HEAD(udev->cmdr_last_cleaned,
tcmu_hdr_get_len(entry->hdr.len_op),
udev->cmdr_size);
-
- handled++;
}
+ if (free_space)
+ free_space = tcmu_run_tmr_queue(udev);
- if (mb->cmd_tail == mb->cmd_head) {
- /* no more pending commands */
- del_timer(&udev->cmd_timer);
-
- if (list_empty(&udev->qfull_queue)) {
- /*
- * no more pending or waiting commands so try to
- * reclaim blocks if needed.
- */
- if (atomic_read(&global_db_count) >
- tcmu_global_max_blocks)
- schedule_delayed_work(&tcmu_unmap_work, 0);
- }
- } else if (udev->cmd_time_out) {
+ if (atomic_read(&global_db_count) > tcmu_global_max_blocks &&
+ idr_is_empty(&udev->commands) && list_empty(&udev->qfull_queue)) {
+ /*
+ * Allocated blocks exceeded global block limit, currently no
+ * more pending or waiting commands so try to reclaim blocks.
+ */
+ schedule_delayed_work(&tcmu_unmap_work, 0);
+ }
+ if (udev->cmd_time_out)
tcmu_set_next_deadline(&udev->inflight_queue, &udev->cmd_timer);
- }
- return handled;
+ return free_space;
}
static void tcmu_check_expired_ring_cmd(struct tcmu_cmd *cmd)
{
struct se_cmd *se_cmd;
- if (!time_after(jiffies, cmd->deadline))
+ if (!time_after_eq(jiffies, cmd->deadline))
return;
set_bit(TCMU_CMD_BIT_EXPIRED, &cmd->flags);
list_del_init(&cmd->queue_entry);
se_cmd = cmd->se_cmd;
+ se_cmd->priv = NULL;
cmd->se_cmd = NULL;
pr_debug("Timing out inflight cmd %u on dev %s.\n",
@@ -1289,7 +1482,7 @@
{
struct se_cmd *se_cmd;
- if (!time_after(jiffies, cmd->deadline))
+ if (!time_after_eq(jiffies, cmd->deadline))
return;
pr_debug("Timing out queued cmd %p on dev %s.\n",
@@ -1373,6 +1566,7 @@
INIT_LIST_HEAD(&udev->node);
INIT_LIST_HEAD(&udev->timedout_entry);
INIT_LIST_HEAD(&udev->qfull_queue);
+ INIT_LIST_HEAD(&udev->tmr_queue);
INIT_LIST_HEAD(&udev->inflight_queue);
idr_init(&udev->commands);
@@ -1447,8 +1641,8 @@
struct tcmu_dev *udev = container_of(info, struct tcmu_dev, uio_info);
mutex_lock(&udev->cmdr_lock);
- tcmu_handle_completions(udev);
- run_qfull_queue(udev, false);
+ if (tcmu_handle_completions(udev))
+ run_qfull_queue(udev, false);
mutex_unlock(&udev->cmdr_lock);
return 0;
@@ -1601,6 +1795,16 @@
}
}
+static void tcmu_remove_all_queued_tmr(struct tcmu_dev *udev)
+{
+ struct tcmu_tmr *tmr, *tmp;
+
+ list_for_each_entry_safe(tmr, tmp, &udev->tmr_queue, queue_entry) {
+ list_del_init(&tmr->queue_entry);
+ kfree(tmr);
+ }
+}
+
static void tcmu_dev_kref_release(struct kref *kref)
{
struct tcmu_dev *udev = container_of(kref, struct tcmu_dev, kref);
@@ -1623,6 +1827,8 @@
if (tcmu_check_and_free_pending_cmd(cmd) != 0)
all_expired = false;
}
+ /* There can be left over TMR cmds. Remove them. */
+ tcmu_remove_all_queued_tmr(udev);
if (!list_empty(&udev->qfull_queue))
all_expired = false;
idr_destroy(&udev->commands);
@@ -1877,7 +2083,9 @@
/* Initialise the mailbox of the ring buffer */
mb = udev->mb_addr;
mb->version = TCMU_MAILBOX_VERSION;
- mb->flags = TCMU_MAILBOX_FLAG_CAP_OOOC | TCMU_MAILBOX_FLAG_CAP_READ_LEN;
+ mb->flags = TCMU_MAILBOX_FLAG_CAP_OOOC |
+ TCMU_MAILBOX_FLAG_CAP_READ_LEN |
+ TCMU_MAILBOX_FLAG_CAP_TMR;
mb->cmdr_off = CMDR_OFF;
mb->cmdr_size = udev->cmdr_size;
@@ -2047,6 +2255,15 @@
del_timer(&udev->cmd_timer);
+ /*
+ * ring is empty and qfull queue never contains aborted commands.
+ * So TMRs in tmr queue do not contain relevant cmd_ids.
+ * After a ring reset userspace should do a fresh start, so
+ * even LUN RESET message is no longer relevant.
+ * Therefore remove all TMRs from qfull queue
+ */
+ tcmu_remove_all_queued_tmr(udev);
+
run_qfull_queue(udev, false);
mutex_unlock(&udev->cmdr_lock);
@@ -2493,6 +2710,39 @@
}
CONFIGFS_ATTR(tcmu_, emulate_write_cache);
+static ssize_t tcmu_tmr_notification_show(struct config_item *item, char *page)
+{
+ struct se_dev_attrib *da = container_of(to_config_group(item),
+ struct se_dev_attrib, da_group);
+ struct tcmu_dev *udev = TCMU_DEV(da->da_dev);
+
+ return snprintf(page, PAGE_SIZE, "%i\n",
+ test_bit(TCMU_DEV_BIT_TMR_NOTIFY, &udev->flags));
+}
+
+static ssize_t tcmu_tmr_notification_store(struct config_item *item,
+ const char *page, size_t count)
+{
+ struct se_dev_attrib *da = container_of(to_config_group(item),
+ struct se_dev_attrib, da_group);
+ struct tcmu_dev *udev = TCMU_DEV(da->da_dev);
+ u8 val;
+ int ret;
+
+ ret = kstrtou8(page, 0, &val);
+ if (ret < 0)
+ return ret;
+ if (val > 1)
+ return -EINVAL;
+
+ if (val)
+ set_bit(TCMU_DEV_BIT_TMR_NOTIFY, &udev->flags);
+ else
+ clear_bit(TCMU_DEV_BIT_TMR_NOTIFY, &udev->flags);
+ return count;
+}
+CONFIGFS_ATTR(tcmu_, tmr_notification);
+
static ssize_t tcmu_block_dev_show(struct config_item *item, char *page)
{
struct se_device *se_dev = container_of(to_config_group(item),
@@ -2574,6 +2824,7 @@
&tcmu_attr_dev_config,
&tcmu_attr_dev_size,
&tcmu_attr_emulate_write_cache,
+ &tcmu_attr_tmr_notification,
&tcmu_attr_nl_reply_supported,
NULL,
};
@@ -2599,6 +2850,7 @@
.destroy_device = tcmu_destroy_device,
.free_device = tcmu_free_device,
.parse_cdb = tcmu_parse_cdb,
+ .tmr_notify = tcmu_tmr_notify,
.set_configfs_dev_params = tcmu_set_configfs_dev_params,
.show_configfs_dev_params = tcmu_show_configfs_dev_params,
.get_device_type = sbc_get_device_type,
@@ -2625,7 +2877,8 @@
}
/* Try to complete the finished commands first */
- tcmu_handle_completions(udev);
+ if (tcmu_handle_completions(udev))
+ run_qfull_queue(udev, false);
/* Skip the udevs in idle */
if (!udev->dbi_thresh) {
diff --git a/drivers/target/target_core_xcopy.c b/drivers/target/target_core_xcopy.c
index 0d00ccb..44e15d7 100644
--- a/drivers/target/target_core_xcopy.c
+++ b/drivers/target/target_core_xcopy.c
@@ -474,7 +474,7 @@
memset(&xcopy_pt_sess, 0, sizeof(struct se_session));
ret = transport_init_session(&xcopy_pt_sess);
if (ret < 0)
- return ret;
+ goto destroy_wq;
xcopy_pt_nacl.se_tpg = &xcopy_pt_tpg;
xcopy_pt_nacl.nacl_sess = &xcopy_pt_sess;
@@ -483,12 +483,19 @@
xcopy_pt_sess.se_node_acl = &xcopy_pt_nacl;
return 0;
+
+destroy_wq:
+ destroy_workqueue(xcopy_wq);
+ xcopy_wq = NULL;
+ return ret;
}
void target_xcopy_release_pt(void)
{
- if (xcopy_wq)
+ if (xcopy_wq) {
destroy_workqueue(xcopy_wq);
+ transport_uninit_session(&xcopy_pt_sess);
+ }
}
/*
diff --git a/drivers/thermal/Kconfig b/drivers/thermal/Kconfig
index 3eb2348..a2e710d 100644
--- a/drivers/thermal/Kconfig
+++ b/drivers/thermal/Kconfig
@@ -17,6 +17,15 @@
if THERMAL
+config THERMAL_NETLINK
+ bool "Thermal netlink management"
+ depends on NET
+ help
+ The thermal framework has a netlink interface to do thermal
+ zones discovery, temperature readings and events such as
+ trip point crossed, cooling device update or governor
+ change. It is recommended to enable the feature.
+
config THERMAL_STATISTICS
bool "Thermal state transition statistics"
help
@@ -180,16 +189,6 @@
idle cycle.
endif
-config CLOCK_THERMAL
- bool "Generic clock cooling support"
- depends on COMMON_CLK
- depends on PM_OPP
- help
- This entry implements the generic clock cooling mechanism through
- frequency clipping. Typically used to cool off co-processors. The
- device that is configured to use this cooling mechanism will be
- controlled to reduce clock frequency whenever temperature is high.
-
config DEVFREQ_THERMAL
bool "Generic device cooling support"
depends on PM_DEVFREQ
diff --git a/drivers/thermal/Makefile b/drivers/thermal/Makefile
index 0c8b84a..b8d96d2 100644
--- a/drivers/thermal/Makefile
+++ b/drivers/thermal/Makefile
@@ -7,6 +7,9 @@
thermal_sys-y += thermal_core.o thermal_sysfs.o \
thermal_helpers.o
+# netlink interface to manage the thermal framework
+thermal_sys-$(CONFIG_THERMAL_NETLINK) += thermal_netlink.o
+
# interface to/from other layers providing sensors
thermal_sys-$(CONFIG_THERMAL_HWMON) += thermal_hwmon.o
thermal_sys-$(CONFIG_THERMAL_OF) += thermal_of.o
@@ -22,9 +25,6 @@
thermal_sys-$(CONFIG_CPU_FREQ_THERMAL) += cpufreq_cooling.o
thermal_sys-$(CONFIG_CPU_IDLE_THERMAL) += cpuidle_cooling.o
-# clock cooling
-thermal_sys-$(CONFIG_CLOCK_THERMAL) += clock_cooling.o
-
# devfreq cooling
thermal_sys-$(CONFIG_DEVFREQ_THERMAL) += devfreq_cooling.o
diff --git a/drivers/thermal/armada_thermal.c b/drivers/thermal/armada_thermal.c
index 7c447cd..c2ebfb5 100644
--- a/drivers/thermal/armada_thermal.c
+++ b/drivers/thermal/armada_thermal.c
@@ -874,6 +874,12 @@
return PTR_ERR(tz);
}
+ ret = thermal_zone_device_enable(tz);
+ if (ret) {
+ thermal_zone_device_unregister(tz);
+ return ret;
+ }
+
drvdata->type = LEGACY;
drvdata->data.tz = tz;
platform_set_drvdata(pdev, drvdata);
diff --git a/drivers/thermal/clock_cooling.c b/drivers/thermal/clock_cooling.c
deleted file mode 100644
index 56cb1f4..0000000
--- a/drivers/thermal/clock_cooling.c
+++ /dev/null
@@ -1,445 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-only
-/*
- * drivers/thermal/clock_cooling.c
- *
- * Copyright (C) 2014 Eduardo Valentin <edubezval@gmail.com>
- *
- * Copyright (C) 2013 Texas Instruments Inc.
- * Contact: Eduardo Valentin <eduardo.valentin@ti.com>
- *
- * Highly based on cpufreq_cooling.c.
- * Copyright (C) 2012 Samsung Electronics Co., Ltd(http://www.samsung.com)
- * Copyright (C) 2012 Amit Daniel <amit.kachhap@linaro.org>
- */
-#include <linux/clk.h>
-#include <linux/clock_cooling.h>
-#include <linux/cpufreq.h>
-#include <linux/device.h>
-#include <linux/err.h>
-#include <linux/export.h>
-#include <linux/idr.h>
-#include <linux/mutex.h>
-#include <linux/pm_opp.h>
-#include <linux/slab.h>
-#include <linux/thermal.h>
-
-/**
- * struct clock_cooling_device - data for cooling device with clock
- * @id: unique integer value corresponding to each clock_cooling_device
- * registered.
- * @dev: struct device pointer to the device being used to cool off using
- * clock frequencies.
- * @cdev: thermal_cooling_device pointer to keep track of the
- * registered cooling device.
- * @clk_rate_change_nb: reference to notifier block used to receive clock
- * rate changes.
- * @freq_table: frequency table used to keep track of available frequencies.
- * @clock_state: integer value representing the current state of clock
- * cooling devices.
- * @clock_val: integer value representing the absolute value of the clipped
- * frequency.
- * @clk: struct clk reference used to enforce clock limits.
- * @lock: mutex lock to protect this struct.
- *
- * This structure is required for keeping information of each
- * clock_cooling_device registered. In order to prevent corruption of this a
- * mutex @lock is used.
- */
-struct clock_cooling_device {
- int id;
- struct device *dev;
- struct thermal_cooling_device *cdev;
- struct notifier_block clk_rate_change_nb;
- struct cpufreq_frequency_table *freq_table;
- unsigned long clock_state;
- unsigned long clock_val;
- struct clk *clk;
- struct mutex lock; /* lock to protect the content of this struct */
-};
-#define to_clock_cooling_device(x) \
- container_of(x, struct clock_cooling_device, clk_rate_change_nb)
-static DEFINE_IDA(clock_ida);
-
-/* Below code defines functions to be used for clock as cooling device */
-
-enum clock_cooling_property {
- GET_LEVEL,
- GET_FREQ,
- GET_MAXL,
-};
-
-/**
- * clock_cooling_get_property - fetch a property of interest for a give cpu.
- * @ccdev: clock cooling device reference
- * @input: query parameter
- * @output: query return
- * @property: type of query (frequency, level, max level)
- *
- * This is the common function to
- * 1. get maximum clock cooling states
- * 2. translate frequency to cooling state
- * 3. translate cooling state to frequency
- * Note that the code may be not in good shape
- * but it is written in this way in order to:
- * a) reduce duplicate code as most of the code can be shared.
- * b) make sure the logic is consistent when translating between
- * cooling states and frequencies.
- *
- * Return: 0 on success, -EINVAL when invalid parameters are passed.
- */
-static int clock_cooling_get_property(struct clock_cooling_device *ccdev,
- unsigned long input,
- unsigned long *output,
- enum clock_cooling_property property)
-{
- int i;
- unsigned long max_level = 0, level = 0;
- unsigned int freq = CPUFREQ_ENTRY_INVALID;
- int descend = -1;
- struct cpufreq_frequency_table *pos, *table = ccdev->freq_table;
-
- if (!output)
- return -EINVAL;
-
- if (!table)
- return -EINVAL;
-
- cpufreq_for_each_valid_entry(pos, table) {
- /* ignore duplicate entry */
- if (freq == pos->frequency)
- continue;
-
- /* get the frequency order */
- if (freq != CPUFREQ_ENTRY_INVALID && descend == -1)
- descend = freq > pos->frequency;
-
- freq = pos->frequency;
- max_level++;
- }
-
- /* No valid cpu frequency entry */
- if (max_level == 0)
- return -EINVAL;
-
- /* max_level is an index, not a counter */
- max_level--;
-
- /* get max level */
- if (property == GET_MAXL) {
- *output = max_level;
- return 0;
- }
-
- if (property == GET_FREQ)
- level = descend ? input : (max_level - input);
-
- i = 0;
- cpufreq_for_each_valid_entry(pos, table) {
- /* ignore duplicate entry */
- if (freq == pos->frequency)
- continue;
-
- /* now we have a valid frequency entry */
- freq = pos->frequency;
-
- if (property == GET_LEVEL && (unsigned int)input == freq) {
- /* get level by frequency */
- *output = descend ? i : (max_level - i);
- return 0;
- }
- if (property == GET_FREQ && level == i) {
- /* get frequency by level */
- *output = freq;
- return 0;
- }
- i++;
- }
-
- return -EINVAL;
-}
-
-/**
- * clock_cooling_get_level - return the cooling level of given clock cooling.
- * @cdev: reference of a thermal cooling device of used as clock cooling device
- * @freq: the frequency of interest
- *
- * This function will match the cooling level corresponding to the
- * requested @freq and return it.
- *
- * Return: The matched cooling level on success or THERMAL_CSTATE_INVALID
- * otherwise.
- */
-unsigned long clock_cooling_get_level(struct thermal_cooling_device *cdev,
- unsigned long freq)
-{
- struct clock_cooling_device *ccdev = cdev->devdata;
- unsigned long val;
-
- if (clock_cooling_get_property(ccdev, (unsigned long)freq, &val,
- GET_LEVEL))
- return THERMAL_CSTATE_INVALID;
-
- return val;
-}
-EXPORT_SYMBOL_GPL(clock_cooling_get_level);
-
-/**
- * clock_cooling_get_frequency - get the absolute value of frequency from level.
- * @ccdev: clock cooling device reference
- * @level: cooling level
- *
- * This function matches cooling level with frequency. Based on a cooling level
- * of frequency, equals cooling state of cpu cooling device, it will return
- * the corresponding frequency.
- * e.g level=0 --> 1st MAX FREQ, level=1 ---> 2nd MAX FREQ, .... etc
- *
- * Return: 0 on error, the corresponding frequency otherwise.
- */
-static unsigned long
-clock_cooling_get_frequency(struct clock_cooling_device *ccdev,
- unsigned long level)
-{
- int ret = 0;
- unsigned long freq;
-
- ret = clock_cooling_get_property(ccdev, level, &freq, GET_FREQ);
- if (ret)
- return 0;
-
- return freq;
-}
-
-/**
- * clock_cooling_apply - function to apply frequency clipping.
- * @ccdev: clock_cooling_device pointer containing frequency clipping data.
- * @cooling_state: value of the cooling state.
- *
- * Function used to make sure the clock layer is aware of current thermal
- * limits. The limits are applied by updating the clock rate in case it is
- * higher than the corresponding frequency based on the requested cooling_state.
- *
- * Return: 0 on success, an error code otherwise (-EINVAL in case wrong
- * cooling state).
- */
-static int clock_cooling_apply(struct clock_cooling_device *ccdev,
- unsigned long cooling_state)
-{
- unsigned long clip_freq, cur_freq;
- int ret = 0;
-
- /* Here we write the clipping */
- /* Check if the old cooling action is same as new cooling action */
- if (ccdev->clock_state == cooling_state)
- return 0;
-
- clip_freq = clock_cooling_get_frequency(ccdev, cooling_state);
- if (!clip_freq)
- return -EINVAL;
-
- cur_freq = clk_get_rate(ccdev->clk);
-
- mutex_lock(&ccdev->lock);
- ccdev->clock_state = cooling_state;
- ccdev->clock_val = clip_freq;
- /* enforce clock level */
- if (cur_freq > clip_freq)
- ret = clk_set_rate(ccdev->clk, clip_freq);
- mutex_unlock(&ccdev->lock);
-
- return ret;
-}
-
-/**
- * clock_cooling_clock_notifier - notifier callback on clock rate changes.
- * @nb: struct notifier_block * with callback info.
- * @event: value showing clock event for which this function invoked.
- * @data: callback-specific data
- *
- * Callback to hijack the notification on clock transition.
- * Every time there is a clock change, we intercept all pre change events
- * and block the transition in case the new rate infringes thermal limits.
- *
- * Return: NOTIFY_DONE (success) or NOTIFY_BAD (new_rate > thermal limit).
- */
-static int clock_cooling_clock_notifier(struct notifier_block *nb,
- unsigned long event, void *data)
-{
- struct clk_notifier_data *ndata = data;
- struct clock_cooling_device *ccdev = to_clock_cooling_device(nb);
-
- switch (event) {
- case PRE_RATE_CHANGE:
- /*
- * checks on current state
- * TODO: current method is not best we can find as it
- * allows possibly voltage transitions, in case DVFS
- * layer is also hijacking clock pre notifications.
- */
- if (ndata->new_rate > ccdev->clock_val)
- return NOTIFY_BAD;
- /* fall through */
- case POST_RATE_CHANGE:
- case ABORT_RATE_CHANGE:
- default:
- return NOTIFY_DONE;
- }
-}
-
-/* clock cooling device thermal callback functions are defined below */
-
-/**
- * clock_cooling_get_max_state - callback function to get the max cooling state.
- * @cdev: thermal cooling device pointer.
- * @state: fill this variable with the max cooling state.
- *
- * Callback for the thermal cooling device to return the clock
- * max cooling state.
- *
- * Return: 0 on success, an error code otherwise.
- */
-static int clock_cooling_get_max_state(struct thermal_cooling_device *cdev,
- unsigned long *state)
-{
- struct clock_cooling_device *ccdev = cdev->devdata;
- unsigned long count = 0;
- int ret;
-
- ret = clock_cooling_get_property(ccdev, 0, &count, GET_MAXL);
- if (!ret)
- *state = count;
-
- return ret;
-}
-
-/**
- * clock_cooling_get_cur_state - function to get the current cooling state.
- * @cdev: thermal cooling device pointer.
- * @state: fill this variable with the current cooling state.
- *
- * Callback for the thermal cooling device to return the clock
- * current cooling state.
- *
- * Return: 0 (success)
- */
-static int clock_cooling_get_cur_state(struct thermal_cooling_device *cdev,
- unsigned long *state)
-{
- struct clock_cooling_device *ccdev = cdev->devdata;
-
- *state = ccdev->clock_state;
-
- return 0;
-}
-
-/**
- * clock_cooling_set_cur_state - function to set the current cooling state.
- * @cdev: thermal cooling device pointer.
- * @state: set this variable to the current cooling state.
- *
- * Callback for the thermal cooling device to change the clock cooling
- * current cooling state.
- *
- * Return: 0 on success, an error code otherwise.
- */
-static int clock_cooling_set_cur_state(struct thermal_cooling_device *cdev,
- unsigned long state)
-{
- struct clock_cooling_device *clock_device = cdev->devdata;
-
- return clock_cooling_apply(clock_device, state);
-}
-
-/* Bind clock callbacks to thermal cooling device ops */
-static struct thermal_cooling_device_ops const clock_cooling_ops = {
- .get_max_state = clock_cooling_get_max_state,
- .get_cur_state = clock_cooling_get_cur_state,
- .set_cur_state = clock_cooling_set_cur_state,
-};
-
-/**
- * clock_cooling_register - function to create clock cooling device.
- * @dev: struct device pointer to the device used as clock cooling device.
- * @clock_name: string containing the clock used as cooling mechanism.
- *
- * This interface function registers the clock cooling device with the name
- * "thermal-clock-%x". The cooling device is based on clock frequencies.
- * The struct device is assumed to be capable of DVFS transitions.
- * The OPP layer is used to fetch and fill the available frequencies for
- * the referred device. The ordered frequency table is used to control
- * the clock cooling device cooling states and to limit clock transitions
- * based on the cooling state requested by the thermal framework.
- *
- * Return: a valid struct thermal_cooling_device pointer on success,
- * on failure, it returns a corresponding ERR_PTR().
- */
-struct thermal_cooling_device *
-clock_cooling_register(struct device *dev, const char *clock_name)
-{
- struct thermal_cooling_device *cdev;
- struct clock_cooling_device *ccdev = NULL;
- char dev_name[THERMAL_NAME_LENGTH];
- int ret = 0;
-
- ccdev = devm_kzalloc(dev, sizeof(*ccdev), GFP_KERNEL);
- if (!ccdev)
- return ERR_PTR(-ENOMEM);
-
- mutex_init(&ccdev->lock);
- ccdev->dev = dev;
- ccdev->clk = devm_clk_get(dev, clock_name);
- if (IS_ERR(ccdev->clk))
- return ERR_CAST(ccdev->clk);
-
- ret = ida_simple_get(&clock_ida, 0, 0, GFP_KERNEL);
- if (ret < 0)
- return ERR_PTR(ret);
- ccdev->id = ret;
-
- snprintf(dev_name, sizeof(dev_name), "thermal-clock-%d", ccdev->id);
-
- cdev = thermal_cooling_device_register(dev_name, ccdev,
- &clock_cooling_ops);
- if (IS_ERR(cdev)) {
- ida_simple_remove(&clock_ida, ccdev->id);
- return ERR_PTR(-EINVAL);
- }
- ccdev->cdev = cdev;
- ccdev->clk_rate_change_nb.notifier_call = clock_cooling_clock_notifier;
-
- /* Assuming someone has already filled the opp table for this device */
- ret = dev_pm_opp_init_cpufreq_table(dev, &ccdev->freq_table);
- if (ret) {
- ida_simple_remove(&clock_ida, ccdev->id);
- return ERR_PTR(ret);
- }
- ccdev->clock_state = 0;
- ccdev->clock_val = clock_cooling_get_frequency(ccdev, 0);
-
- clk_notifier_register(ccdev->clk, &ccdev->clk_rate_change_nb);
-
- return cdev;
-}
-EXPORT_SYMBOL_GPL(clock_cooling_register);
-
-/**
- * clock_cooling_unregister - function to remove clock cooling device.
- * @cdev: thermal cooling device pointer.
- *
- * This interface function unregisters the "thermal-clock-%x" cooling device.
- */
-void clock_cooling_unregister(struct thermal_cooling_device *cdev)
-{
- struct clock_cooling_device *ccdev;
-
- if (!cdev)
- return;
-
- ccdev = cdev->devdata;
-
- clk_notifier_unregister(ccdev->clk, &ccdev->clk_rate_change_nb);
- dev_pm_opp_free_cpufreq_table(ccdev->dev, &ccdev->freq_table);
-
- thermal_cooling_device_unregister(ccdev->cdev);
- ida_simple_remove(&clock_ida, ccdev->id);
-}
-EXPORT_SYMBOL_GPL(clock_cooling_unregister);
diff --git a/drivers/thermal/da9062-thermal.c b/drivers/thermal/da9062-thermal.c
index c32709b..4d74994 100644
--- a/drivers/thermal/da9062-thermal.c
+++ b/drivers/thermal/da9062-thermal.c
@@ -49,7 +49,6 @@
struct da9062 *hw;
struct delayed_work work;
struct thermal_zone_device *zone;
- enum thermal_device_mode mode;
struct mutex lock; /* protection for da9062_thermal temperature */
int temperature;
int irq;
@@ -121,14 +120,6 @@
return IRQ_HANDLED;
}
-static int da9062_thermal_get_mode(struct thermal_zone_device *z,
- enum thermal_device_mode *mode)
-{
- struct da9062_thermal *thermal = z->devdata;
- *mode = thermal->mode;
- return 0;
-}
-
static int da9062_thermal_get_trip_type(struct thermal_zone_device *z,
int trip,
enum thermal_trip_type *type)
@@ -181,7 +172,6 @@
static struct thermal_zone_device_ops da9062_thermal_ops = {
.get_temp = da9062_thermal_get_temp,
- .get_mode = da9062_thermal_get_mode,
.get_trip_type = da9062_thermal_get_trip_type,
.get_trip_temp = da9062_thermal_get_trip_temp,
};
@@ -233,7 +223,6 @@
thermal->config = match->data;
thermal->hw = chip;
- thermal->mode = THERMAL_DEVICE_ENABLED;
thermal->dev = &pdev->dev;
INIT_DELAYED_WORK(&thermal->work, da9062_thermal_poll_on);
@@ -248,6 +237,11 @@
ret = PTR_ERR(thermal->zone);
goto err;
}
+ ret = thermal_zone_device_enable(thermal->zone);
+ if (ret) {
+ dev_err(&pdev->dev, "Cannot enable thermal zone device\n");
+ goto err_zone;
+ }
dev_dbg(&pdev->dev,
"TJUNC temperature polling period set at %d ms\n",
diff --git a/drivers/thermal/devfreq_cooling.c b/drivers/thermal/devfreq_cooling.c
index f7f32e9..a12d290 100644
--- a/drivers/thermal/devfreq_cooling.c
+++ b/drivers/thermal/devfreq_cooling.c
@@ -1,18 +1,10 @@
+// SPDX-License-Identifier: GPL-2.0
/*
* devfreq_cooling: Thermal cooling device implementation for devices using
* devfreq
*
* Copyright (C) 2014-2015 ARM Limited
*
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
- *
- * This program is distributed "as is" WITHOUT ANY WARRANTY of any
- * kind, whether express or implied; without even the implied warranty
- * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU General Public License for more details.
- *
* TODO:
* - If OPPs are added or removed after devfreq cooling has
* registered, the devfreq cooling won't react to it.
diff --git a/drivers/thermal/dove_thermal.c b/drivers/thermal/dove_thermal.c
index 75901ce..73182eb 100644
--- a/drivers/thermal/dove_thermal.c
+++ b/drivers/thermal/dove_thermal.c
@@ -153,6 +153,12 @@
return PTR_ERR(thermal);
}
+ ret = thermal_zone_device_enable(thermal);
+ if (ret) {
+ thermal_zone_device_unregister(thermal);
+ return ret;
+ }
+
platform_set_drvdata(pdev, thermal);
return 0;
diff --git a/drivers/thermal/gov_power_allocator.c b/drivers/thermal/gov_power_allocator.c
index 4463647..5cb518d 100644
--- a/drivers/thermal/gov_power_allocator.c
+++ b/drivers/thermal/gov_power_allocator.c
@@ -1,16 +1,9 @@
+// SPDX-License-Identifier: GPL-2.0
/*
* A power allocator to manage temperature
*
* Copyright (C) 2014 ARM Ltd.
*
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
- *
- * This program is distributed "as is" WITHOUT ANY WARRANTY of any
- * kind, whether express or implied; without even the implied warranty
- * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU General Public License for more details.
*/
#define pr_fmt(fmt) "Power allocator: " fmt
diff --git a/drivers/thermal/hisi_thermal.c b/drivers/thermal/hisi_thermal.c
index 2d26ae8..ee05950 100644
--- a/drivers/thermal/hisi_thermal.c
+++ b/drivers/thermal/hisi_thermal.c
@@ -549,8 +549,10 @@
{
struct thermal_zone_device *tzd = sensor->tzd;
- tzd->ops->set_mode(tzd,
- on ? THERMAL_DEVICE_ENABLED : THERMAL_DEVICE_DISABLED);
+ if (on)
+ thermal_zone_device_enable(tzd);
+ else
+ thermal_zone_device_disable(tzd);
}
static int hisi_thermal_probe(struct platform_device *pdev)
diff --git a/drivers/thermal/imx8mm_thermal.c b/drivers/thermal/imx8mm_thermal.c
index e6061e2..f5124f1 100644
--- a/drivers/thermal/imx8mm_thermal.c
+++ b/drivers/thermal/imx8mm_thermal.c
@@ -220,6 +220,7 @@
{ .compatible = "fsl,imx8mp-tmu", .data = &imx8mp_tmu_data, },
{ },
};
+MODULE_DEVICE_TABLE(of, imx8mm_tmu_table);
static struct platform_driver imx8mm_tmu = {
.driver = {
diff --git a/drivers/thermal/imx_thermal.c b/drivers/thermal/imx_thermal.c
index 1b84ea6..3f74ab4 100644
--- a/drivers/thermal/imx_thermal.c
+++ b/drivers/thermal/imx_thermal.c
@@ -197,7 +197,6 @@
struct cpufreq_policy *policy;
struct thermal_zone_device *tz;
struct thermal_cooling_device *cdev;
- enum thermal_device_mode mode;
struct regmap *tempmon;
u32 c1, c2; /* See formula in imx_init_calib() */
int temp_passive;
@@ -253,10 +252,11 @@
const struct thermal_soc_data *soc_data = data->socdata;
struct regmap *map = data->tempmon;
unsigned int n_meas;
- bool wait;
+ bool wait, run_measurement;
u32 val;
- if (data->mode == THERMAL_DEVICE_ENABLED) {
+ run_measurement = !data->irq_enabled;
+ if (!run_measurement) {
/* Check if a measurement is currently in progress */
regmap_read(map, soc_data->temp_data, &val);
wait = !(val & soc_data->temp_valid_mask);
@@ -283,7 +283,7 @@
regmap_read(map, soc_data->temp_data, &val);
- if (data->mode != THERMAL_DEVICE_ENABLED) {
+ if (run_measurement) {
regmap_write(map, soc_data->sensor_ctrl + REG_CLR,
soc_data->measure_temp_mask);
regmap_write(map, soc_data->sensor_ctrl + REG_SET,
@@ -331,27 +331,14 @@
return 0;
}
-static int imx_get_mode(struct thermal_zone_device *tz,
- enum thermal_device_mode *mode)
-{
- struct imx_thermal_data *data = tz->devdata;
-
- *mode = data->mode;
-
- return 0;
-}
-
-static int imx_set_mode(struct thermal_zone_device *tz,
- enum thermal_device_mode mode)
+static int imx_change_mode(struct thermal_zone_device *tz,
+ enum thermal_device_mode mode)
{
struct imx_thermal_data *data = tz->devdata;
struct regmap *map = data->tempmon;
const struct thermal_soc_data *soc_data = data->socdata;
if (mode == THERMAL_DEVICE_ENABLED) {
- tz->polling_delay = IMX_POLLING_DELAY;
- tz->passive_delay = IMX_PASSIVE_DELAY;
-
regmap_write(map, soc_data->sensor_ctrl + REG_CLR,
soc_data->power_down_mask);
regmap_write(map, soc_data->sensor_ctrl + REG_SET,
@@ -367,18 +354,12 @@
regmap_write(map, soc_data->sensor_ctrl + REG_SET,
soc_data->power_down_mask);
- tz->polling_delay = 0;
- tz->passive_delay = 0;
-
if (data->irq_enabled) {
disable_irq(data->irq);
data->irq_enabled = false;
}
}
- data->mode = mode;
- thermal_zone_device_update(tz, THERMAL_EVENT_UNSPECIFIED);
-
return 0;
}
@@ -467,8 +448,7 @@
.bind = imx_bind,
.unbind = imx_unbind,
.get_temp = imx_get_temp,
- .get_mode = imx_get_mode,
- .set_mode = imx_set_mode,
+ .change_mode = imx_change_mode,
.get_trip_type = imx_get_trip_type,
.get_trip_temp = imx_get_trip_temp,
.get_crit_temp = imx_get_crit_temp,
@@ -832,7 +812,9 @@
data->socdata->measure_temp_mask);
data->irq_enabled = true;
- data->mode = THERMAL_DEVICE_ENABLED;
+ ret = thermal_zone_device_enable(data->tz);
+ if (ret)
+ goto thermal_zone_unregister;
ret = devm_request_threaded_irq(&pdev->dev, data->irq,
imx_thermal_alarm_irq, imx_thermal_alarm_irq_thread,
@@ -874,19 +856,18 @@
static int __maybe_unused imx_thermal_suspend(struct device *dev)
{
struct imx_thermal_data *data = dev_get_drvdata(dev);
- struct regmap *map = data->tempmon;
+ int ret;
/*
* Need to disable thermal sensor, otherwise, when thermal core
* try to get temperature before thermal sensor resume, a wrong
* temperature will be read as the thermal sensor is powered
- * down.
+ * down. This is done in change_mode() operation called from
+ * thermal_zone_device_disable()
*/
- regmap_write(map, data->socdata->sensor_ctrl + REG_CLR,
- data->socdata->measure_temp_mask);
- regmap_write(map, data->socdata->sensor_ctrl + REG_SET,
- data->socdata->power_down_mask);
- data->mode = THERMAL_DEVICE_DISABLED;
+ ret = thermal_zone_device_disable(data->tz);
+ if (ret)
+ return ret;
clk_disable_unprepare(data->thermal_clk);
return 0;
@@ -895,18 +876,15 @@
static int __maybe_unused imx_thermal_resume(struct device *dev)
{
struct imx_thermal_data *data = dev_get_drvdata(dev);
- struct regmap *map = data->tempmon;
int ret;
ret = clk_prepare_enable(data->thermal_clk);
if (ret)
return ret;
/* Enabled thermal sensor after resume */
- regmap_write(map, data->socdata->sensor_ctrl + REG_CLR,
- data->socdata->power_down_mask);
- regmap_write(map, data->socdata->sensor_ctrl + REG_SET,
- data->socdata->measure_temp_mask);
- data->mode = THERMAL_DEVICE_ENABLED;
+ ret = thermal_zone_device_enable(data->tz);
+ if (ret)
+ return ret;
return 0;
}
diff --git a/drivers/thermal/intel/int340x_thermal/int3400_thermal.c b/drivers/thermal/intel/int340x_thermal/int3400_thermal.c
index 12448cc..4f5859d 100644
--- a/drivers/thermal/intel/int340x_thermal/int3400_thermal.c
+++ b/drivers/thermal/intel/int340x_thermal/int3400_thermal.c
@@ -48,7 +48,6 @@
struct acpi_device *adev;
struct platform_device *pdev;
struct thermal_zone_device *thermal;
- int mode;
int art_count;
struct art *arts;
int trt_count;
@@ -383,42 +382,20 @@
return 0;
}
-static int int3400_thermal_get_mode(struct thermal_zone_device *thermal,
- enum thermal_device_mode *mode)
+static int int3400_thermal_change_mode(struct thermal_zone_device *thermal,
+ enum thermal_device_mode mode)
{
struct int3400_thermal_priv *priv = thermal->devdata;
-
- if (!priv)
- return -EINVAL;
-
- *mode = priv->mode;
-
- return 0;
-}
-
-static int int3400_thermal_set_mode(struct thermal_zone_device *thermal,
- enum thermal_device_mode mode)
-{
- struct int3400_thermal_priv *priv = thermal->devdata;
- bool enable;
int result = 0;
if (!priv)
return -EINVAL;
- if (mode == THERMAL_DEVICE_ENABLED)
- enable = true;
- else if (mode == THERMAL_DEVICE_DISABLED)
- enable = false;
- else
- return -EINVAL;
-
- if (enable != priv->mode) {
- priv->mode = enable;
+ if (mode != thermal->mode)
result = int3400_thermal_run_osc(priv->adev->handle,
- priv->current_uuid_index,
- enable);
- }
+ priv->current_uuid_index,
+ mode == THERMAL_DEVICE_ENABLED);
+
evaluate_odvp(priv);
@@ -427,8 +404,7 @@
static struct thermal_zone_device_ops int3400_thermal_ops = {
.get_temp = int3400_thermal_get_temp,
- .get_mode = int3400_thermal_get_mode,
- .set_mode = int3400_thermal_set_mode,
+ .change_mode = int3400_thermal_change_mode,
};
static struct thermal_zone_params int3400_thermal_params = {
diff --git a/drivers/thermal/intel/int340x_thermal/int340x_thermal_zone.c b/drivers/thermal/intel/int340x_thermal/int340x_thermal_zone.c
index 4322132..6e479de 100644
--- a/drivers/thermal/intel/int340x_thermal/int340x_thermal_zone.c
+++ b/drivers/thermal/intel/int340x_thermal/int340x_thermal_zone.c
@@ -259,9 +259,14 @@
ret = PTR_ERR(int34x_thermal_zone->zone);
goto err_thermal_zone;
}
+ ret = thermal_zone_device_enable(int34x_thermal_zone->zone);
+ if (ret)
+ goto err_enable;
return int34x_thermal_zone;
+err_enable:
+ thermal_zone_device_unregister(int34x_thermal_zone->zone);
err_thermal_zone:
acpi_lpat_free_conversion_table(int34x_thermal_zone->lpat_table);
kfree(int34x_thermal_zone->aux_trips);
diff --git a/drivers/thermal/intel/int340x_thermal/processor_thermal_device.c b/drivers/thermal/intel/int340x_thermal/processor_thermal_device.c
index 297db1d..81e8b15 100644
--- a/drivers/thermal/intel/int340x_thermal/processor_thermal_device.c
+++ b/drivers/thermal/intel/int340x_thermal/processor_thermal_device.c
@@ -43,7 +43,7 @@
#define PCI_DEVICE_ID_PROC_ICL_THERMAL 0x8a03
/* JasperLake thermal reporting device */
-#define PCI_DEVICE_ID_PROC_JSL_THERMAL 0x4503
+#define PCI_DEVICE_ID_PROC_JSL_THERMAL 0x4E03
/* TigerLake thermal reporting device */
#define PCI_DEVICE_ID_PROC_TGL_THERMAL 0x9A03
diff --git a/drivers/thermal/intel/intel_pch_thermal.c b/drivers/thermal/intel/intel_pch_thermal.c
index 56401fd..3b813eb 100644
--- a/drivers/thermal/intel/intel_pch_thermal.c
+++ b/drivers/thermal/intel/intel_pch_thermal.c
@@ -24,6 +24,7 @@
#define PCH_THERMAL_DID_SKL_H 0xA131 /* Skylake PCH 100 series */
#define PCH_THERMAL_DID_CNL 0x9Df9 /* CNL PCH */
#define PCH_THERMAL_DID_CNL_H 0xA379 /* CNL-H PCH */
+#define PCH_THERMAL_DID_CNL_LP 0x02F9 /* CNL-LP PCH */
#define PCH_THERMAL_DID_CML_H 0X06F9 /* CML-H PCH */
/* Wildcat Point-LP PCH Thermal registers */
@@ -352,9 +353,14 @@
err = PTR_ERR(ptd->tzd);
goto error_cleanup;
}
+ err = thermal_zone_device_enable(ptd->tzd);
+ if (err)
+ goto err_unregister;
return 0;
+err_unregister:
+ thermal_zone_device_unregister(ptd->tzd);
error_cleanup:
iounmap(ptd->hw_base);
error_release:
@@ -405,6 +411,8 @@
.driver_data = board_cnl, },
{ PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCH_THERMAL_DID_CNL_H),
.driver_data = board_cnl, },
+ { PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCH_THERMAL_DID_CNL_LP),
+ .driver_data = board_cnl, },
{ PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCH_THERMAL_DID_CML_H),
.driver_data = board_cml, },
{ 0, },
diff --git a/drivers/thermal/intel/intel_quark_dts_thermal.c b/drivers/thermal/intel/intel_quark_dts_thermal.c
index d704fc1..3eafc6b 100644
--- a/drivers/thermal/intel/intel_quark_dts_thermal.c
+++ b/drivers/thermal/intel/intel_quark_dts_thermal.c
@@ -103,7 +103,6 @@
bool locked;
u32 store_ptps;
u32 store_dts_enable;
- enum thermal_device_mode mode;
struct thermal_zone_device *tzone;
};
@@ -127,10 +126,8 @@
if (ret)
return ret;
- if (out & QRK_DTS_ENABLE_BIT) {
- aux_entry->mode = THERMAL_DEVICE_ENABLED;
+ if (out & QRK_DTS_ENABLE_BIT)
return 0;
- }
if (!aux_entry->locked) {
out |= QRK_DTS_ENABLE_BIT;
@@ -138,10 +135,7 @@
QRK_DTS_REG_OFFSET_ENABLE, out);
if (ret)
return ret;
-
- aux_entry->mode = THERMAL_DEVICE_ENABLED;
} else {
- aux_entry->mode = THERMAL_DEVICE_DISABLED;
pr_info("DTS is locked. Cannot enable DTS\n");
ret = -EPERM;
}
@@ -160,10 +154,8 @@
if (ret)
return ret;
- if (!(out & QRK_DTS_ENABLE_BIT)) {
- aux_entry->mode = THERMAL_DEVICE_DISABLED;
+ if (!(out & QRK_DTS_ENABLE_BIT))
return 0;
- }
if (!aux_entry->locked) {
out &= ~QRK_DTS_ENABLE_BIT;
@@ -172,10 +164,7 @@
if (ret)
return ret;
-
- aux_entry->mode = THERMAL_DEVICE_DISABLED;
} else {
- aux_entry->mode = THERMAL_DEVICE_ENABLED;
pr_info("DTS is locked. Cannot disable DTS\n");
ret = -EPERM;
}
@@ -309,16 +298,8 @@
return 0;
}
-static int sys_get_mode(struct thermal_zone_device *tzd,
- enum thermal_device_mode *mode)
-{
- struct soc_sensor_entry *aux_entry = tzd->devdata;
- *mode = aux_entry->mode;
- return 0;
-}
-
-static int sys_set_mode(struct thermal_zone_device *tzd,
- enum thermal_device_mode mode)
+static int sys_change_mode(struct thermal_zone_device *tzd,
+ enum thermal_device_mode mode)
{
int ret;
@@ -338,8 +319,7 @@
.get_trip_type = sys_get_trip_type,
.set_trip_temp = sys_set_trip_temp,
.get_crit_temp = sys_get_crit_temp,
- .get_mode = sys_get_mode,
- .set_mode = sys_set_mode,
+ .change_mode = sys_change_mode,
};
static void free_soc_dts(struct soc_sensor_entry *aux_entry)
@@ -414,9 +394,7 @@
goto err_ret;
}
- mutex_lock(&dts_update_mutex);
- err = soc_dts_enable(aux_entry->tzone);
- mutex_unlock(&dts_update_mutex);
+ err = thermal_zone_device_enable(aux_entry->tzone);
if (err)
goto err_aux_status;
diff --git a/drivers/thermal/intel/intel_soc_dts_iosf.c b/drivers/thermal/intel/intel_soc_dts_iosf.c
index f75271b..4f1a2f7 100644
--- a/drivers/thermal/intel/intel_soc_dts_iosf.c
+++ b/drivers/thermal/intel/intel_soc_dts_iosf.c
@@ -329,6 +329,9 @@
ret = PTR_ERR(dts->tzone);
goto err_ret;
}
+ ret = thermal_zone_device_enable(dts->tzone);
+ if (ret)
+ goto err_enable;
ret = soc_dts_enable(id);
if (ret)
diff --git a/drivers/thermal/intel/x86_pkg_temp_thermal.c b/drivers/thermal/intel/x86_pkg_temp_thermal.c
index a006b9f..b81c332 100644
--- a/drivers/thermal/intel/x86_pkg_temp_thermal.c
+++ b/drivers/thermal/intel/x86_pkg_temp_thermal.c
@@ -363,6 +363,12 @@
kfree(zonedev);
return err;
}
+ err = thermal_zone_device_enable(zonedev->tzone);
+ if (err) {
+ thermal_zone_device_unregister(zonedev->tzone);
+ kfree(zonedev);
+ return err;
+ }
/* Store MSR value for package thermal interrupt, to restore at exit */
rdmsr(MSR_IA32_PACKAGE_THERM_INTERRUPT, zonedev->msr_pkg_therm_low,
zonedev->msr_pkg_therm_high);
diff --git a/drivers/thermal/kirkwood_thermal.c b/drivers/thermal/kirkwood_thermal.c
index 189b675..7fb6e47 100644
--- a/drivers/thermal/kirkwood_thermal.c
+++ b/drivers/thermal/kirkwood_thermal.c
@@ -65,6 +65,7 @@
struct thermal_zone_device *thermal = NULL;
struct kirkwood_thermal_priv *priv;
struct resource *res;
+ int ret;
priv = devm_kzalloc(&pdev->dev, sizeof(*priv), GFP_KERNEL);
if (!priv)
@@ -82,6 +83,12 @@
"Failed to register thermal zone device\n");
return PTR_ERR(thermal);
}
+ ret = thermal_zone_device_enable(thermal);
+ if (ret) {
+ thermal_zone_device_unregister(thermal);
+ dev_err(&pdev->dev, "Failed to enable thermal zone device\n");
+ return ret;
+ }
platform_set_drvdata(pdev, thermal);
diff --git a/drivers/thermal/mtk_thermal.c b/drivers/thermal/mtk_thermal.c
index 42c9cd0..0bd7aa5 100644
--- a/drivers/thermal/mtk_thermal.c
+++ b/drivers/thermal/mtk_thermal.c
@@ -38,6 +38,7 @@
#define TEMP_MONIDET0 0x014
#define TEMP_MONIDET1 0x018
#define TEMP_MSRCTL0 0x038
+#define TEMP_MSRCTL1 0x03c
#define TEMP_AHBPOLL 0x040
#define TEMP_AHBTO 0x044
#define TEMP_ADCPNP0 0x048
@@ -120,18 +121,32 @@
* MT2701 has 3 sensors and needs 3 VTS calibration data.
* MT2712 has 4 sensors and needs 4 VTS calibration data.
*/
-#define CALIB_BUF0_VALID BIT(0)
-#define CALIB_BUF1_ADC_GE(x) (((x) >> 22) & 0x3ff)
-#define CALIB_BUF0_VTS_TS1(x) (((x) >> 17) & 0x1ff)
-#define CALIB_BUF0_VTS_TS2(x) (((x) >> 8) & 0x1ff)
-#define CALIB_BUF1_VTS_TS3(x) (((x) >> 0) & 0x1ff)
-#define CALIB_BUF2_VTS_TS4(x) (((x) >> 23) & 0x1ff)
-#define CALIB_BUF2_VTS_TS5(x) (((x) >> 5) & 0x1ff)
-#define CALIB_BUF2_VTS_TSABB(x) (((x) >> 14) & 0x1ff)
-#define CALIB_BUF0_DEGC_CALI(x) (((x) >> 1) & 0x3f)
-#define CALIB_BUF0_O_SLOPE(x) (((x) >> 26) & 0x3f)
-#define CALIB_BUF0_O_SLOPE_SIGN(x) (((x) >> 7) & 0x1)
-#define CALIB_BUF1_ID(x) (((x) >> 9) & 0x1)
+#define CALIB_BUF0_VALID_V1 BIT(0)
+#define CALIB_BUF1_ADC_GE_V1(x) (((x) >> 22) & 0x3ff)
+#define CALIB_BUF0_VTS_TS1_V1(x) (((x) >> 17) & 0x1ff)
+#define CALIB_BUF0_VTS_TS2_V1(x) (((x) >> 8) & 0x1ff)
+#define CALIB_BUF1_VTS_TS3_V1(x) (((x) >> 0) & 0x1ff)
+#define CALIB_BUF2_VTS_TS4_V1(x) (((x) >> 23) & 0x1ff)
+#define CALIB_BUF2_VTS_TS5_V1(x) (((x) >> 5) & 0x1ff)
+#define CALIB_BUF2_VTS_TSABB_V1(x) (((x) >> 14) & 0x1ff)
+#define CALIB_BUF0_DEGC_CALI_V1(x) (((x) >> 1) & 0x3f)
+#define CALIB_BUF0_O_SLOPE_V1(x) (((x) >> 26) & 0x3f)
+#define CALIB_BUF0_O_SLOPE_SIGN_V1(x) (((x) >> 7) & 0x1)
+#define CALIB_BUF1_ID_V1(x) (((x) >> 9) & 0x1)
+
+/*
+ * Layout of the fuses providing the calibration data
+ * These macros could be used for MT7622.
+ */
+#define CALIB_BUF0_ADC_OE_V2(x) (((x) >> 22) & 0x3ff)
+#define CALIB_BUF0_ADC_GE_V2(x) (((x) >> 12) & 0x3ff)
+#define CALIB_BUF0_DEGC_CALI_V2(x) (((x) >> 6) & 0x3f)
+#define CALIB_BUF0_O_SLOPE_V2(x) (((x) >> 0) & 0x3f)
+#define CALIB_BUF1_VTS_TS1_V2(x) (((x) >> 23) & 0x1ff)
+#define CALIB_BUF1_VTS_TS2_V2(x) (((x) >> 14) & 0x1ff)
+#define CALIB_BUF1_VTS_TSABB_V2(x) (((x) >> 5) & 0x1ff)
+#define CALIB_BUF1_VALID_V2(x) (((x) >> 4) & 0x1)
+#define CALIB_BUF1_O_SLOPE_SIGN_V2(x) (((x) >> 3) & 0x1)
enum {
VTS1,
@@ -143,6 +158,11 @@
MAX_NUM_VTS,
};
+enum mtk_thermal_version {
+ MTK_THERMAL_V1 = 1,
+ MTK_THERMAL_V2,
+};
+
/* MT2701 thermal sensors */
#define MT2701_TS1 0
#define MT2701_TS2 1
@@ -248,6 +268,7 @@
const int *controller_offset;
bool need_switch_bank;
struct thermal_bank_cfg bank_data[MAX_NUM_ZONES];
+ enum mtk_thermal_version version;
};
struct mtk_thermal {
@@ -261,8 +282,10 @@
/* Calibration values */
s32 adc_ge;
+ s32 adc_oe;
s32 degc_cali;
s32 o_slope;
+ s32 o_slope_sign;
s32 vts[MAX_NUM_VTS];
const struct mtk_thermal_data *conf;
@@ -401,6 +424,7 @@
.msr = mt8173_msr,
.adcpnp = mt8173_adcpnp,
.sensor_mux_values = mt8173_mux_values,
+ .version = MTK_THERMAL_V1,
};
/*
@@ -431,6 +455,7 @@
.msr = mt2701_msr,
.adcpnp = mt2701_adcpnp,
.sensor_mux_values = mt2701_mux_values,
+ .version = MTK_THERMAL_V1,
};
/*
@@ -461,6 +486,7 @@
.msr = mt2712_msr,
.adcpnp = mt2712_adcpnp,
.sensor_mux_values = mt2712_mux_values,
+ .version = MTK_THERMAL_V1,
};
/*
@@ -485,6 +511,7 @@
.msr = mt7622_msr,
.adcpnp = mt7622_adcpnp,
.sensor_mux_values = mt7622_mux_values,
+ .version = MTK_THERMAL_V2,
};
/*
@@ -517,6 +544,7 @@
.msr = mt8183_msr,
.adcpnp = mt8183_adcpnp,
.sensor_mux_values = mt8183_mux_values,
+ .version = MTK_THERMAL_V1,
};
/**
@@ -528,7 +556,7 @@
* This converts the raw ADC value to mcelsius using the SoC specific
* calibration constants
*/
-static int raw_to_mcelsius(struct mtk_thermal *mt, int sensno, s32 raw)
+static int raw_to_mcelsius_v1(struct mtk_thermal *mt, int sensno, s32 raw)
{
s32 tmp;
@@ -543,6 +571,36 @@
return mt->degc_cali * 500 - tmp;
}
+static int raw_to_mcelsius_v2(struct mtk_thermal *mt, int sensno, s32 raw)
+{
+ s32 format_1 = 0;
+ s32 format_2 = 0;
+ s32 g_oe = 1;
+ s32 g_gain = 1;
+ s32 g_x_roomt = 0;
+ s32 tmp = 0;
+
+ if (raw == 0)
+ return 0;
+
+ raw &= 0xfff;
+ g_gain = 10000 + (((mt->adc_ge - 512) * 10000) >> 12);
+ g_oe = mt->adc_oe - 512;
+ format_1 = mt->vts[VTS2] + 3105 - g_oe;
+ format_2 = (mt->degc_cali * 10) >> 1;
+ g_x_roomt = (((format_1 * 10000) >> 12) * 10000) / g_gain;
+
+ tmp = (((((raw - g_oe) * 10000) >> 12) * 10000) / g_gain) - g_x_roomt;
+ tmp = tmp * 10 * 100 / 11;
+
+ if (mt->o_slope_sign == 0)
+ tmp = tmp / (165 - mt->o_slope);
+ else
+ tmp = tmp / (165 + mt->o_slope);
+
+ return (format_2 - tmp) * 100;
+}
+
/**
* mtk_thermal_get_bank - get bank
* @bank: The bank
@@ -596,9 +654,13 @@
for (i = 0; i < conf->bank_data[bank->id].num_sensors; i++) {
raw = readl(mt->thermal_base + conf->msr[i]);
- temp = raw_to_mcelsius(mt,
- conf->bank_data[bank->id].sensors[i],
- raw);
+ if (mt->conf->version == MTK_THERMAL_V1) {
+ temp = raw_to_mcelsius_v1(
+ mt, conf->bank_data[bank->id].sensors[i], raw);
+ } else {
+ temp = raw_to_mcelsius_v2(
+ mt, conf->bank_data[bank->id].sensors[i], raw);
+ }
/*
* The first read of a sensor often contains very high bogus
@@ -700,9 +762,11 @@
writel(auxadc_phys_base + AUXADC_CON1_CLR_V,
controller_base + TEMP_ADCMUXADDR);
- /* AHB address for pnp sensor mux selection */
- writel(apmixed_phys_base + APMIXED_SYS_TS_CON1,
- controller_base + TEMP_PNPMUXADDR);
+ if (mt->conf->version == MTK_THERMAL_V1) {
+ /* AHB address for pnp sensor mux selection */
+ writel(apmixed_phys_base + APMIXED_SYS_TS_CON1,
+ controller_base + TEMP_PNPMUXADDR);
+ }
/* AHB value for auxadc enable */
writel(BIT(conf->auxadc_channel), controller_base + TEMP_ADCEN);
@@ -759,6 +823,68 @@
return of_translate_address(np, regaddr_p);
}
+static int mtk_thermal_extract_efuse_v1(struct mtk_thermal *mt, u32 *buf)
+{
+ int i;
+
+ if (!(buf[0] & CALIB_BUF0_VALID_V1))
+ return -EINVAL;
+
+ mt->adc_ge = CALIB_BUF1_ADC_GE_V1(buf[1]);
+
+ for (i = 0; i < mt->conf->num_sensors; i++) {
+ switch (mt->conf->vts_index[i]) {
+ case VTS1:
+ mt->vts[VTS1] = CALIB_BUF0_VTS_TS1_V1(buf[0]);
+ break;
+ case VTS2:
+ mt->vts[VTS2] = CALIB_BUF0_VTS_TS2_V1(buf[0]);
+ break;
+ case VTS3:
+ mt->vts[VTS3] = CALIB_BUF1_VTS_TS3_V1(buf[1]);
+ break;
+ case VTS4:
+ mt->vts[VTS4] = CALIB_BUF2_VTS_TS4_V1(buf[2]);
+ break;
+ case VTS5:
+ mt->vts[VTS5] = CALIB_BUF2_VTS_TS5_V1(buf[2]);
+ break;
+ case VTSABB:
+ mt->vts[VTSABB] =
+ CALIB_BUF2_VTS_TSABB_V1(buf[2]);
+ break;
+ default:
+ break;
+ }
+ }
+
+ mt->degc_cali = CALIB_BUF0_DEGC_CALI_V1(buf[0]);
+ if (CALIB_BUF1_ID_V1(buf[1]) &
+ CALIB_BUF0_O_SLOPE_SIGN_V1(buf[0]))
+ mt->o_slope = -CALIB_BUF0_O_SLOPE_V1(buf[0]);
+ else
+ mt->o_slope = CALIB_BUF0_O_SLOPE_V1(buf[0]);
+
+ return 0;
+}
+
+static int mtk_thermal_extract_efuse_v2(struct mtk_thermal *mt, u32 *buf)
+{
+ if (!CALIB_BUF1_VALID_V2(buf[1]))
+ return -EINVAL;
+
+ mt->adc_oe = CALIB_BUF0_ADC_OE_V2(buf[0]);
+ mt->adc_ge = CALIB_BUF0_ADC_GE_V2(buf[0]);
+ mt->degc_cali = CALIB_BUF0_DEGC_CALI_V2(buf[0]);
+ mt->o_slope = CALIB_BUF0_O_SLOPE_V2(buf[0]);
+ mt->vts[VTS1] = CALIB_BUF1_VTS_TS1_V2(buf[1]);
+ mt->vts[VTS2] = CALIB_BUF1_VTS_TS2_V2(buf[1]);
+ mt->vts[VTSABB] = CALIB_BUF1_VTS_TSABB_V2(buf[1]);
+ mt->o_slope_sign = CALIB_BUF1_O_SLOPE_SIGN_V2(buf[1]);
+
+ return 0;
+}
+
static int mtk_thermal_get_calibration_data(struct device *dev,
struct mtk_thermal *mt)
{
@@ -794,42 +920,14 @@
goto out;
}
- if (buf[0] & CALIB_BUF0_VALID) {
- mt->adc_ge = CALIB_BUF1_ADC_GE(buf[1]);
+ if (mt->conf->version == MTK_THERMAL_V1)
+ ret = mtk_thermal_extract_efuse_v1(mt, buf);
+ else
+ ret = mtk_thermal_extract_efuse_v2(mt, buf);
- for (i = 0; i < mt->conf->num_sensors; i++) {
- switch (mt->conf->vts_index[i]) {
- case VTS1:
- mt->vts[VTS1] = CALIB_BUF0_VTS_TS1(buf[0]);
- break;
- case VTS2:
- mt->vts[VTS2] = CALIB_BUF0_VTS_TS2(buf[0]);
- break;
- case VTS3:
- mt->vts[VTS3] = CALIB_BUF1_VTS_TS3(buf[1]);
- break;
- case VTS4:
- mt->vts[VTS4] = CALIB_BUF2_VTS_TS4(buf[2]);
- break;
- case VTS5:
- mt->vts[VTS5] = CALIB_BUF2_VTS_TS5(buf[2]);
- break;
- case VTSABB:
- mt->vts[VTSABB] = CALIB_BUF2_VTS_TSABB(buf[2]);
- break;
- default:
- break;
- }
- }
-
- mt->degc_cali = CALIB_BUF0_DEGC_CALI(buf[0]);
- if (CALIB_BUF1_ID(buf[1]) &
- CALIB_BUF0_O_SLOPE_SIGN(buf[0]))
- mt->o_slope = -CALIB_BUF0_O_SLOPE(buf[0]);
- else
- mt->o_slope = CALIB_BUF0_O_SLOPE(buf[0]);
- } else {
+ if (ret) {
dev_info(dev, "Device not calibrated, using default calibration values\n");
+ ret = 0;
}
out:
@@ -863,6 +961,28 @@
};
MODULE_DEVICE_TABLE(of, mtk_thermal_of_match);
+static void mtk_thermal_turn_on_buffer(void __iomem *apmixed_base)
+{
+ int tmp;
+
+ tmp = readl(apmixed_base + APMIXED_SYS_TS_CON1);
+ tmp &= ~(0x37);
+ tmp |= 0x1;
+ writel(tmp, apmixed_base + APMIXED_SYS_TS_CON1);
+ udelay(200);
+}
+
+static void mtk_thermal_release_periodic_ts(struct mtk_thermal *mt,
+ void __iomem *auxadc_base)
+{
+ int tmp;
+
+ writel(0x800, auxadc_base + AUXADC_CON1_SET_V);
+ writel(0x1, mt->thermal_base + TEMP_MONCTL0);
+ tmp = readl(mt->thermal_base + TEMP_MSRCTL1);
+ writel((tmp & (~0x10e)), mt->thermal_base + TEMP_MSRCTL1);
+}
+
static int mtk_thermal_probe(struct platform_device *pdev)
{
int ret, i, ctrl_id;
@@ -871,6 +991,7 @@
struct resource *res;
u64 auxadc_phys_base, apmixed_phys_base;
struct thermal_zone_device *tzdev;
+ void __iomem *apmixed_base, *auxadc_base;
mt = devm_kzalloc(&pdev->dev, sizeof(*mt), GFP_KERNEL);
if (!mt)
@@ -905,6 +1026,7 @@
return -ENODEV;
}
+ auxadc_base = of_iomap(auxadc, 0);
auxadc_phys_base = of_get_phys_base(auxadc);
of_node_put(auxadc);
@@ -920,6 +1042,7 @@
return -ENODEV;
}
+ apmixed_base = of_iomap(apmixedsys, 0);
apmixed_phys_base = of_get_phys_base(apmixedsys);
of_node_put(apmixedsys);
@@ -945,6 +1068,11 @@
goto err_disable_clk_auxadc;
}
+ if (mt->conf->version == MTK_THERMAL_V2) {
+ mtk_thermal_turn_on_buffer(apmixed_base);
+ mtk_thermal_release_periodic_ts(mt, auxadc_base);
+ }
+
for (ctrl_id = 0; ctrl_id < mt->conf->num_controller ; ctrl_id++)
for (i = 0; i < mt->conf->num_banks; i++)
mtk_thermal_init_bank(mt, i, apmixed_phys_base,
diff --git a/drivers/thermal/qcom/tsens-v0_1.c b/drivers/thermal/qcom/tsens-v0_1.c
index 959a937..e64db5f 100644
--- a/drivers/thermal/qcom/tsens-v0_1.c
+++ b/drivers/thermal/qcom/tsens-v0_1.c
@@ -48,6 +48,63 @@
#define MSM8916_CAL_SEL_MASK 0xe0000000
#define MSM8916_CAL_SEL_SHIFT 29
+/* eeprom layout data for 8939 */
+#define MSM8939_BASE0_MASK 0x000000ff
+#define MSM8939_BASE1_MASK 0xff000000
+#define MSM8939_BASE0_SHIFT 0
+#define MSM8939_BASE1_SHIFT 24
+
+#define MSM8939_S0_P1_MASK 0x000001f8
+#define MSM8939_S1_P1_MASK 0x001f8000
+#define MSM8939_S2_P1_MASK_0_4 0xf8000000
+#define MSM8939_S2_P1_MASK_5 0x00000001
+#define MSM8939_S3_P1_MASK 0x00001f80
+#define MSM8939_S4_P1_MASK 0x01f80000
+#define MSM8939_S5_P1_MASK 0x00003f00
+#define MSM8939_S6_P1_MASK 0x03f00000
+#define MSM8939_S7_P1_MASK 0x0000003f
+#define MSM8939_S8_P1_MASK 0x0003f000
+#define MSM8939_S9_P1_MASK 0x07e00000
+
+#define MSM8939_S0_P2_MASK 0x00007e00
+#define MSM8939_S1_P2_MASK 0x07e00000
+#define MSM8939_S2_P2_MASK 0x0000007e
+#define MSM8939_S3_P2_MASK 0x0007e000
+#define MSM8939_S4_P2_MASK 0x7e000000
+#define MSM8939_S5_P2_MASK 0x000fc000
+#define MSM8939_S6_P2_MASK 0xfc000000
+#define MSM8939_S7_P2_MASK 0x00000fc0
+#define MSM8939_S8_P2_MASK 0x00fc0000
+#define MSM8939_S9_P2_MASK_0_4 0xf8000000
+#define MSM8939_S9_P2_MASK_5 0x00002000
+
+#define MSM8939_S0_P1_SHIFT 3
+#define MSM8939_S1_P1_SHIFT 15
+#define MSM8939_S2_P1_SHIFT_0_4 27
+#define MSM8939_S2_P1_SHIFT_5 0
+#define MSM8939_S3_P1_SHIFT 7
+#define MSM8939_S4_P1_SHIFT 19
+#define MSM8939_S5_P1_SHIFT 8
+#define MSM8939_S6_P1_SHIFT 20
+#define MSM8939_S7_P1_SHIFT 0
+#define MSM8939_S8_P1_SHIFT 12
+#define MSM8939_S9_P1_SHIFT 21
+
+#define MSM8939_S0_P2_SHIFT 9
+#define MSM8939_S1_P2_SHIFT 21
+#define MSM8939_S2_P2_SHIFT 1
+#define MSM8939_S3_P2_SHIFT 13
+#define MSM8939_S4_P2_SHIFT 25
+#define MSM8939_S5_P2_SHIFT 14
+#define MSM8939_S6_P2_SHIFT 26
+#define MSM8939_S7_P2_SHIFT 6
+#define MSM8939_S8_P2_SHIFT 18
+#define MSM8939_S9_P2_SHIFT_0_4 27
+#define MSM8939_S9_P2_SHIFT_5 13
+
+#define MSM8939_CAL_SEL_MASK 0x7
+#define MSM8939_CAL_SEL_SHIFT 0
+
/* eeprom layout data for 8974 */
#define BASE1_MASK 0xff
#define S0_P1_MASK 0x3f00
@@ -189,6 +246,76 @@
return 0;
}
+static int calibrate_8939(struct tsens_priv *priv)
+{
+ int base0 = 0, base1 = 0, i;
+ u32 p1[10], p2[10];
+ int mode = 0;
+ u32 *qfprom_cdata;
+ u32 cdata[6];
+
+ qfprom_cdata = (u32 *)qfprom_read(priv->dev, "calib");
+ if (IS_ERR(qfprom_cdata))
+ return PTR_ERR(qfprom_cdata);
+
+ /* Mapping between qfprom nvmem and calibration data */
+ cdata[0] = qfprom_cdata[12];
+ cdata[1] = qfprom_cdata[13];
+ cdata[2] = qfprom_cdata[0];
+ cdata[3] = qfprom_cdata[1];
+ cdata[4] = qfprom_cdata[22];
+ cdata[5] = qfprom_cdata[21];
+
+ mode = (cdata[0] & MSM8939_CAL_SEL_MASK) >> MSM8939_CAL_SEL_SHIFT;
+ dev_dbg(priv->dev, "calibration mode is %d\n", mode);
+
+ switch (mode) {
+ case TWO_PT_CALIB:
+ base1 = (cdata[3] & MSM8939_BASE1_MASK) >> MSM8939_BASE1_SHIFT;
+ p2[0] = (cdata[0] & MSM8939_S0_P2_MASK) >> MSM8939_S0_P2_SHIFT;
+ p2[1] = (cdata[0] & MSM8939_S1_P2_MASK) >> MSM8939_S1_P2_SHIFT;
+ p2[2] = (cdata[1] & MSM8939_S2_P2_MASK) >> MSM8939_S2_P2_SHIFT;
+ p2[3] = (cdata[1] & MSM8939_S3_P2_MASK) >> MSM8939_S3_P2_SHIFT;
+ p2[4] = (cdata[1] & MSM8939_S4_P2_MASK) >> MSM8939_S4_P2_SHIFT;
+ p2[5] = (cdata[2] & MSM8939_S5_P2_MASK) >> MSM8939_S5_P2_SHIFT;
+ p2[6] = (cdata[2] & MSM8939_S6_P2_MASK) >> MSM8939_S6_P2_SHIFT;
+ p2[7] = (cdata[3] & MSM8939_S7_P2_MASK) >> MSM8939_S7_P2_SHIFT;
+ p2[8] = (cdata[3] & MSM8939_S8_P2_MASK) >> MSM8939_S8_P2_SHIFT;
+ p2[9] = (cdata[4] & MSM8939_S9_P2_MASK_0_4) >> MSM8939_S9_P2_SHIFT_0_4;
+ p2[9] |= ((cdata[5] & MSM8939_S9_P2_MASK_5) >> MSM8939_S9_P2_SHIFT_5) << 5;
+ for (i = 0; i < priv->num_sensors; i++)
+ p2[i] = (base1 + p2[i]) << 2;
+ fallthrough;
+ case ONE_PT_CALIB2:
+ base0 = (cdata[2] & MSM8939_BASE0_MASK) >> MSM8939_BASE0_SHIFT;
+ p1[0] = (cdata[0] & MSM8939_S0_P1_MASK) >> MSM8939_S0_P1_SHIFT;
+ p1[1] = (cdata[0] & MSM8939_S1_P1_MASK) >> MSM8939_S1_P1_SHIFT;
+ p1[2] = (cdata[0] & MSM8939_S2_P1_MASK_0_4) >> MSM8939_S2_P1_SHIFT_0_4;
+ p1[2] |= ((cdata[1] & MSM8939_S2_P1_MASK_5) >> MSM8939_S2_P1_SHIFT_5) << 5;
+ p1[3] = (cdata[1] & MSM8939_S3_P1_MASK) >> MSM8939_S3_P1_SHIFT;
+ p1[4] = (cdata[1] & MSM8939_S4_P1_MASK) >> MSM8939_S4_P1_SHIFT;
+ p1[5] = (cdata[2] & MSM8939_S5_P1_MASK) >> MSM8939_S5_P1_SHIFT;
+ p1[6] = (cdata[2] & MSM8939_S6_P1_MASK) >> MSM8939_S6_P1_SHIFT;
+ p1[7] = (cdata[3] & MSM8939_S7_P1_MASK) >> MSM8939_S7_P1_SHIFT;
+ p1[8] = (cdata[3] & MSM8939_S8_P1_MASK) >> MSM8939_S8_P1_SHIFT;
+ p1[9] = (cdata[4] & MSM8939_S9_P1_MASK) >> MSM8939_S9_P1_SHIFT;
+ for (i = 0; i < priv->num_sensors; i++)
+ p1[i] = ((base0) + p1[i]) << 2;
+ break;
+ default:
+ for (i = 0; i < priv->num_sensors; i++) {
+ p1[i] = 500;
+ p2[i] = 780;
+ }
+ break;
+ }
+
+ compute_intercept_slope(priv, p1, p2, mode);
+ kfree(qfprom_cdata);
+
+ return 0;
+}
+
static int calibrate_8974(struct tsens_priv *priv)
{
int base1 = 0, base2 = 0, i;
@@ -325,7 +452,7 @@
return 0;
}
-/* v0.1: 8916, 8974 */
+/* v0.1: 8916, 8939, 8974 */
static struct tsens_features tsens_v0_1_feat = {
.ver_major = VER_0_1,
@@ -386,6 +513,21 @@
.fields = tsens_v0_1_regfields,
};
+static const struct tsens_ops ops_8939 = {
+ .init = init_common,
+ .calibrate = calibrate_8939,
+ .get_temp = get_temp_common,
+};
+
+struct tsens_plat_data data_8939 = {
+ .num_sensors = 10,
+ .ops = &ops_8939,
+ .hw_ids = (unsigned int []){ 0, 1, 2, 4, 5, 6, 7, 8, 9, 10 },
+
+ .feat = &tsens_v0_1_feat,
+ .fields = tsens_v0_1_regfields,
+};
+
static const struct tsens_ops ops_8974 = {
.init = init_common,
.calibrate = calibrate_8974,
diff --git a/drivers/thermal/qcom/tsens.c b/drivers/thermal/qcom/tsens.c
index 39c4462..d8ce3a6 100644
--- a/drivers/thermal/qcom/tsens.c
+++ b/drivers/thermal/qcom/tsens.c
@@ -898,6 +898,9 @@
.compatible = "qcom,msm8916-tsens",
.data = &data_8916,
}, {
+ .compatible = "qcom,msm8939-tsens",
+ .data = &data_8939,
+ }, {
.compatible = "qcom,msm8974-tsens",
.data = &data_8974,
}, {
diff --git a/drivers/thermal/qcom/tsens.h b/drivers/thermal/qcom/tsens.h
index 59d0116..f40b625 100644
--- a/drivers/thermal/qcom/tsens.h
+++ b/drivers/thermal/qcom/tsens.h
@@ -585,7 +585,7 @@
extern struct tsens_plat_data data_8960;
/* TSENS v0.1 targets */
-extern struct tsens_plat_data data_8916, data_8974;
+extern struct tsens_plat_data data_8916, data_8939, data_8974;
/* TSENS v1 targets */
extern struct tsens_plat_data data_tsens_v1, data_8976;
diff --git a/drivers/thermal/rcar_gen3_thermal.c b/drivers/thermal/rcar_gen3_thermal.c
index c48c5e9..0dd47dc 100644
--- a/drivers/thermal/rcar_gen3_thermal.c
+++ b/drivers/thermal/rcar_gen3_thermal.c
@@ -63,7 +63,7 @@
#define TSC_MAX_NUM 3
/* default THCODE values if FUSEs are missing */
-static const int thcode[TSC_MAX_NUM][3] = {
+static const int thcodes[TSC_MAX_NUM][3] = {
{ 3397, 2800, 2221 },
{ 3393, 2795, 2216 },
{ 3389, 2805, 2237 },
@@ -172,7 +172,7 @@
/* Read register and convert to mili Celsius */
reg = rcar_gen3_thermal_read(tsc, REG_GEN3_TEMP) & CTEMP_MASK;
- if (reg <= thcode[tsc->id][1])
+ if (reg <= thcodes[tsc->id][1])
val = FIXPT_DIV(FIXPT_INT(reg) - tsc->coef.b1,
tsc->coef.a1);
else
@@ -315,6 +315,10 @@
.data = &rcar_gen3_ths_tj_1,
},
{
+ .compatible = "renesas,r8a774e1-thermal",
+ .data = &rcar_gen3_ths_tj_1,
+ },
+ {
.compatible = "renesas,r8a7795-thermal",
.data = &rcar_gen3_ths_tj_1,
},
@@ -430,7 +434,7 @@
priv->tscs[i] = tsc;
priv->thermal_init(tsc);
- rcar_gen3_thermal_calc_coefs(tsc, ptat, thcode[i],
+ rcar_gen3_thermal_calc_coefs(tsc, ptat, thcodes[i],
*rcar_gen3_ths_tj_1);
zone = devm_thermal_zone_of_sensor_register(dev, i, tsc,
diff --git a/drivers/thermal/rcar_thermal.c b/drivers/thermal/rcar_thermal.c
index 46aeb28..787710b 100644
--- a/drivers/thermal/rcar_thermal.c
+++ b/drivers/thermal/rcar_thermal.c
@@ -550,12 +550,19 @@
priv->zone = devm_thermal_zone_of_sensor_register(
dev, i, priv,
&rcar_thermal_zone_of_ops);
- else
+ else {
priv->zone = thermal_zone_device_register(
"rcar_thermal",
1, 0, priv,
&rcar_thermal_zone_ops, NULL, 0,
idle);
+
+ ret = thermal_zone_device_enable(priv->zone);
+ if (ret) {
+ thermal_zone_device_unregister(priv->zone);
+ priv->zone = ERR_PTR(ret);
+ }
+ }
if (IS_ERR(priv->zone)) {
dev_err(dev, "can't register thermal zone\n");
ret = PTR_ERR(priv->zone);
diff --git a/drivers/thermal/rockchip_thermal.c b/drivers/thermal/rockchip_thermal.c
index 15a71ec..aa9e0e3 100644
--- a/drivers/thermal/rockchip_thermal.c
+++ b/drivers/thermal/rockchip_thermal.c
@@ -1068,8 +1068,10 @@
{
struct thermal_zone_device *tzd = sensor->tzd;
- tzd->ops->set_mode(tzd,
- on ? THERMAL_DEVICE_ENABLED : THERMAL_DEVICE_DISABLED);
+ if (on)
+ thermal_zone_device_enable(tzd);
+ else
+ thermal_zone_device_disable(tzd);
}
static irqreturn_t rockchip_thermal_alarm_irq_thread(int irq, void *dev)
diff --git a/drivers/thermal/spear_thermal.c b/drivers/thermal/spear_thermal.c
index f68f581..ee33ed6 100644
--- a/drivers/thermal/spear_thermal.c
+++ b/drivers/thermal/spear_thermal.c
@@ -131,6 +131,11 @@
ret = PTR_ERR(spear_thermal);
goto disable_clk;
}
+ ret = thermal_zone_device_enable(spear_thermal);
+ if (ret) {
+ dev_err(&pdev->dev, "Cannot enable thermal zone\n");
+ goto unregister_tzd;
+ }
platform_set_drvdata(pdev, spear_thermal);
@@ -139,6 +144,8 @@
return 0;
+unregister_tzd:
+ thermal_zone_device_unregister(spear_thermal);
disable_clk:
clk_disable(stdev->clk);
diff --git a/drivers/thermal/sprd_thermal.c b/drivers/thermal/sprd_thermal.c
index 4cde70d..3682edb 100644
--- a/drivers/thermal/sprd_thermal.c
+++ b/drivers/thermal/sprd_thermal.c
@@ -322,8 +322,10 @@
{
struct thermal_zone_device *tzd = sen->tzd;
- tzd->ops->set_mode(tzd,
- on ? THERMAL_DEVICE_ENABLED : THERMAL_DEVICE_DISABLED);
+ if (on)
+ thermal_zone_device_enable(tzd);
+ else
+ thermal_zone_device_disable(tzd);
}
static int sprd_thm_probe(struct platform_device *pdev)
diff --git a/drivers/thermal/st/st_thermal.c b/drivers/thermal/st/st_thermal.c
index b928ca6..1276b95 100644
--- a/drivers/thermal/st/st_thermal.c
+++ b/drivers/thermal/st/st_thermal.c
@@ -246,11 +246,16 @@
ret = PTR_ERR(sensor->thermal_dev);
goto sensor_off;
}
+ ret = thermal_zone_device_enable(sensor->thermal_dev);
+ if (ret)
+ goto tzd_unregister;
platform_set_drvdata(pdev, sensor);
return 0;
+tzd_unregister:
+ thermal_zone_device_unregister(sensor->thermal_dev);
sensor_off:
st_thermal_sensor_off(sensor);
diff --git a/drivers/thermal/thermal_core.c b/drivers/thermal/thermal_core.c
index b71196e..72bf159 100644
--- a/drivers/thermal/thermal_core.c
+++ b/drivers/thermal/thermal_core.c
@@ -215,6 +215,8 @@
mutex_unlock(&tz->lock);
mutex_unlock(&thermal_governor_lock);
+ thermal_notify_tz_gov_change(tz->id, policy);
+
return ret;
}
@@ -301,13 +303,22 @@
cancel_delayed_work(&tz->poll_queue);
}
+static inline bool should_stop_polling(struct thermal_zone_device *tz)
+{
+ return !thermal_zone_device_is_enabled(tz);
+}
+
static void monitor_thermal_zone(struct thermal_zone_device *tz)
{
+ bool stop;
+
+ stop = should_stop_polling(tz);
+
mutex_lock(&tz->lock);
- if (tz->passive)
+ if (!stop && tz->passive)
thermal_zone_device_set_polling(tz, tz->passive_delay);
- else if (tz->polling_delay)
+ else if (!stop && tz->polling_delay)
thermal_zone_device_set_polling(tz, tz->polling_delay);
else
thermal_zone_device_set_polling(tz, 0);
@@ -406,12 +417,25 @@
static void handle_thermal_trip(struct thermal_zone_device *tz, int trip)
{
enum thermal_trip_type type;
+ int trip_temp, hyst = 0;
/* Ignore disabled trip points */
if (test_bit(trip, &tz->trips_disabled))
return;
+ tz->ops->get_trip_temp(tz, trip, &trip_temp);
tz->ops->get_trip_type(tz, trip, &type);
+ if (tz->ops->get_trip_hyst)
+ tz->ops->get_trip_hyst(tz, trip, &hyst);
+
+ if (tz->last_temperature != THERMAL_TEMP_INVALID) {
+ if (tz->last_temperature < trip_temp &&
+ tz->temperature >= trip_temp)
+ thermal_notify_tz_trip_up(tz->id, trip);
+ if (tz->last_temperature >= trip_temp &&
+ tz->temperature < (trip_temp - hyst))
+ thermal_notify_tz_trip_down(tz->id, trip);
+ }
if (type == THERMAL_TRIP_CRITICAL || type == THERMAL_TRIP_HOT)
handle_critical_trips(tz, trip, type);
@@ -443,6 +467,8 @@
mutex_unlock(&tz->lock);
trace_thermal_temperature(tz);
+
+ thermal_genl_sampling_temp(tz->id, temp);
}
static void thermal_zone_device_init(struct thermal_zone_device *tz)
@@ -459,11 +485,71 @@
thermal_zone_device_init(tz);
}
+static int thermal_zone_device_set_mode(struct thermal_zone_device *tz,
+ enum thermal_device_mode mode)
+{
+ int ret = 0;
+
+ mutex_lock(&tz->lock);
+
+ /* do nothing if mode isn't changing */
+ if (mode == tz->mode) {
+ mutex_unlock(&tz->lock);
+
+ return ret;
+ }
+
+ if (tz->ops->change_mode)
+ ret = tz->ops->change_mode(tz, mode);
+
+ if (!ret)
+ tz->mode = mode;
+
+ mutex_unlock(&tz->lock);
+
+ thermal_zone_device_update(tz, THERMAL_EVENT_UNSPECIFIED);
+
+ if (mode == THERMAL_DEVICE_ENABLED)
+ thermal_notify_tz_enable(tz->id);
+ else
+ thermal_notify_tz_disable(tz->id);
+
+ return ret;
+}
+
+int thermal_zone_device_enable(struct thermal_zone_device *tz)
+{
+ return thermal_zone_device_set_mode(tz, THERMAL_DEVICE_ENABLED);
+}
+EXPORT_SYMBOL_GPL(thermal_zone_device_enable);
+
+int thermal_zone_device_disable(struct thermal_zone_device *tz)
+{
+ return thermal_zone_device_set_mode(tz, THERMAL_DEVICE_DISABLED);
+}
+EXPORT_SYMBOL_GPL(thermal_zone_device_disable);
+
+int thermal_zone_device_is_enabled(struct thermal_zone_device *tz)
+{
+ enum thermal_device_mode mode;
+
+ mutex_lock(&tz->lock);
+
+ mode = tz->mode;
+
+ mutex_unlock(&tz->lock);
+
+ return mode == THERMAL_DEVICE_ENABLED;
+}
+
void thermal_zone_device_update(struct thermal_zone_device *tz,
enum thermal_notify_event event)
{
int count;
+ if (should_stop_polling(tz))
+ return;
+
if (atomic_read(&in_suspend))
return;
@@ -617,6 +703,73 @@
mutex_unlock(&thermal_list_lock);
}
+int for_each_thermal_governor(int (*cb)(struct thermal_governor *, void *),
+ void *data)
+{
+ struct thermal_governor *gov;
+ int ret = 0;
+
+ mutex_lock(&thermal_governor_lock);
+ list_for_each_entry(gov, &thermal_governor_list, governor_list) {
+ ret = cb(gov, data);
+ if (ret)
+ break;
+ }
+ mutex_unlock(&thermal_governor_lock);
+
+ return ret;
+}
+
+int for_each_thermal_cooling_device(int (*cb)(struct thermal_cooling_device *,
+ void *), void *data)
+{
+ struct thermal_cooling_device *cdev;
+ int ret = 0;
+
+ mutex_lock(&thermal_list_lock);
+ list_for_each_entry(cdev, &thermal_cdev_list, node) {
+ ret = cb(cdev, data);
+ if (ret)
+ break;
+ }
+ mutex_unlock(&thermal_list_lock);
+
+ return ret;
+}
+
+int for_each_thermal_zone(int (*cb)(struct thermal_zone_device *, void *),
+ void *data)
+{
+ struct thermal_zone_device *tz;
+ int ret = 0;
+
+ mutex_lock(&thermal_list_lock);
+ list_for_each_entry(tz, &thermal_tz_list, node) {
+ ret = cb(tz, data);
+ if (ret)
+ break;
+ }
+ mutex_unlock(&thermal_list_lock);
+
+ return ret;
+}
+
+struct thermal_zone_device *thermal_zone_get_by_id(int id)
+{
+ struct thermal_zone_device *tz, *match = NULL;
+
+ mutex_lock(&thermal_list_lock);
+ list_for_each_entry(tz, &thermal_tz_list, node) {
+ if (tz->id == id) {
+ match = tz;
+ break;
+ }
+ }
+ mutex_unlock(&thermal_list_lock);
+
+ return match;
+}
+
void thermal_zone_device_unbind_exception(struct thermal_zone_device *tz,
const char *cdev_type, size_t size)
{
@@ -1340,6 +1493,8 @@
if (atomic_cmpxchg(&tz->need_update, 1, 0))
thermal_zone_device_update(tz, THERMAL_EVENT_UNSPECIFIED);
+ thermal_notify_tz_create(tz->id, tz->type);
+
return tz;
unregister:
@@ -1411,6 +1566,8 @@
ida_destroy(&tz->ida);
mutex_destroy(&tz->lock);
device_unregister(&tz->device);
+
+ thermal_notify_tz_delete(tz->id);
}
EXPORT_SYMBOL_GPL(thermal_zone_device_unregister);
@@ -1456,7 +1613,6 @@
unsigned long mode, void *_unused)
{
struct thermal_zone_device *tz;
- enum thermal_device_mode tz_mode;
switch (mode) {
case PM_HIBERNATION_PREPARE:
@@ -1469,11 +1625,7 @@
case PM_POST_SUSPEND:
atomic_set(&in_suspend, 0);
list_for_each_entry(tz, &thermal_tz_list, node) {
- tz_mode = THERMAL_DEVICE_ENABLED;
- if (tz->ops->get_mode)
- tz->ops->get_mode(tz, &tz_mode);
-
- if (tz_mode == THERMAL_DEVICE_DISABLED)
+ if (!thermal_zone_device_is_enabled(tz))
continue;
thermal_zone_device_init(tz);
@@ -1495,6 +1647,10 @@
{
int result;
+ result = thermal_netlink_init();
+ if (result)
+ goto error;
+
mutex_init(&poweroff_lock);
result = thermal_register_governors();
if (result)
@@ -1527,4 +1683,4 @@
mutex_destroy(&poweroff_lock);
return result;
}
-core_initcall(thermal_init);
+postcore_initcall(thermal_init);
diff --git a/drivers/thermal/thermal_core.h b/drivers/thermal/thermal_core.h
index c956895..e00fc55 100644
--- a/drivers/thermal/thermal_core.h
+++ b/drivers/thermal/thermal_core.h
@@ -12,6 +12,8 @@
#include <linux/device.h>
#include <linux/thermal.h>
+#include "thermal_netlink.h"
+
/* Default Thermal Governor */
#if defined(CONFIG_THERMAL_DEFAULT_GOV_STEP_WISE)
#define DEFAULT_THERMAL_GOVERNOR "step_wise"
@@ -41,6 +43,17 @@
__governor < __governor_thermal_table_end; \
__governor++)
+int for_each_thermal_zone(int (*cb)(struct thermal_zone_device *, void *),
+ void *);
+
+int for_each_thermal_cooling_device(int (*cb)(struct thermal_cooling_device *,
+ void *), void *);
+
+int for_each_thermal_governor(int (*cb)(struct thermal_governor *, void *),
+ void *thermal_governor);
+
+struct thermal_zone_device *thermal_zone_get_by_id(int id);
+
struct thermal_attr {
struct device_attribute attr;
char name[THERMAL_NAME_LENGTH];
@@ -166,4 +179,6 @@
}
#endif
+int thermal_zone_device_is_enabled(struct thermal_zone_device *tz);
+
#endif /* __THERMAL_CORE_H__ */
diff --git a/drivers/thermal/thermal_helpers.c b/drivers/thermal/thermal_helpers.c
index 87b1256..c94bc82 100644
--- a/drivers/thermal/thermal_helpers.c
+++ b/drivers/thermal/thermal_helpers.c
@@ -175,6 +175,16 @@
mutex_unlock(&tz->lock);
}
+static void thermal_cdev_set_cur_state(struct thermal_cooling_device *cdev,
+ int target)
+{
+ if (cdev->ops->set_cur_state(cdev, target))
+ return;
+
+ thermal_notify_cdev_state_update(cdev->id, target);
+ thermal_cooling_device_stats_update(cdev, target);
+}
+
void thermal_cdev_update(struct thermal_cooling_device *cdev)
{
struct thermal_instance *instance;
@@ -197,8 +207,7 @@
target = instance->target;
}
- if (!cdev->ops->set_cur_state(cdev, target))
- thermal_cooling_device_stats_update(cdev, target);
+ thermal_cdev_set_cur_state(cdev, target);
cdev->updated = true;
mutex_unlock(&cdev->lock);
diff --git a/drivers/thermal/thermal_netlink.c b/drivers/thermal/thermal_netlink.c
new file mode 100644
index 0000000..af7b238
--- /dev/null
+++ b/drivers/thermal/thermal_netlink.c
@@ -0,0 +1,647 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright 2020 Linaro Limited
+ *
+ * Author: Daniel Lezcano <daniel.lezcano@linaro.org>
+ *
+ * Generic netlink for thermal management framework
+ */
+#include <linux/module.h>
+#include <linux/kernel.h>
+#include <net/genetlink.h>
+#include <uapi/linux/thermal.h>
+
+#include "thermal_core.h"
+
+static const struct genl_multicast_group thermal_genl_mcgrps[] = {
+ { .name = THERMAL_GENL_SAMPLING_GROUP_NAME, },
+ { .name = THERMAL_GENL_EVENT_GROUP_NAME, },
+};
+
+static const struct nla_policy thermal_genl_policy[THERMAL_GENL_ATTR_MAX + 1] = {
+ /* Thermal zone */
+ [THERMAL_GENL_ATTR_TZ] = { .type = NLA_NESTED },
+ [THERMAL_GENL_ATTR_TZ_ID] = { .type = NLA_U32 },
+ [THERMAL_GENL_ATTR_TZ_TEMP] = { .type = NLA_U32 },
+ [THERMAL_GENL_ATTR_TZ_TRIP] = { .type = NLA_NESTED },
+ [THERMAL_GENL_ATTR_TZ_TRIP_ID] = { .type = NLA_U32 },
+ [THERMAL_GENL_ATTR_TZ_TRIP_TEMP] = { .type = NLA_U32 },
+ [THERMAL_GENL_ATTR_TZ_TRIP_TYPE] = { .type = NLA_U32 },
+ [THERMAL_GENL_ATTR_TZ_TRIP_HYST] = { .type = NLA_U32 },
+ [THERMAL_GENL_ATTR_TZ_MODE] = { .type = NLA_U32 },
+ [THERMAL_GENL_ATTR_TZ_CDEV_WEIGHT] = { .type = NLA_U32 },
+ [THERMAL_GENL_ATTR_TZ_NAME] = { .type = NLA_STRING,
+ .len = THERMAL_NAME_LENGTH },
+ /* Governor(s) */
+ [THERMAL_GENL_ATTR_TZ_GOV] = { .type = NLA_NESTED },
+ [THERMAL_GENL_ATTR_TZ_GOV_NAME] = { .type = NLA_STRING,
+ .len = THERMAL_NAME_LENGTH },
+ /* Cooling devices */
+ [THERMAL_GENL_ATTR_CDEV] = { .type = NLA_NESTED },
+ [THERMAL_GENL_ATTR_CDEV_ID] = { .type = NLA_U32 },
+ [THERMAL_GENL_ATTR_CDEV_CUR_STATE] = { .type = NLA_U32 },
+ [THERMAL_GENL_ATTR_CDEV_MAX_STATE] = { .type = NLA_U32 },
+ [THERMAL_GENL_ATTR_CDEV_NAME] = { .type = NLA_STRING,
+ .len = THERMAL_NAME_LENGTH },
+};
+
+struct param {
+ struct nlattr **attrs;
+ struct sk_buff *msg;
+ const char *name;
+ int tz_id;
+ int cdev_id;
+ int trip_id;
+ int trip_temp;
+ int trip_type;
+ int trip_hyst;
+ int temp;
+ int cdev_state;
+ int cdev_max_state;
+};
+
+typedef int (*cb_t)(struct param *);
+
+static struct genl_family thermal_gnl_family;
+
+/************************** Sampling encoding *******************************/
+
+int thermal_genl_sampling_temp(int id, int temp)
+{
+ struct sk_buff *skb;
+ void *hdr;
+
+ skb = genlmsg_new(NLMSG_GOODSIZE, GFP_KERNEL);
+ if (!skb)
+ return -ENOMEM;
+
+ hdr = genlmsg_put(skb, 0, 0, &thermal_gnl_family, 0,
+ THERMAL_GENL_SAMPLING_TEMP);
+ if (!hdr)
+ return -EMSGSIZE;
+
+ if (nla_put_u32(skb, THERMAL_GENL_ATTR_TZ_ID, id))
+ goto out_cancel;
+
+ if (nla_put_u32(skb, THERMAL_GENL_ATTR_TZ_TEMP, temp))
+ goto out_cancel;
+
+ genlmsg_end(skb, hdr);
+
+ genlmsg_multicast(&thermal_gnl_family, skb, 0, 0, GFP_KERNEL);
+
+ return 0;
+out_cancel:
+ genlmsg_cancel(skb, hdr);
+ nlmsg_free(skb);
+
+ return -EMSGSIZE;
+}
+
+/**************************** Event encoding *********************************/
+
+static int thermal_genl_event_tz_create(struct param *p)
+{
+ if (nla_put_u32(p->msg, THERMAL_GENL_ATTR_TZ_ID, p->tz_id) ||
+ nla_put_string(p->msg, THERMAL_GENL_ATTR_TZ_NAME, p->name))
+ return -EMSGSIZE;
+
+ return 0;
+}
+
+static int thermal_genl_event_tz(struct param *p)
+{
+ if (nla_put_u32(p->msg, THERMAL_GENL_ATTR_TZ_ID, p->tz_id))
+ return -EMSGSIZE;
+
+ return 0;
+}
+
+static int thermal_genl_event_tz_trip_up(struct param *p)
+{
+ if (nla_put_u32(p->msg, THERMAL_GENL_ATTR_TZ_ID, p->tz_id) ||
+ nla_put_u32(p->msg, THERMAL_GENL_ATTR_TZ_TRIP_ID, p->trip_id))
+ return -EMSGSIZE;
+
+ return 0;
+}
+
+static int thermal_genl_event_tz_trip_add(struct param *p)
+{
+ if (nla_put_u32(p->msg, THERMAL_GENL_ATTR_TZ_ID, p->tz_id) ||
+ nla_put_u32(p->msg, THERMAL_GENL_ATTR_TZ_TRIP_ID, p->trip_id) ||
+ nla_put_u32(p->msg, THERMAL_GENL_ATTR_TZ_TRIP_TYPE, p->trip_type) ||
+ nla_put_u32(p->msg, THERMAL_GENL_ATTR_TZ_TRIP_TEMP, p->trip_temp) ||
+ nla_put_u32(p->msg, THERMAL_GENL_ATTR_TZ_TRIP_HYST, p->trip_hyst))
+ return -EMSGSIZE;
+
+ return 0;
+}
+
+static int thermal_genl_event_tz_trip_delete(struct param *p)
+{
+ if (nla_put_u32(p->msg, THERMAL_GENL_ATTR_TZ_ID, p->tz_id) ||
+ nla_put_u32(p->msg, THERMAL_GENL_ATTR_TZ_TRIP_ID, p->trip_id))
+ return -EMSGSIZE;
+
+ return 0;
+}
+
+static int thermal_genl_event_cdev_add(struct param *p)
+{
+ if (nla_put_string(p->msg, THERMAL_GENL_ATTR_CDEV_NAME,
+ p->name) ||
+ nla_put_u32(p->msg, THERMAL_GENL_ATTR_CDEV_ID,
+ p->cdev_id) ||
+ nla_put_u32(p->msg, THERMAL_GENL_ATTR_CDEV_MAX_STATE,
+ p->cdev_max_state))
+ return -EMSGSIZE;
+
+ return 0;
+}
+
+static int thermal_genl_event_cdev_delete(struct param *p)
+{
+ if (nla_put_u32(p->msg, THERMAL_GENL_ATTR_CDEV_ID, p->cdev_id))
+ return -EMSGSIZE;
+
+ return 0;
+}
+
+static int thermal_genl_event_cdev_state_update(struct param *p)
+{
+ if (nla_put_u32(p->msg, THERMAL_GENL_ATTR_CDEV_ID,
+ p->cdev_id) ||
+ nla_put_u32(p->msg, THERMAL_GENL_ATTR_CDEV_CUR_STATE,
+ p->cdev_state))
+ return -EMSGSIZE;
+
+ return 0;
+}
+
+static int thermal_genl_event_gov_change(struct param *p)
+{
+ if (nla_put_u32(p->msg, THERMAL_GENL_ATTR_TZ_ID, p->tz_id) ||
+ nla_put_string(p->msg, THERMAL_GENL_ATTR_GOV_NAME, p->name))
+ return -EMSGSIZE;
+
+ return 0;
+}
+
+int thermal_genl_event_tz_delete(struct param *p)
+ __attribute__((alias("thermal_genl_event_tz")));
+
+int thermal_genl_event_tz_enable(struct param *p)
+ __attribute__((alias("thermal_genl_event_tz")));
+
+int thermal_genl_event_tz_disable(struct param *p)
+ __attribute__((alias("thermal_genl_event_tz")));
+
+int thermal_genl_event_tz_trip_down(struct param *p)
+ __attribute__((alias("thermal_genl_event_tz_trip_up")));
+
+int thermal_genl_event_tz_trip_change(struct param *p)
+ __attribute__((alias("thermal_genl_event_tz_trip_add")));
+
+static cb_t event_cb[] = {
+ [THERMAL_GENL_EVENT_TZ_CREATE] = thermal_genl_event_tz_create,
+ [THERMAL_GENL_EVENT_TZ_DELETE] = thermal_genl_event_tz_delete,
+ [THERMAL_GENL_EVENT_TZ_ENABLE] = thermal_genl_event_tz_enable,
+ [THERMAL_GENL_EVENT_TZ_DISABLE] = thermal_genl_event_tz_disable,
+ [THERMAL_GENL_EVENT_TZ_TRIP_UP] = thermal_genl_event_tz_trip_up,
+ [THERMAL_GENL_EVENT_TZ_TRIP_DOWN] = thermal_genl_event_tz_trip_down,
+ [THERMAL_GENL_EVENT_TZ_TRIP_CHANGE] = thermal_genl_event_tz_trip_change,
+ [THERMAL_GENL_EVENT_TZ_TRIP_ADD] = thermal_genl_event_tz_trip_add,
+ [THERMAL_GENL_EVENT_TZ_TRIP_DELETE] = thermal_genl_event_tz_trip_delete,
+ [THERMAL_GENL_EVENT_CDEV_ADD] = thermal_genl_event_cdev_add,
+ [THERMAL_GENL_EVENT_CDEV_DELETE] = thermal_genl_event_cdev_delete,
+ [THERMAL_GENL_EVENT_CDEV_STATE_UPDATE] = thermal_genl_event_cdev_state_update,
+ [THERMAL_GENL_EVENT_TZ_GOV_CHANGE] = thermal_genl_event_gov_change,
+};
+
+/*
+ * Generic netlink event encoding
+ */
+static int thermal_genl_send_event(enum thermal_genl_event event,
+ struct param *p)
+{
+ struct sk_buff *msg;
+ int ret = -EMSGSIZE;
+ void *hdr;
+
+ msg = genlmsg_new(NLMSG_GOODSIZE, GFP_KERNEL);
+ if (!msg)
+ return -ENOMEM;
+ p->msg = msg;
+
+ hdr = genlmsg_put(msg, 0, 0, &thermal_gnl_family, 0, event);
+ if (!hdr)
+ goto out_free_msg;
+
+ ret = event_cb[event](p);
+ if (ret)
+ goto out_cancel_msg;
+
+ genlmsg_end(msg, hdr);
+
+ genlmsg_multicast(&thermal_gnl_family, msg, 0, 1, GFP_KERNEL);
+
+ return 0;
+
+out_cancel_msg:
+ genlmsg_cancel(msg, hdr);
+out_free_msg:
+ nlmsg_free(msg);
+
+ return ret;
+}
+
+int thermal_notify_tz_create(int tz_id, const char *name)
+{
+ struct param p = { .tz_id = tz_id, .name = name };
+
+ return thermal_genl_send_event(THERMAL_GENL_EVENT_TZ_CREATE, &p);
+}
+
+int thermal_notify_tz_delete(int tz_id)
+{
+ struct param p = { .tz_id = tz_id };
+
+ return thermal_genl_send_event(THERMAL_GENL_EVENT_TZ_DELETE, &p);
+}
+
+int thermal_notify_tz_enable(int tz_id)
+{
+ struct param p = { .tz_id = tz_id };
+
+ return thermal_genl_send_event(THERMAL_GENL_EVENT_TZ_ENABLE, &p);
+}
+
+int thermal_notify_tz_disable(int tz_id)
+{
+ struct param p = { .tz_id = tz_id };
+
+ return thermal_genl_send_event(THERMAL_GENL_EVENT_TZ_DISABLE, &p);
+}
+
+int thermal_notify_tz_trip_down(int tz_id, int trip_id)
+{
+ struct param p = { .tz_id = tz_id, .trip_id = trip_id };
+
+ return thermal_genl_send_event(THERMAL_GENL_EVENT_TZ_TRIP_DOWN, &p);
+}
+
+int thermal_notify_tz_trip_up(int tz_id, int trip_id)
+{
+ struct param p = { .tz_id = tz_id, .trip_id = trip_id };
+
+ return thermal_genl_send_event(THERMAL_GENL_EVENT_TZ_TRIP_UP, &p);
+}
+
+int thermal_notify_tz_trip_add(int tz_id, int trip_id, int trip_type,
+ int trip_temp, int trip_hyst)
+{
+ struct param p = { .tz_id = tz_id, .trip_id = trip_id,
+ .trip_type = trip_type, .trip_temp = trip_temp,
+ .trip_hyst = trip_hyst };
+
+ return thermal_genl_send_event(THERMAL_GENL_EVENT_TZ_TRIP_ADD, &p);
+}
+
+int thermal_notify_tz_trip_delete(int tz_id, int trip_id)
+{
+ struct param p = { .tz_id = tz_id, .trip_id = trip_id };
+
+ return thermal_genl_send_event(THERMAL_GENL_EVENT_TZ_TRIP_DELETE, &p);
+}
+
+int thermal_notify_tz_trip_change(int tz_id, int trip_id, int trip_type,
+ int trip_temp, int trip_hyst)
+{
+ struct param p = { .tz_id = tz_id, .trip_id = trip_id,
+ .trip_type = trip_type, .trip_temp = trip_temp,
+ .trip_hyst = trip_hyst };
+
+ return thermal_genl_send_event(THERMAL_GENL_EVENT_TZ_TRIP_CHANGE, &p);
+}
+
+int thermal_notify_cdev_state_update(int cdev_id, int cdev_state)
+{
+ struct param p = { .cdev_id = cdev_id, .cdev_state = cdev_state };
+
+ return thermal_genl_send_event(THERMAL_GENL_EVENT_CDEV_STATE_UPDATE, &p);
+}
+
+int thermal_notify_cdev_add(int cdev_id, const char *name, int cdev_max_state)
+{
+ struct param p = { .cdev_id = cdev_id, .name = name,
+ .cdev_max_state = cdev_max_state };
+
+ return thermal_genl_send_event(THERMAL_GENL_EVENT_CDEV_ADD, &p);
+}
+
+int thermal_notify_cdev_delete(int cdev_id)
+{
+ struct param p = { .cdev_id = cdev_id };
+
+ return thermal_genl_send_event(THERMAL_GENL_EVENT_CDEV_DELETE, &p);
+}
+
+int thermal_notify_tz_gov_change(int tz_id, const char *name)
+{
+ struct param p = { .tz_id = tz_id, .name = name };
+
+ return thermal_genl_send_event(THERMAL_GENL_EVENT_TZ_GOV_CHANGE, &p);
+}
+
+/*************************** Command encoding ********************************/
+
+static int __thermal_genl_cmd_tz_get_id(struct thermal_zone_device *tz,
+ void *data)
+{
+ struct sk_buff *msg = data;
+
+ if (nla_put_u32(msg, THERMAL_GENL_ATTR_TZ_ID, tz->id) ||
+ nla_put_string(msg, THERMAL_GENL_ATTR_TZ_NAME, tz->type))
+ return -EMSGSIZE;
+
+ return 0;
+}
+
+static int thermal_genl_cmd_tz_get_id(struct param *p)
+{
+ struct sk_buff *msg = p->msg;
+ struct nlattr *start_tz;
+ int ret;
+
+ start_tz = nla_nest_start(msg, THERMAL_GENL_ATTR_TZ);
+ if (!start_tz)
+ return -EMSGSIZE;
+
+ ret = for_each_thermal_zone(__thermal_genl_cmd_tz_get_id, msg);
+ if (ret)
+ goto out_cancel_nest;
+
+ nla_nest_end(msg, start_tz);
+
+ return 0;
+
+out_cancel_nest:
+ nla_nest_cancel(msg, start_tz);
+
+ return ret;
+}
+
+static int thermal_genl_cmd_tz_get_trip(struct param *p)
+{
+ struct sk_buff *msg = p->msg;
+ struct thermal_zone_device *tz;
+ struct nlattr *start_trip;
+ int i, id;
+
+ if (!p->attrs[THERMAL_GENL_ATTR_TZ_ID])
+ return -EINVAL;
+
+ id = nla_get_u32(p->attrs[THERMAL_GENL_ATTR_TZ_ID]);
+
+ tz = thermal_zone_get_by_id(id);
+ if (!tz)
+ return -EINVAL;
+
+ start_trip = nla_nest_start(msg, THERMAL_GENL_ATTR_TZ_TRIP);
+ if (!start_trip)
+ return -EMSGSIZE;
+
+ mutex_lock(&tz->lock);
+
+ for (i = 0; i < tz->trips; i++) {
+
+ enum thermal_trip_type type;
+ int temp, hyst;
+
+ tz->ops->get_trip_type(tz, i, &type);
+ tz->ops->get_trip_temp(tz, i, &temp);
+ tz->ops->get_trip_hyst(tz, i, &hyst);
+
+ if (nla_put_u32(msg, THERMAL_GENL_ATTR_TZ_TRIP_ID, i) ||
+ nla_put_u32(msg, THERMAL_GENL_ATTR_TZ_TRIP_TYPE, type) ||
+ nla_put_u32(msg, THERMAL_GENL_ATTR_TZ_TRIP_TEMP, temp) ||
+ nla_put_u32(msg, THERMAL_GENL_ATTR_TZ_TRIP_HYST, hyst))
+ goto out_cancel_nest;
+ }
+
+ mutex_unlock(&tz->lock);
+
+ nla_nest_end(msg, start_trip);
+
+ return 0;
+
+out_cancel_nest:
+ mutex_unlock(&tz->lock);
+
+ return -EMSGSIZE;
+}
+
+static int thermal_genl_cmd_tz_get_temp(struct param *p)
+{
+ struct sk_buff *msg = p->msg;
+ struct thermal_zone_device *tz;
+ int temp, ret, id;
+
+ if (!p->attrs[THERMAL_GENL_ATTR_TZ_ID])
+ return -EINVAL;
+
+ id = nla_get_u32(p->attrs[THERMAL_GENL_ATTR_TZ_ID]);
+
+ tz = thermal_zone_get_by_id(id);
+ if (!tz)
+ return -EINVAL;
+
+ ret = thermal_zone_get_temp(tz, &temp);
+ if (ret)
+ return ret;
+
+ if (nla_put_u32(msg, THERMAL_GENL_ATTR_TZ_ID, id) ||
+ nla_put_u32(msg, THERMAL_GENL_ATTR_TZ_TEMP, temp))
+ return -EMSGSIZE;
+
+ return 0;
+}
+
+static int thermal_genl_cmd_tz_get_gov(struct param *p)
+{
+ struct sk_buff *msg = p->msg;
+ struct thermal_zone_device *tz;
+ int id, ret = 0;
+
+ if (!p->attrs[THERMAL_GENL_ATTR_TZ_ID])
+ return -EINVAL;
+
+ id = nla_get_u32(p->attrs[THERMAL_GENL_ATTR_TZ_ID]);
+
+ tz = thermal_zone_get_by_id(id);
+ if (!tz)
+ return -EINVAL;
+
+ mutex_lock(&tz->lock);
+
+ if (nla_put_u32(msg, THERMAL_GENL_ATTR_TZ_ID, id) ||
+ nla_put_string(msg, THERMAL_GENL_ATTR_TZ_GOV_NAME,
+ tz->governor->name))
+ ret = -EMSGSIZE;
+
+ mutex_unlock(&tz->lock);
+
+ return ret;
+}
+
+static int __thermal_genl_cmd_cdev_get(struct thermal_cooling_device *cdev,
+ void *data)
+{
+ struct sk_buff *msg = data;
+
+ if (nla_put_u32(msg, THERMAL_GENL_ATTR_CDEV_ID, cdev->id))
+ return -EMSGSIZE;
+
+ if (nla_put_string(msg, THERMAL_GENL_ATTR_CDEV_NAME, cdev->type))
+ return -EMSGSIZE;
+
+ return 0;
+}
+
+static int thermal_genl_cmd_cdev_get(struct param *p)
+{
+ struct sk_buff *msg = p->msg;
+ struct nlattr *start_cdev;
+ int ret;
+
+ start_cdev = nla_nest_start(msg, THERMAL_GENL_ATTR_CDEV);
+ if (!start_cdev)
+ return -EMSGSIZE;
+
+ ret = for_each_thermal_cooling_device(__thermal_genl_cmd_cdev_get, msg);
+ if (ret)
+ goto out_cancel_nest;
+
+ nla_nest_end(msg, start_cdev);
+
+ return 0;
+out_cancel_nest:
+ nla_nest_cancel(msg, start_cdev);
+
+ return ret;
+}
+
+static cb_t cmd_cb[] = {
+ [THERMAL_GENL_CMD_TZ_GET_ID] = thermal_genl_cmd_tz_get_id,
+ [THERMAL_GENL_CMD_TZ_GET_TRIP] = thermal_genl_cmd_tz_get_trip,
+ [THERMAL_GENL_CMD_TZ_GET_TEMP] = thermal_genl_cmd_tz_get_temp,
+ [THERMAL_GENL_CMD_TZ_GET_GOV] = thermal_genl_cmd_tz_get_gov,
+ [THERMAL_GENL_CMD_CDEV_GET] = thermal_genl_cmd_cdev_get,
+};
+
+static int thermal_genl_cmd_dumpit(struct sk_buff *skb,
+ struct netlink_callback *cb)
+{
+ struct param p = { .msg = skb };
+ const struct genl_dumpit_info *info = genl_dumpit_info(cb);
+ int cmd = info->ops->cmd;
+ int ret;
+ void *hdr;
+
+ hdr = genlmsg_put(skb, 0, 0, &thermal_gnl_family, 0, cmd);
+ if (!hdr)
+ return -EMSGSIZE;
+
+ ret = cmd_cb[cmd](&p);
+ if (ret)
+ goto out_cancel_msg;
+
+ genlmsg_end(skb, hdr);
+
+ return 0;
+
+out_cancel_msg:
+ genlmsg_cancel(skb, hdr);
+
+ return ret;
+}
+
+static int thermal_genl_cmd_doit(struct sk_buff *skb,
+ struct genl_info *info)
+{
+ struct param p = { .attrs = info->attrs };
+ struct sk_buff *msg;
+ void *hdr;
+ int cmd = info->genlhdr->cmd;
+ int ret = -EMSGSIZE;
+
+ msg = genlmsg_new(NLMSG_GOODSIZE, GFP_KERNEL);
+ if (!msg)
+ return -ENOMEM;
+ p.msg = msg;
+
+ hdr = genlmsg_put_reply(msg, info, &thermal_gnl_family, 0, cmd);
+ if (!hdr)
+ goto out_free_msg;
+
+ ret = cmd_cb[cmd](&p);
+ if (ret)
+ goto out_cancel_msg;
+
+ genlmsg_end(msg, hdr);
+
+ return genlmsg_reply(msg, info);
+
+out_cancel_msg:
+ genlmsg_cancel(msg, hdr);
+out_free_msg:
+ nlmsg_free(msg);
+
+ return ret;
+}
+
+static const struct genl_ops thermal_genl_ops[] = {
+ {
+ .cmd = THERMAL_GENL_CMD_TZ_GET_ID,
+ .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
+ .dumpit = thermal_genl_cmd_dumpit,
+ },
+ {
+ .cmd = THERMAL_GENL_CMD_TZ_GET_TRIP,
+ .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
+ .doit = thermal_genl_cmd_doit,
+ },
+ {
+ .cmd = THERMAL_GENL_CMD_TZ_GET_TEMP,
+ .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
+ .doit = thermal_genl_cmd_doit,
+ },
+ {
+ .cmd = THERMAL_GENL_CMD_TZ_GET_GOV,
+ .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
+ .doit = thermal_genl_cmd_doit,
+ },
+ {
+ .cmd = THERMAL_GENL_CMD_CDEV_GET,
+ .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
+ .dumpit = thermal_genl_cmd_dumpit,
+ },
+};
+
+static struct genl_family thermal_gnl_family __ro_after_init = {
+ .hdrsize = 0,
+ .name = THERMAL_GENL_FAMILY_NAME,
+ .version = THERMAL_GENL_VERSION,
+ .maxattr = THERMAL_GENL_ATTR_MAX,
+ .policy = thermal_genl_policy,
+ .ops = thermal_genl_ops,
+ .n_ops = ARRAY_SIZE(thermal_genl_ops),
+ .mcgrps = thermal_genl_mcgrps,
+ .n_mcgrps = ARRAY_SIZE(thermal_genl_mcgrps),
+};
+
+int __init thermal_netlink_init(void)
+{
+ return genl_register_family(&thermal_gnl_family);
+}
diff --git a/drivers/thermal/thermal_netlink.h b/drivers/thermal/thermal_netlink.h
new file mode 100644
index 0000000..828d1dd
--- /dev/null
+++ b/drivers/thermal/thermal_netlink.h
@@ -0,0 +1,104 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (C) Linaro Ltd 2020
+ * Author: Daniel Lezcano <daniel.lezcano@linaro.org>
+ */
+
+/* Netlink notification function */
+#ifdef CONFIG_THERMAL_NETLINK
+int __init thermal_netlink_init(void);
+int thermal_notify_tz_create(int tz_id, const char *name);
+int thermal_notify_tz_delete(int tz_id);
+int thermal_notify_tz_enable(int tz_id);
+int thermal_notify_tz_disable(int tz_id);
+int thermal_notify_tz_trip_down(int tz_id, int id);
+int thermal_notify_tz_trip_up(int tz_id, int id);
+int thermal_notify_tz_trip_delete(int tz_id, int id);
+int thermal_notify_tz_trip_add(int tz_id, int id, int type,
+ int temp, int hyst);
+int thermal_notify_tz_trip_change(int tz_id, int id, int type,
+ int temp, int hyst);
+int thermal_notify_cdev_state_update(int cdev_id, int state);
+int thermal_notify_cdev_add(int cdev_id, const char *name, int max_state);
+int thermal_notify_cdev_delete(int cdev_id);
+int thermal_notify_tz_gov_change(int tz_id, const char *name);
+int thermal_genl_sampling_temp(int id, int temp);
+#else
+static inline int thermal_netlink_init(void)
+{
+ return 0;
+}
+
+static inline int thermal_notify_tz_create(int tz_id, const char *name)
+{
+ return 0;
+}
+
+static inline int thermal_notify_tz_delete(int tz_id)
+{
+ return 0;
+}
+
+static inline int thermal_notify_tz_enable(int tz_id)
+{
+ return 0;
+}
+
+static inline int thermal_notify_tz_disable(int tz_id)
+{
+ return 0;
+}
+
+static inline int thermal_notify_tz_trip_down(int tz_id, int id)
+{
+ return 0;
+}
+
+static inline int thermal_notify_tz_trip_up(int tz_id, int id)
+{
+ return 0;
+}
+
+static inline int thermal_notify_tz_trip_delete(int tz_id, int id)
+{
+ return 0;
+}
+
+static inline int thermal_notify_tz_trip_add(int tz_id, int id, int type,
+ int temp, int hyst)
+{
+ return 0;
+}
+
+static inline int thermal_notify_tz_trip_change(int tz_id, int id, int type,
+ int temp, int hyst)
+{
+ return 0;
+}
+
+static inline int thermal_notify_cdev_state_update(int cdev_id, int state)
+{
+ return 0;
+}
+
+static inline int thermal_notify_cdev_add(int cdev_id, const char *name,
+ int max_state)
+{
+ return 0;
+}
+
+static inline int thermal_notify_cdev_delete(int cdev_id)
+{
+ return 0;
+}
+
+static inline int thermal_notify_tz_gov_change(int tz_id, const char *name)
+{
+ return 0;
+}
+
+static inline int thermal_genl_sampling_temp(int id, int temp)
+{
+ return 0;
+}
+#endif /* CONFIG_THERMAL_NETLINK */
diff --git a/drivers/thermal/thermal_of.c b/drivers/thermal/thermal_of.c
index ddf88db..69ef12f 100644
--- a/drivers/thermal/thermal_of.c
+++ b/drivers/thermal/thermal_of.c
@@ -51,7 +51,6 @@
/**
* struct __thermal_zone - internal representation of a thermal zone
- * @mode: current thermal zone device mode (enabled/disabled)
* @passive_delay: polling interval while passive cooling is activated
* @polling_delay: zone polling interval
* @slope: slope of the temperature adjustment curve
@@ -65,7 +64,6 @@
*/
struct __thermal_zone {
- enum thermal_device_mode mode;
int passive_delay;
int polling_delay;
int slope;
@@ -269,39 +267,6 @@
return 0;
}
-static int of_thermal_get_mode(struct thermal_zone_device *tz,
- enum thermal_device_mode *mode)
-{
- struct __thermal_zone *data = tz->devdata;
-
- *mode = data->mode;
-
- return 0;
-}
-
-static int of_thermal_set_mode(struct thermal_zone_device *tz,
- enum thermal_device_mode mode)
-{
- struct __thermal_zone *data = tz->devdata;
-
- mutex_lock(&tz->lock);
-
- if (mode == THERMAL_DEVICE_ENABLED) {
- tz->polling_delay = data->polling_delay;
- tz->passive_delay = data->passive_delay;
- } else {
- tz->polling_delay = 0;
- tz->passive_delay = 0;
- }
-
- mutex_unlock(&tz->lock);
-
- data->mode = mode;
- thermal_zone_device_update(tz, THERMAL_EVENT_UNSPECIFIED);
-
- return 0;
-}
-
static int of_thermal_get_trip_type(struct thermal_zone_device *tz, int trip,
enum thermal_trip_type *type)
{
@@ -393,9 +358,6 @@
}
static struct thermal_zone_device_ops of_thermal_ops = {
- .get_mode = of_thermal_get_mode,
- .set_mode = of_thermal_set_mode,
-
.get_trip_type = of_thermal_get_trip_type,
.get_trip_temp = of_thermal_get_trip_temp,
.set_trip_temp = of_thermal_set_trip_temp,
@@ -554,7 +516,7 @@
tzd = thermal_zone_of_add_sensor(child, sensor_np,
data, ops);
if (!IS_ERR(tzd))
- tzd->ops->set_mode(tzd, THERMAL_DEVICE_ENABLED);
+ thermal_zone_device_enable(tzd);
of_node_put(child);
goto exit;
@@ -979,7 +941,6 @@
finish:
of_node_put(child);
- tz->mode = THERMAL_DEVICE_DISABLED;
return tz;
diff --git a/drivers/thermal/thermal_sysfs.c b/drivers/thermal/thermal_sysfs.c
index aa99edb..8c23121 100644
--- a/drivers/thermal/thermal_sysfs.c
+++ b/drivers/thermal/thermal_sysfs.c
@@ -49,18 +49,9 @@
mode_show(struct device *dev, struct device_attribute *attr, char *buf)
{
struct thermal_zone_device *tz = to_thermal_zone(dev);
- enum thermal_device_mode mode;
- int result;
+ int enabled = thermal_zone_device_is_enabled(tz);
- if (!tz->ops->get_mode)
- return -EPERM;
-
- result = tz->ops->get_mode(tz, &mode);
- if (result)
- return result;
-
- return sprintf(buf, "%s\n", mode == THERMAL_DEVICE_ENABLED ? "enabled"
- : "disabled");
+ return sprintf(buf, "%s\n", enabled ? "enabled" : "disabled");
}
static ssize_t
@@ -70,13 +61,10 @@
struct thermal_zone_device *tz = to_thermal_zone(dev);
int result;
- if (!tz->ops->set_mode)
- return -EPERM;
-
if (!strncmp(buf, "enabled", sizeof("enabled") - 1))
- result = tz->ops->set_mode(tz, THERMAL_DEVICE_ENABLED);
+ result = thermal_zone_device_enable(tz);
else if (!strncmp(buf, "disabled", sizeof("disabled") - 1))
- result = tz->ops->set_mode(tz, THERMAL_DEVICE_DISABLED);
+ result = thermal_zone_device_disable(tz);
else
result = -EINVAL;
@@ -124,7 +112,8 @@
{
struct thermal_zone_device *tz = to_thermal_zone(dev);
int trip, ret;
- int temperature;
+ int temperature, hyst = 0;
+ enum thermal_trip_type type;
if (!tz->ops->set_trip_temp)
return -EPERM;
@@ -139,6 +128,18 @@
if (ret)
return ret;
+ if (tz->ops->get_trip_hyst) {
+ ret = tz->ops->get_trip_hyst(tz, trip, &hyst);
+ if (ret)
+ return ret;
+ }
+
+ ret = tz->ops->get_trip_type(tz, trip, &type);
+ if (ret)
+ return ret;
+
+ thermal_notify_tz_trip_change(tz->id, trip, type, temperature, hyst);
+
thermal_zone_device_update(tz, THERMAL_EVENT_UNSPECIFIED);
return count;
@@ -428,30 +429,13 @@
.attrs = thermal_zone_dev_attrs,
};
-/* We expose mode only if .get_mode is present */
static struct attribute *thermal_zone_mode_attrs[] = {
&dev_attr_mode.attr,
NULL,
};
-static umode_t thermal_zone_mode_is_visible(struct kobject *kobj,
- struct attribute *attr,
- int attrno)
-{
- struct device *dev = container_of(kobj, struct device, kobj);
- struct thermal_zone_device *tz;
-
- tz = container_of(dev, struct thermal_zone_device, device);
-
- if (tz->ops->get_mode)
- return attr->mode;
-
- return 0;
-}
-
static struct attribute_group thermal_zone_mode_attribute_group = {
.attrs = thermal_zone_mode_attrs,
- .is_visible = thermal_zone_mode_is_visible,
};
/* We expose passive only if passive trips are present */
diff --git a/drivers/thermal/ti-soc-thermal/ti-thermal-common.c b/drivers/thermal/ti-soc-thermal/ti-thermal-common.c
index 85776db..2ce4b19 100644
--- a/drivers/thermal/ti-soc-thermal/ti-thermal-common.c
+++ b/drivers/thermal/ti-soc-thermal/ti-thermal-common.c
@@ -169,7 +169,7 @@
data = ti_bandgap_get_sensor_data(bgp, id);
- if (!IS_ERR_OR_NULL(data))
+ if (IS_ERR_OR_NULL(data))
data = ti_thermal_build_data(bgp, id);
if (!data)
diff --git a/drivers/xen/balloon.c b/drivers/xen/balloon.c
index f5c838a..37ffccd 100644
--- a/drivers/xen/balloon.c
+++ b/drivers/xen/balloon.c
@@ -265,20 +265,6 @@
return NULL;
}
-#ifdef CONFIG_SPARSEMEM
- {
- unsigned long limit = 1UL << (MAX_PHYSMEM_BITS - PAGE_SHIFT);
- unsigned long pfn = res->start >> PAGE_SHIFT;
-
- if (pfn > limit) {
- pr_err("New System RAM resource outside addressable RAM (%lu > %lu)\n",
- pfn, limit);
- release_memory_resource(res);
- return NULL;
- }
- }
-#endif
-
return res;
}
@@ -567,11 +553,13 @@
if (xen_hotplug_unpopulated) {
st = reserve_additional_memory();
if (st != BP_ECANCELED) {
+ int rc;
+
mutex_unlock(&balloon_mutex);
- wait_event(balloon_wq,
+ rc = wait_event_interruptible(balloon_wq,
!list_empty(&ballooned_pages));
mutex_lock(&balloon_mutex);
- return 0;
+ return rc ? -ENOMEM : 0;
}
}
@@ -629,6 +617,12 @@
out_undo:
mutex_unlock(&balloon_mutex);
free_xenballooned_pages(pgno, pages);
+ /*
+ * NB: free_xenballooned_pages will only subtract pgno pages, but since
+ * target_unpopulated is incremented with nr_pages at the start we need
+ * to remove the remaining ones also, or accounting will be screwed.
+ */
+ balloon_stats.target_unpopulated -= nr_pages - pgno;
return ret;
}
EXPORT_SYMBOL(alloc_xenballooned_pages);
diff --git a/drivers/xen/privcmd.c b/drivers/xen/privcmd.c
index 5dfc59f..63abe6c 100644
--- a/drivers/xen/privcmd.c
+++ b/drivers/xen/privcmd.c
@@ -579,13 +579,13 @@
static int lock_pages(
struct privcmd_dm_op_buf kbufs[], unsigned int num,
- struct page *pages[], unsigned int nr_pages)
+ struct page *pages[], unsigned int nr_pages, unsigned int *pinned)
{
unsigned int i;
for (i = 0; i < num; i++) {
unsigned int requested;
- int pinned;
+ int page_count;
requested = DIV_ROUND_UP(
offset_in_page(kbufs[i].uptr) + kbufs[i].size,
@@ -593,14 +593,15 @@
if (requested > nr_pages)
return -ENOSPC;
- pinned = get_user_pages_fast(
+ page_count = pin_user_pages_fast(
(unsigned long) kbufs[i].uptr,
requested, FOLL_WRITE, pages);
- if (pinned < 0)
- return pinned;
+ if (page_count < 0)
+ return page_count;
- nr_pages -= pinned;
- pages += pinned;
+ *pinned += page_count;
+ nr_pages -= page_count;
+ pages += page_count;
}
return 0;
@@ -608,15 +609,7 @@
static void unlock_pages(struct page *pages[], unsigned int nr_pages)
{
- unsigned int i;
-
- if (!pages)
- return;
-
- for (i = 0; i < nr_pages; i++) {
- if (pages[i])
- put_page(pages[i]);
- }
+ unpin_user_pages_dirty_lock(pages, nr_pages, true);
}
static long privcmd_ioctl_dm_op(struct file *file, void __user *udata)
@@ -629,6 +622,7 @@
struct xen_dm_op_buf *xbufs = NULL;
unsigned int i;
long rc;
+ unsigned int pinned = 0;
if (copy_from_user(&kdata, udata, sizeof(kdata)))
return -EFAULT;
@@ -682,9 +676,11 @@
goto out;
}
- rc = lock_pages(kbufs, kdata.num, pages, nr_pages);
- if (rc)
+ rc = lock_pages(kbufs, kdata.num, pages, nr_pages, &pinned);
+ if (rc < 0) {
+ nr_pages = pinned;
goto out;
+ }
for (i = 0; i < kdata.num; i++) {
set_xen_guest_handle(xbufs[i].h, kbufs[i].uptr);
diff --git a/drivers/xen/swiotlb-xen.c b/drivers/xen/swiotlb-xen.c
index b6d2776..39a0f2e 100644
--- a/drivers/xen/swiotlb-xen.c
+++ b/drivers/xen/swiotlb-xen.c
@@ -52,37 +52,39 @@
* Quick lookup value of the bus address of the IOTLB.
*/
-static u64 start_dma_addr;
-
-/*
- * Both of these functions should avoid XEN_PFN_PHYS because phys_addr_t
- * can be 32bit when dma_addr_t is 64bit leading to a loss in
- * information if the shift is done before casting to 64bit.
- */
-static inline dma_addr_t xen_phys_to_bus(phys_addr_t paddr)
+static inline phys_addr_t xen_phys_to_bus(struct device *dev, phys_addr_t paddr)
{
unsigned long bfn = pfn_to_bfn(XEN_PFN_DOWN(paddr));
- dma_addr_t dma = (dma_addr_t)bfn << XEN_PAGE_SHIFT;
+ phys_addr_t baddr = (phys_addr_t)bfn << XEN_PAGE_SHIFT;
- dma |= paddr & ~XEN_PAGE_MASK;
-
- return dma;
+ baddr |= paddr & ~XEN_PAGE_MASK;
+ return baddr;
}
-static inline phys_addr_t xen_bus_to_phys(dma_addr_t baddr)
+static inline dma_addr_t xen_phys_to_dma(struct device *dev, phys_addr_t paddr)
+{
+ return phys_to_dma(dev, xen_phys_to_bus(dev, paddr));
+}
+
+static inline phys_addr_t xen_bus_to_phys(struct device *dev,
+ phys_addr_t baddr)
{
unsigned long xen_pfn = bfn_to_pfn(XEN_PFN_DOWN(baddr));
- dma_addr_t dma = (dma_addr_t)xen_pfn << XEN_PAGE_SHIFT;
- phys_addr_t paddr = dma;
-
- paddr |= baddr & ~XEN_PAGE_MASK;
+ phys_addr_t paddr = (xen_pfn << XEN_PAGE_SHIFT) |
+ (baddr & ~XEN_PAGE_MASK);
return paddr;
}
-static inline dma_addr_t xen_virt_to_bus(void *address)
+static inline phys_addr_t xen_dma_to_phys(struct device *dev,
+ dma_addr_t dma_addr)
{
- return xen_phys_to_bus(virt_to_phys(address));
+ return xen_bus_to_phys(dev, dma_to_phys(dev, dma_addr));
+}
+
+static inline dma_addr_t xen_virt_to_bus(struct device *dev, void *address)
+{
+ return xen_phys_to_dma(dev, virt_to_phys(address));
}
static inline int range_straddles_page_boundary(phys_addr_t p, size_t size)
@@ -99,11 +101,11 @@
return 0;
}
-static int is_xen_swiotlb_buffer(dma_addr_t dma_addr)
+static int is_xen_swiotlb_buffer(struct device *dev, dma_addr_t dma_addr)
{
- unsigned long bfn = XEN_PFN_DOWN(dma_addr);
+ unsigned long bfn = XEN_PFN_DOWN(dma_to_phys(dev, dma_addr));
unsigned long xen_pfn = bfn_to_local_pfn(bfn);
- phys_addr_t paddr = XEN_PFN_PHYS(xen_pfn);
+ phys_addr_t paddr = (phys_addr_t)xen_pfn << XEN_PAGE_SHIFT;
/* If the address is outside our domain, it CAN
* have the same virtual address as another address
@@ -241,7 +243,6 @@
m_ret = XEN_SWIOTLB_EFIXUP;
goto error;
}
- start_dma_addr = xen_virt_to_bus(xen_io_tlb_start);
if (early) {
if (swiotlb_init_with_tbl(xen_io_tlb_start, xen_io_tlb_nslabs,
verbose))
@@ -307,12 +308,12 @@
if (hwdev && hwdev->coherent_dma_mask)
dma_mask = hwdev->coherent_dma_mask;
- /* At this point dma_handle is the physical address, next we are
+ /* At this point dma_handle is the dma address, next we are
* going to set it to the machine address.
* Do not use virt_to_phys(ret) because on ARM it doesn't correspond
* to *dma_handle. */
- phys = *dma_handle;
- dev_addr = xen_phys_to_bus(phys);
+ phys = dma_to_phys(hwdev, *dma_handle);
+ dev_addr = xen_phys_to_dma(hwdev, phys);
if (((dev_addr + size - 1 <= dma_mask)) &&
!range_straddles_page_boundary(phys, size))
*dma_handle = dev_addr;
@@ -322,6 +323,7 @@
xen_free_coherent_pages(hwdev, size, ret, (dma_addr_t)phys, attrs);
return NULL;
}
+ *dma_handle = phys_to_dma(hwdev, *dma_handle);
SetPageXenRemapped(virt_to_page(ret));
}
memset(ret, 0, size);
@@ -335,23 +337,30 @@
int order = get_order(size);
phys_addr_t phys;
u64 dma_mask = DMA_BIT_MASK(32);
+ struct page *page;
if (hwdev && hwdev->coherent_dma_mask)
dma_mask = hwdev->coherent_dma_mask;
/* do not use virt_to_phys because on ARM it doesn't return you the
* physical address */
- phys = xen_bus_to_phys(dev_addr);
+ phys = xen_dma_to_phys(hwdev, dev_addr);
/* Convert the size to actually allocated. */
size = 1UL << (order + XEN_PAGE_SHIFT);
+ if (is_vmalloc_addr(vaddr))
+ page = vmalloc_to_page(vaddr);
+ else
+ page = virt_to_page(vaddr);
+
if (!WARN_ON((dev_addr + size - 1 > dma_mask) ||
range_straddles_page_boundary(phys, size)) &&
- TestClearPageXenRemapped(virt_to_page(vaddr)))
+ TestClearPageXenRemapped(page))
xen_destroy_contiguous_region(phys, order);
- xen_free_coherent_pages(hwdev, size, vaddr, (dma_addr_t)phys, attrs);
+ xen_free_coherent_pages(hwdev, size, vaddr, phys_to_dma(hwdev, phys),
+ attrs);
}
/*
@@ -367,7 +376,7 @@
unsigned long attrs)
{
phys_addr_t map, phys = page_to_phys(page) + offset;
- dma_addr_t dev_addr = xen_phys_to_bus(phys);
+ dma_addr_t dev_addr = xen_phys_to_dma(dev, phys);
BUG_ON(dir == DMA_NONE);
/*
@@ -386,13 +395,13 @@
*/
trace_swiotlb_bounced(dev, dev_addr, size, swiotlb_force);
- map = swiotlb_tbl_map_single(dev, start_dma_addr, phys,
- size, size, dir, attrs);
+ map = swiotlb_tbl_map_single(dev, virt_to_phys(xen_io_tlb_start),
+ phys, size, size, dir, attrs);
if (map == (phys_addr_t)DMA_MAPPING_ERROR)
return DMA_MAPPING_ERROR;
phys = map;
- dev_addr = xen_phys_to_bus(map);
+ dev_addr = xen_phys_to_dma(dev, map);
/*
* Ensure that the address returned is DMA'ble
@@ -404,8 +413,12 @@
}
done:
- if (!dev_is_dma_coherent(dev) && !(attrs & DMA_ATTR_SKIP_CPU_SYNC))
- xen_dma_sync_for_device(dev_addr, phys, size, dir);
+ if (!dev_is_dma_coherent(dev) && !(attrs & DMA_ATTR_SKIP_CPU_SYNC)) {
+ if (pfn_valid(PFN_DOWN(dma_to_phys(dev, dev_addr))))
+ arch_sync_dma_for_device(phys, size, dir);
+ else
+ xen_dma_sync_for_device(dev, dev_addr, size, dir);
+ }
return dev_addr;
}
@@ -420,15 +433,19 @@
static void xen_swiotlb_unmap_page(struct device *hwdev, dma_addr_t dev_addr,
size_t size, enum dma_data_direction dir, unsigned long attrs)
{
- phys_addr_t paddr = xen_bus_to_phys(dev_addr);
+ phys_addr_t paddr = xen_dma_to_phys(hwdev, dev_addr);
BUG_ON(dir == DMA_NONE);
- if (!dev_is_dma_coherent(hwdev) && !(attrs & DMA_ATTR_SKIP_CPU_SYNC))
- xen_dma_sync_for_cpu(dev_addr, paddr, size, dir);
+ if (!dev_is_dma_coherent(hwdev) && !(attrs & DMA_ATTR_SKIP_CPU_SYNC)) {
+ if (pfn_valid(PFN_DOWN(dma_to_phys(hwdev, dev_addr))))
+ arch_sync_dma_for_cpu(paddr, size, dir);
+ else
+ xen_dma_sync_for_cpu(hwdev, dev_addr, size, dir);
+ }
/* NOTE: We use dev_addr here, not paddr! */
- if (is_xen_swiotlb_buffer(dev_addr))
+ if (is_xen_swiotlb_buffer(hwdev, dev_addr))
swiotlb_tbl_unmap_single(hwdev, paddr, size, size, dir, attrs);
}
@@ -436,12 +453,16 @@
xen_swiotlb_sync_single_for_cpu(struct device *dev, dma_addr_t dma_addr,
size_t size, enum dma_data_direction dir)
{
- phys_addr_t paddr = xen_bus_to_phys(dma_addr);
+ phys_addr_t paddr = xen_dma_to_phys(dev, dma_addr);
- if (!dev_is_dma_coherent(dev))
- xen_dma_sync_for_cpu(dma_addr, paddr, size, dir);
+ if (!dev_is_dma_coherent(dev)) {
+ if (pfn_valid(PFN_DOWN(dma_to_phys(dev, dma_addr))))
+ arch_sync_dma_for_cpu(paddr, size, dir);
+ else
+ xen_dma_sync_for_cpu(dev, dma_addr, size, dir);
+ }
- if (is_xen_swiotlb_buffer(dma_addr))
+ if (is_xen_swiotlb_buffer(dev, dma_addr))
swiotlb_tbl_sync_single(dev, paddr, size, dir, SYNC_FOR_CPU);
}
@@ -449,13 +470,17 @@
xen_swiotlb_sync_single_for_device(struct device *dev, dma_addr_t dma_addr,
size_t size, enum dma_data_direction dir)
{
- phys_addr_t paddr = xen_bus_to_phys(dma_addr);
+ phys_addr_t paddr = xen_dma_to_phys(dev, dma_addr);
- if (is_xen_swiotlb_buffer(dma_addr))
+ if (is_xen_swiotlb_buffer(dev, dma_addr))
swiotlb_tbl_sync_single(dev, paddr, size, dir, SYNC_FOR_DEVICE);
- if (!dev_is_dma_coherent(dev))
- xen_dma_sync_for_device(dma_addr, paddr, size, dir);
+ if (!dev_is_dma_coherent(dev)) {
+ if (pfn_valid(PFN_DOWN(dma_to_phys(dev, dma_addr))))
+ arch_sync_dma_for_device(paddr, size, dir);
+ else
+ xen_dma_sync_for_device(dev, dma_addr, size, dir);
+ }
}
/*
@@ -536,7 +561,7 @@
static int
xen_swiotlb_dma_supported(struct device *hwdev, u64 mask)
{
- return xen_virt_to_bus(xen_io_tlb_end - 1) <= mask;
+ return xen_virt_to_bus(hwdev, xen_io_tlb_end - 1) <= mask;
}
const struct dma_map_ops xen_swiotlb_dma_ops = {
diff --git a/fs/Makefile b/fs/Makefile
index 2ce5112..1c7b0e3 100644
--- a/fs/Makefile
+++ b/fs/Makefile
@@ -13,7 +13,7 @@
seq_file.o xattr.o libfs.o fs-writeback.o \
pnode.o splice.o sync.o utimes.o d_path.o \
stack.o fs_struct.o statfs.o fs_pin.o nsfs.o \
- fs_types.o fs_context.o fs_parser.o fsopen.o
+ fs_types.o fs_context.o fs_parser.o fsopen.o init.o
ifeq ($(CONFIG_BLOCK),y)
obj-y += buffer.o block_dev.o direct-io.o mpage.o
diff --git a/fs/binfmt_elf.c b/fs/binfmt_elf.c
index 9fe3b51..e922a6a 100644
--- a/fs/binfmt_elf.c
+++ b/fs/binfmt_elf.c
@@ -1821,7 +1821,7 @@
long signr, size_t *total)
{
unsigned int i;
- unsigned int regset0_size = regset_size(t->task, &view->regsets[0]);
+ int regset0_size;
/*
* NT_PRSTATUS is the one special case, because the regset data
@@ -1830,8 +1830,10 @@
* We assume that regset 0 is NT_PRSTATUS.
*/
fill_prstatus(&t->prstatus, t->task, signr);
- (void) view->regsets[0].get(t->task, &view->regsets[0], 0, regset0_size,
- &t->prstatus.pr_reg, NULL);
+ regset0_size = regset_get(t->task, &view->regsets[0],
+ sizeof(t->prstatus.pr_reg), &t->prstatus.pr_reg);
+ if (regset0_size < 0)
+ return 0;
fill_note(&t->notes[0], "CORE", NT_PRSTATUS,
PRSTATUS_SIZE(t->prstatus, regset0_size), &t->prstatus);
@@ -1846,32 +1848,28 @@
*/
for (i = 1; i < view->n; ++i) {
const struct user_regset *regset = &view->regsets[i];
+ int note_type = regset->core_note_type;
+ bool is_fpreg = note_type == NT_PRFPREG;
+ void *data;
+ int ret;
+
do_thread_regset_writeback(t->task, regset);
- if (regset->core_note_type && regset->get &&
- (!regset->active || regset->active(t->task, regset) > 0)) {
- int ret;
- size_t size = regset_size(t->task, regset);
- void *data = kzalloc(size, GFP_KERNEL);
- if (unlikely(!data))
- return 0;
- ret = regset->get(t->task, regset,
- 0, size, data, NULL);
- if (unlikely(ret))
- kfree(data);
- else {
- if (regset->core_note_type != NT_PRFPREG)
- fill_note(&t->notes[i], "LINUX",
- regset->core_note_type,
- size, data);
- else {
- SET_PR_FPVALID(&t->prstatus,
- 1, regset0_size);
- fill_note(&t->notes[i], "CORE",
- NT_PRFPREG, size, data);
- }
- *total += notesize(&t->notes[i]);
- }
- }
+ if (!note_type) // not for coredumps
+ continue;
+ if (regset->active && regset->active(t->task, regset) <= 0)
+ continue;
+
+ ret = regset_get_alloc(t->task, regset, ~0U, &data);
+ if (ret < 0)
+ continue;
+
+ if (is_fpreg)
+ SET_PR_FPVALID(&t->prstatus, 1, regset0_size);
+
+ fill_note(&t->notes[i], is_fpreg ? "CORE" : "LINUX",
+ note_type, ret, data);
+
+ *total += notesize(&t->notes[i]);
}
return 1;
diff --git a/fs/cifs/cifsacl.h b/fs/cifs/cifsacl.h
index 17562ea..45665ff 100644
--- a/fs/cifs/cifsacl.h
+++ b/fs/cifs/cifsacl.h
@@ -132,7 +132,7 @@
/*
* The current SMB3 form of security descriptor is similar to what was used for
* cifs (see above) but some fields are split, and fields in the struct below
- * matches names of fields to the the spec, MS-DTYP (see sections 2.4.5 and
+ * matches names of fields to the spec, MS-DTYP (see sections 2.4.5 and
* 2.4.6). Note that "CamelCase" fields are used in this struct in order to
* match the MS-DTYP and MS-SMB2 specs which define the wire format.
*/
@@ -178,7 +178,7 @@
/*
* Used to store the special 'NFS SIDs' used to persist the POSIX uid and gid
- * See See http://technet.microsoft.com/en-us/library/hh509017(v=ws.10).aspx
+ * See http://technet.microsoft.com/en-us/library/hh509017(v=ws.10).aspx
*/
struct owner_sid {
u8 Revision;
diff --git a/fs/cifs/cifsglob.h b/fs/cifs/cifsglob.h
index f4b88cd..b296964 100644
--- a/fs/cifs/cifsglob.h
+++ b/fs/cifs/cifsglob.h
@@ -1466,7 +1466,7 @@
struct list_head llist; /* locks helb by this inode */
/*
* NOTE: Some code paths call down_read(lock_sem) twice, so
- * we must always use use cifs_down_write() instead of down_write()
+ * we must always use cifs_down_write() instead of down_write()
* for this semaphore to avoid deadlocks.
*/
struct rw_semaphore lock_sem; /* protect the fields above */
diff --git a/fs/cifs/cifsproto.h b/fs/cifs/cifsproto.h
index 7a836ec..bb68cbf 100644
--- a/fs/cifs/cifsproto.h
+++ b/fs/cifs/cifsproto.h
@@ -154,6 +154,7 @@
extern int cifs_convert_address(struct sockaddr *dst, const char *src, int len);
extern void cifs_set_port(struct sockaddr *addr, const unsigned short int port);
extern int map_smb_to_linux_error(char *buf, bool logErr);
+extern int map_and_check_smb_error(struct mid_q_entry *mid, bool logErr);
extern void header_assemble(struct smb_hdr *, char /* command */ ,
const struct cifs_tcon *, int /* length of
fixed section (word count) in two byte units */);
@@ -271,6 +272,9 @@
extern void cifs_free_llist(struct list_head *llist);
extern void cifs_del_lock_waiters(struct cifsLockInfo *lock);
+extern int cifs_tree_connect(const unsigned int xid, struct cifs_tcon *tcon,
+ const struct nls_table *nlsc);
+
extern int cifs_negotiate_protocol(const unsigned int xid,
struct cifs_ses *ses);
extern int cifs_setup_session(const unsigned int xid, struct cifs_ses *ses,
@@ -344,7 +348,7 @@
extern int CIFSSMBSetPathInfo(const unsigned int xid, struct cifs_tcon *tcon,
const char *fileName, const FILE_BASIC_INFO *data,
const struct nls_table *nls_codepage,
- int remap_special_chars);
+ struct cifs_sb_info *cifs_sb);
extern int CIFSSMBSetFileInfo(const unsigned int xid, struct cifs_tcon *tcon,
const FILE_BASIC_INFO *data, __u16 fid,
__u32 pid_of_opener);
@@ -613,8 +617,7 @@
struct super_block *cifs_get_tcp_super(struct TCP_Server_Info *server);
void cifs_put_tcp_super(struct super_block *sb);
-int update_super_prepath(struct cifs_tcon *tcon, const char *prefix,
- size_t prefix_len);
+int update_super_prepath(struct cifs_tcon *tcon, char *prefix);
#ifdef CONFIG_CIFS_DFS_UPCALL
static inline int get_dfs_path(const unsigned int xid, struct cifs_ses *ses,
diff --git a/fs/cifs/cifssmb.c b/fs/cifs/cifssmb.c
index bf41ee0..0e763d2 100644
--- a/fs/cifs/cifssmb.c
+++ b/fs/cifs/cifssmb.c
@@ -124,116 +124,6 @@
*/
}
-#ifdef CONFIG_CIFS_DFS_UPCALL
-static int __cifs_reconnect_tcon(const struct nls_table *nlsc,
- struct cifs_tcon *tcon)
-{
- int rc;
- struct TCP_Server_Info *server = tcon->ses->server;
- struct dfs_cache_tgt_list tl;
- struct dfs_cache_tgt_iterator *it = NULL;
- char *tree;
- const char *tcp_host;
- size_t tcp_host_len;
- const char *dfs_host;
- size_t dfs_host_len;
-
- tree = kzalloc(MAX_TREE_SIZE, GFP_KERNEL);
- if (!tree)
- return -ENOMEM;
-
- if (!tcon->dfs_path) {
- if (tcon->ipc) {
- scnprintf(tree, MAX_TREE_SIZE, "\\\\%s\\IPC$",
- server->hostname);
- rc = CIFSTCon(0, tcon->ses, tree, tcon, nlsc);
- } else {
- rc = CIFSTCon(0, tcon->ses, tcon->treeName, tcon, nlsc);
- }
- goto out;
- }
-
- rc = dfs_cache_noreq_find(tcon->dfs_path + 1, NULL, &tl);
- if (rc)
- goto out;
-
- extract_unc_hostname(server->hostname, &tcp_host, &tcp_host_len);
-
- for (it = dfs_cache_get_tgt_iterator(&tl); it;
- it = dfs_cache_get_next_tgt(&tl, it)) {
- const char *share, *prefix;
- size_t share_len, prefix_len;
- bool target_match;
-
- rc = dfs_cache_get_tgt_share(it, &share, &share_len, &prefix,
- &prefix_len);
- if (rc) {
- cifs_dbg(VFS, "%s: failed to parse target share %d\n",
- __func__, rc);
- continue;
- }
-
- extract_unc_hostname(share, &dfs_host, &dfs_host_len);
-
- if (dfs_host_len != tcp_host_len
- || strncasecmp(dfs_host, tcp_host, dfs_host_len) != 0) {
- cifs_dbg(FYI, "%s: %.*s doesn't match %.*s\n",
- __func__,
- (int)dfs_host_len, dfs_host,
- (int)tcp_host_len, tcp_host);
-
- rc = match_target_ip(server, dfs_host, dfs_host_len,
- &target_match);
- if (rc) {
- cifs_dbg(VFS, "%s: failed to match target ip: %d\n",
- __func__, rc);
- break;
- }
-
- if (!target_match) {
- cifs_dbg(FYI, "%s: skipping target\n", __func__);
- continue;
- }
- }
-
- if (tcon->ipc) {
- scnprintf(tree, MAX_TREE_SIZE, "\\\\%.*s\\IPC$",
- (int)share_len, share);
- rc = CIFSTCon(0, tcon->ses, tree, tcon, nlsc);
- } else {
- scnprintf(tree, MAX_TREE_SIZE, "\\%.*s", (int)share_len,
- share);
- rc = CIFSTCon(0, tcon->ses, tree, tcon, nlsc);
- if (!rc) {
- rc = update_super_prepath(tcon, prefix,
- prefix_len);
- break;
- }
- }
- if (rc == -EREMOTE)
- break;
- }
-
- if (!rc) {
- if (it)
- rc = dfs_cache_noreq_update_tgthint(tcon->dfs_path + 1,
- it);
- else
- rc = -ENOENT;
- }
- dfs_cache_free_tgts(&tl);
-out:
- kfree(tree);
- return rc;
-}
-#else
-static inline int __cifs_reconnect_tcon(const struct nls_table *nlsc,
- struct cifs_tcon *tcon)
-{
- return CIFSTCon(0, tcon->ses, tcon->treeName, tcon, nlsc);
-}
-#endif
-
/* reconnect the socket, tcon, and smb session if needed */
static int
cifs_reconnect_tcon(struct cifs_tcon *tcon, int smb_command)
@@ -338,7 +228,7 @@
}
cifs_mark_open_files_invalid(tcon);
- rc = __cifs_reconnect_tcon(nls_codepage, tcon);
+ rc = cifs_tree_connect(0, tcon, nls_codepage);
mutex_unlock(&ses->session_mutex);
cifs_dbg(FYI, "reconnect tcon rc = %d\n", rc);
@@ -5913,10 +5803,42 @@
return rc;
}
+static int
+CIFSSMBSetPathInfoFB(const unsigned int xid, struct cifs_tcon *tcon,
+ const char *fileName, const FILE_BASIC_INFO *data,
+ const struct nls_table *nls_codepage,
+ struct cifs_sb_info *cifs_sb)
+{
+ int oplock = 0;
+ struct cifs_open_parms oparms;
+ struct cifs_fid fid;
+ int rc;
+
+ oparms.tcon = tcon;
+ oparms.cifs_sb = cifs_sb;
+ oparms.desired_access = GENERIC_WRITE;
+ oparms.create_options = cifs_create_options(cifs_sb, 0);
+ oparms.disposition = FILE_OPEN;
+ oparms.path = fileName;
+ oparms.fid = &fid;
+ oparms.reconnect = false;
+
+ rc = CIFS_open(xid, &oparms, &oplock, NULL);
+ if (rc)
+ goto out;
+
+ rc = CIFSSMBSetFileInfo(xid, tcon, data, fid.netfid, current->tgid);
+ CIFSSMBClose(xid, tcon, fid.netfid);
+out:
+
+ return rc;
+}
+
int
CIFSSMBSetPathInfo(const unsigned int xid, struct cifs_tcon *tcon,
const char *fileName, const FILE_BASIC_INFO *data,
- const struct nls_table *nls_codepage, int remap)
+ const struct nls_table *nls_codepage,
+ struct cifs_sb_info *cifs_sb)
{
TRANSACTION2_SPI_REQ *pSMB = NULL;
TRANSACTION2_SPI_RSP *pSMBr = NULL;
@@ -5925,6 +5847,7 @@
int bytes_returned = 0;
char *data_offset;
__u16 params, param_offset, offset, byte_count, count;
+ int remap = cifs_remap(cifs_sb);
cifs_dbg(FYI, "In SetTimes\n");
@@ -5987,6 +5910,10 @@
if (rc == -EAGAIN)
goto SetTimesRetry;
+ if (rc == -EOPNOTSUPP)
+ return CIFSSMBSetPathInfoFB(xid, tcon, fileName, data,
+ nls_codepage, cifs_sb);
+
return rc;
}
diff --git a/fs/cifs/connect.c b/fs/cifs/connect.c
index 889fee5..0ad1309 100644
--- a/fs/cifs/connect.c
+++ b/fs/cifs/connect.c
@@ -393,15 +393,14 @@
#ifdef CONFIG_CIFS_DFS_UPCALL
/* These functions must be called with server->srv_mutex held */
-static void reconn_inval_dfs_target(struct TCP_Server_Info *server,
- struct cifs_sb_info *cifs_sb,
- struct dfs_cache_tgt_list *tgt_list,
- struct dfs_cache_tgt_iterator **tgt_it)
+static void reconn_set_next_dfs_target(struct TCP_Server_Info *server,
+ struct cifs_sb_info *cifs_sb,
+ struct dfs_cache_tgt_list *tgt_list,
+ struct dfs_cache_tgt_iterator **tgt_it)
{
const char *name;
- if (!cifs_sb || !cifs_sb->origin_fullpath || !tgt_list ||
- !server->nr_targets)
+ if (!cifs_sb || !cifs_sb->origin_fullpath)
return;
if (!*tgt_it) {
@@ -471,11 +470,13 @@
sb = NULL;
} else {
cifs_sb = CIFS_SB(sb);
-
rc = reconn_setup_dfs_targets(cifs_sb, &tgt_list);
- if (rc && (rc != -EOPNOTSUPP)) {
- cifs_server_dbg(VFS, "%s: no target servers for DFS failover\n",
- __func__);
+ if (rc) {
+ cifs_sb = NULL;
+ if (rc != -EOPNOTSUPP) {
+ cifs_server_dbg(VFS, "%s: no target servers for DFS failover\n",
+ __func__);
+ }
} else {
server->nr_targets = dfs_cache_get_nr_tgts(&tgt_list);
}
@@ -578,7 +579,7 @@
* feature is disabled, then we will retry last server we
* connected to before.
*/
- reconn_inval_dfs_target(server, cifs_sb, &tgt_list, &tgt_it);
+ reconn_set_next_dfs_target(server, cifs_sb, &tgt_list, &tgt_it);
#endif
rc = reconn_set_ipaddr(server);
if (rc) {
@@ -4422,11 +4423,11 @@
static int
expand_dfs_referral(const unsigned int xid, struct cifs_ses *ses,
struct smb_vol *volume_info, struct cifs_sb_info *cifs_sb,
- int check_prefix)
+ char *ref_path)
{
int rc;
struct dfs_info3_param referral = {0};
- char *full_path = NULL, *ref_path = NULL, *mdata = NULL;
+ char *full_path = NULL, *mdata = NULL;
if (cifs_sb->mnt_cifs_flags & CIFS_MOUNT_NO_DFS)
return -EREMOTE;
@@ -4435,9 +4436,6 @@
if (IS_ERR(full_path))
return PTR_ERR(full_path);
- /* For DFS paths, skip the first '\' of the UNC */
- ref_path = check_prefix ? full_path + 1 : volume_info->UNC + 1;
-
rc = dfs_cache_find(xid, ses, cifs_sb->local_nls, cifs_remap(cifs_sb),
ref_path, &referral, NULL);
if (!rc) {
@@ -4500,13 +4498,10 @@
return 0;
}
-static int setup_dfs_tgt_conn(const char *path,
+static int setup_dfs_tgt_conn(const char *path, const char *full_path,
const struct dfs_cache_tgt_iterator *tgt_it,
- struct cifs_sb_info *cifs_sb,
- struct smb_vol *vol,
- unsigned int *xid,
- struct TCP_Server_Info **server,
- struct cifs_ses **ses,
+ struct cifs_sb_info *cifs_sb, struct smb_vol *vol, unsigned int *xid,
+ struct TCP_Server_Info **server, struct cifs_ses **ses,
struct cifs_tcon **tcon)
{
int rc;
@@ -4520,8 +4515,7 @@
if (rc)
return rc;
- mdata = cifs_compose_mount_options(cifs_sb->mountdata, path, &ref,
- &fake_devname);
+ mdata = cifs_compose_mount_options(cifs_sb->mountdata, full_path + 1, &ref, &fake_devname);
free_dfs_info_param(&ref);
if (IS_ERR(mdata)) {
@@ -4544,7 +4538,7 @@
mount_put_conns(cifs_sb, *xid, *server, *ses, *tcon);
rc = mount_get_conns(&fake_vol, cifs_sb, xid, server, ses,
tcon);
- if (!rc) {
+ if (!rc || (*server && *ses)) {
/*
* We were able to connect to new target server.
* Update current volume info with new target server.
@@ -4556,14 +4550,10 @@
return rc;
}
-static int mount_do_dfs_failover(const char *path,
- struct cifs_sb_info *cifs_sb,
- struct smb_vol *vol,
- struct cifs_ses *root_ses,
- unsigned int *xid,
- struct TCP_Server_Info **server,
- struct cifs_ses **ses,
- struct cifs_tcon **tcon)
+static int do_dfs_failover(const char *path, const char *full_path, struct cifs_sb_info *cifs_sb,
+ struct smb_vol *vol, struct cifs_ses *root_ses, unsigned int *xid,
+ struct TCP_Server_Info **server, struct cifs_ses **ses,
+ struct cifs_tcon **tcon)
{
int rc;
struct dfs_cache_tgt_list tgt_list;
@@ -4582,9 +4572,9 @@
if (rc)
break;
/* Connect to next DFS target */
- rc = setup_dfs_tgt_conn(path, tgt_it, cifs_sb, vol, xid, server,
- ses, tcon);
- if (!rc || rc == -EACCES || rc == -EOPNOTSUPP)
+ rc = setup_dfs_tgt_conn(path, full_path, tgt_it, cifs_sb, vol, xid, server, ses,
+ tcon);
+ if (!rc || (*server && *ses))
break;
}
if (!rc) {
@@ -4754,207 +4744,210 @@
}
#ifdef CONFIG_CIFS_DFS_UPCALL
-static inline void set_root_tcon(struct cifs_sb_info *cifs_sb,
- struct cifs_tcon *tcon,
- struct cifs_tcon **root)
+static void set_root_ses(struct cifs_sb_info *cifs_sb, struct cifs_ses *ses,
+ struct cifs_ses **root_ses)
{
- spin_lock(&cifs_tcp_ses_lock);
- tcon->tc_count++;
- tcon->remap = cifs_remap(cifs_sb);
- spin_unlock(&cifs_tcp_ses_lock);
- *root = tcon;
+ if (ses) {
+ spin_lock(&cifs_tcp_ses_lock);
+ ses->ses_count++;
+ ses->tcon_ipc->remap = cifs_remap(cifs_sb);
+ spin_unlock(&cifs_tcp_ses_lock);
+ }
+ *root_ses = ses;
+}
+
+static void put_root_ses(struct cifs_ses *ses)
+{
+ if (ses)
+ cifs_put_smb_ses(ses);
+}
+
+/* Check if a path component is remote and then update @dfs_path accordingly */
+static int check_dfs_prepath(struct cifs_sb_info *cifs_sb, struct smb_vol *vol,
+ const unsigned int xid, struct TCP_Server_Info *server,
+ struct cifs_tcon *tcon, char **dfs_path)
+{
+ char *path, *s;
+ char sep = CIFS_DIR_SEP(cifs_sb), tmp;
+ char *npath;
+ int rc = 0;
+ int added_treename = tcon->Flags & SMB_SHARE_IS_IN_DFS;
+ int skip = added_treename;
+
+ path = cifs_build_path_to_root(vol, cifs_sb, tcon, added_treename);
+ if (!path)
+ return -ENOMEM;
+
+ /*
+ * Walk through the path components in @path and check if they're accessible. In case any of
+ * the components is -EREMOTE, then update @dfs_path with the next DFS referral request path
+ * (NOT including the remaining components).
+ */
+ s = path;
+ do {
+ /* skip separators */
+ while (*s && *s == sep)
+ s++;
+ if (!*s)
+ break;
+ /* next separator */
+ while (*s && *s != sep)
+ s++;
+ /*
+ * if the treename is added, we then have to skip the first
+ * part within the separators
+ */
+ if (skip) {
+ skip = 0;
+ continue;
+ }
+ tmp = *s;
+ *s = 0;
+ rc = server->ops->is_path_accessible(xid, tcon, cifs_sb, path);
+ if (rc && rc == -EREMOTE) {
+ struct smb_vol v = {NULL};
+ /* if @path contains a tree name, skip it in the prefix path */
+ if (added_treename) {
+ rc = cifs_parse_devname(path, &v);
+ if (rc)
+ break;
+ rc = -EREMOTE;
+ npath = build_unc_path_to_root(&v, cifs_sb, true);
+ cifs_cleanup_volume_info_contents(&v);
+ } else {
+ v.UNC = vol->UNC;
+ v.prepath = path + 1;
+ npath = build_unc_path_to_root(&v, cifs_sb, true);
+ }
+ if (IS_ERR(npath)) {
+ rc = PTR_ERR(npath);
+ break;
+ }
+ kfree(*dfs_path);
+ *dfs_path = npath;
+ }
+ *s = tmp;
+ } while (rc == 0);
+
+ kfree(path);
+ return rc;
}
int cifs_mount(struct cifs_sb_info *cifs_sb, struct smb_vol *vol)
{
int rc = 0;
unsigned int xid;
- struct cifs_ses *ses;
- struct cifs_tcon *root_tcon = NULL;
+ struct TCP_Server_Info *server = NULL;
+ struct cifs_ses *ses = NULL, *root_ses = NULL;
struct cifs_tcon *tcon = NULL;
- struct TCP_Server_Info *server;
- char *root_path = NULL, *full_path = NULL;
- char *old_mountdata, *origin_mountdata = NULL;
- int count;
+ int count = 0;
+ char *ref_path = NULL, *full_path = NULL;
+ char *oldmnt = NULL;
+ char *mntdata = NULL;
rc = mount_get_conns(vol, cifs_sb, &xid, &server, &ses, &tcon);
- if (!rc && tcon) {
- /* If not a standalone DFS root, then check if path is remote */
- rc = dfs_cache_find(xid, ses, cifs_sb->local_nls,
- cifs_remap(cifs_sb), vol->UNC + 1, NULL,
- NULL);
- if (rc) {
- rc = is_path_remote(cifs_sb, vol, xid, server, tcon);
- if (!rc)
- goto out;
- if (rc != -EREMOTE)
- goto error;
- }
- }
/*
- * If first DFS target server went offline and we failed to connect it,
- * server and ses pointers are NULL at this point, though we still have
- * chance to get a cached DFS referral in expand_dfs_referral() and
- * retry next target available in it.
+ * Unconditionally try to get an DFS referral (even cached) to determine whether it is an
+ * DFS mount.
*
- * If a NULL ses ptr is passed to dfs_cache_find(), a lookup will be
- * performed against DFS path and *no* requests will be sent to server
- * for any new DFS referrals. Hence it's safe to skip checking whether
- * server or ses ptr is NULL.
+ * Skip prefix path to provide support for DFS referrals from w2k8 servers which don't seem
+ * to respond with PATH_NOT_COVERED to requests that include the prefix.
*/
- if (rc == -EACCES || rc == -EOPNOTSUPP)
- goto error;
-
- root_path = build_unc_path_to_root(vol, cifs_sb, false);
- if (IS_ERR(root_path)) {
- rc = PTR_ERR(root_path);
- root_path = NULL;
- goto error;
+ if (dfs_cache_find(xid, ses, cifs_sb->local_nls, cifs_remap(cifs_sb), vol->UNC + 1, NULL,
+ NULL)) {
+ /* No DFS referral was returned. Looks like a regular share. */
+ if (rc)
+ goto error;
+ /* Check if it is fully accessible and then mount it */
+ rc = is_path_remote(cifs_sb, vol, xid, server, tcon);
+ if (!rc)
+ goto out;
+ if (rc != -EREMOTE)
+ goto error;
}
-
- full_path = build_unc_path_to_root(vol, cifs_sb, true);
- if (IS_ERR(full_path)) {
- rc = PTR_ERR(full_path);
- full_path = NULL;
- goto error;
- }
- /*
- * Perform an unconditional check for whether there are DFS
- * referrals for this path without prefix, to provide support
- * for DFS referrals from w2k8 servers which don't seem to respond
- * with PATH_NOT_COVERED to requests that include the prefix.
- * Chase the referral if found, otherwise continue normally.
- */
- old_mountdata = cifs_sb->mountdata;
- (void)expand_dfs_referral(xid, ses, vol, cifs_sb, false);
-
- if (cifs_sb->mountdata == NULL) {
- rc = -ENOENT;
- goto error;
- }
-
- /* Save DFS root volume information for DFS refresh worker */
- origin_mountdata = kstrndup(cifs_sb->mountdata,
- strlen(cifs_sb->mountdata), GFP_KERNEL);
- if (!origin_mountdata) {
+ /* Save mount options */
+ mntdata = kstrndup(cifs_sb->mountdata, strlen(cifs_sb->mountdata), GFP_KERNEL);
+ if (!mntdata) {
rc = -ENOMEM;
goto error;
}
-
- if (cifs_sb->mountdata != old_mountdata) {
- /* If we were redirected, reconnect to new target server */
- mount_put_conns(cifs_sb, xid, server, ses, tcon);
- rc = mount_get_conns(vol, cifs_sb, &xid, &server, &ses, &tcon);
- }
- if (rc) {
- if (rc == -EACCES || rc == -EOPNOTSUPP)
- goto error;
- /* Perform DFS failover to any other DFS targets */
- rc = mount_do_dfs_failover(root_path + 1, cifs_sb, vol, NULL,
- &xid, &server, &ses, &tcon);
- if (rc)
- goto error;
- }
-
- kfree(root_path);
- root_path = build_unc_path_to_root(vol, cifs_sb, false);
- if (IS_ERR(root_path)) {
- rc = PTR_ERR(root_path);
- root_path = NULL;
+ /* Get path of DFS root */
+ ref_path = build_unc_path_to_root(vol, cifs_sb, false);
+ if (IS_ERR(ref_path)) {
+ rc = PTR_ERR(ref_path);
+ ref_path = NULL;
goto error;
}
- /* Cache out resolved root server */
- (void)dfs_cache_find(xid, ses, cifs_sb->local_nls, cifs_remap(cifs_sb),
- root_path + 1, NULL, NULL);
- kfree(root_path);
- root_path = NULL;
- set_root_tcon(cifs_sb, tcon, &root_tcon);
-
- for (count = 1; ;) {
- if (!rc && tcon) {
- rc = is_path_remote(cifs_sb, vol, xid, server, tcon);
- if (!rc || rc != -EREMOTE)
- break;
- }
- /*
- * BB: when we implement proper loop detection,
- * we will remove this check. But now we need it
- * to prevent an indefinite loop if 'DFS tree' is
- * misconfigured (i.e. has loops).
- */
- if (count++ > MAX_NESTED_LINKS) {
- rc = -ELOOP;
- break;
- }
-
+ set_root_ses(cifs_sb, ses, &root_ses);
+ do {
+ /* Save full path of last DFS path we used to resolve final target server */
kfree(full_path);
- full_path = build_unc_path_to_root(vol, cifs_sb, true);
+ full_path = build_unc_path_to_root(vol, cifs_sb, !!count);
if (IS_ERR(full_path)) {
rc = PTR_ERR(full_path);
- full_path = NULL;
break;
}
-
- old_mountdata = cifs_sb->mountdata;
- rc = expand_dfs_referral(xid, root_tcon->ses, vol, cifs_sb,
- true);
+ /* Chase referral */
+ oldmnt = cifs_sb->mountdata;
+ rc = expand_dfs_referral(xid, root_ses, vol, cifs_sb, ref_path + 1);
if (rc)
break;
-
- if (cifs_sb->mountdata != old_mountdata) {
+ /* Connect to new DFS target only if we were redirected */
+ if (oldmnt != cifs_sb->mountdata) {
mount_put_conns(cifs_sb, xid, server, ses, tcon);
- rc = mount_get_conns(vol, cifs_sb, &xid, &server, &ses,
- &tcon);
- /*
- * Ensure that DFS referrals go through new root server.
- */
- if (!rc && tcon &&
- (tcon->share_flags & (SHI1005_FLAGS_DFS |
- SHI1005_FLAGS_DFS_ROOT))) {
- cifs_put_tcon(root_tcon);
- set_root_tcon(cifs_sb, tcon, &root_tcon);
- }
+ rc = mount_get_conns(vol, cifs_sb, &xid, &server, &ses, &tcon);
}
- if (rc) {
- if (rc == -EACCES || rc == -EOPNOTSUPP)
- break;
- /* Perform DFS failover to any other DFS targets */
- rc = mount_do_dfs_failover(full_path + 1, cifs_sb, vol,
- root_tcon->ses, &xid,
- &server, &ses, &tcon);
- if (rc == -EACCES || rc == -EOPNOTSUPP || !server ||
- !ses)
- goto error;
+ if (rc && !server && !ses) {
+ /* Failed to connect. Try to connect to other targets in the referral. */
+ rc = do_dfs_failover(ref_path + 1, full_path, cifs_sb, vol, root_ses, &xid,
+ &server, &ses, &tcon);
}
- }
- cifs_put_tcon(root_tcon);
+ if (rc == -EACCES || rc == -EOPNOTSUPP || !server || !ses)
+ break;
+ if (!tcon)
+ continue;
+ /* Make sure that requests go through new root servers */
+ if (tcon->share_flags & (SHI1005_FLAGS_DFS | SHI1005_FLAGS_DFS_ROOT)) {
+ put_root_ses(root_ses);
+ set_root_ses(cifs_sb, ses, &root_ses);
+ }
+ /* Check for remaining path components and then continue chasing them (-EREMOTE) */
+ rc = check_dfs_prepath(cifs_sb, vol, xid, server, tcon, &ref_path);
+ /* Prevent recursion on broken link referrals */
+ if (rc == -EREMOTE && ++count > MAX_NESTED_LINKS)
+ rc = -ELOOP;
+ } while (rc == -EREMOTE);
if (rc)
goto error;
-
- spin_lock(&cifs_tcp_ses_lock);
- if (!tcon->dfs_path) {
- /* Save full path in new tcon to do failover when reconnecting tcons */
- tcon->dfs_path = full_path;
- full_path = NULL;
- tcon->remap = cifs_remap(cifs_sb);
- }
- cifs_sb->origin_fullpath = kstrndup(tcon->dfs_path,
- strlen(tcon->dfs_path),
- GFP_ATOMIC);
+ put_root_ses(root_ses);
+ root_ses = NULL;
+ kfree(ref_path);
+ ref_path = NULL;
+ /*
+ * Store DFS full path in both superblock and tree connect structures.
+ *
+ * For DFS root mounts, the prefix path (cifs_sb->prepath) is preserved during reconnect so
+ * only the root path is set in cifs_sb->origin_fullpath and tcon->dfs_path. And for DFS
+ * links, the prefix path is included in both and may be changed during reconnect. See
+ * cifs_tree_connect().
+ */
+ cifs_sb->origin_fullpath = kstrndup(full_path, strlen(full_path), GFP_KERNEL);
if (!cifs_sb->origin_fullpath) {
- spin_unlock(&cifs_tcp_ses_lock);
rc = -ENOMEM;
goto error;
}
+ spin_lock(&cifs_tcp_ses_lock);
+ tcon->dfs_path = full_path;
+ full_path = NULL;
+ tcon->remap = cifs_remap(cifs_sb);
spin_unlock(&cifs_tcp_ses_lock);
- rc = dfs_cache_add_vol(origin_mountdata, vol, cifs_sb->origin_fullpath);
- if (rc) {
- kfree(cifs_sb->origin_fullpath);
+ /* Add original volume information for DFS cache to be used when refreshing referrals */
+ rc = dfs_cache_add_vol(mntdata, vol, cifs_sb->origin_fullpath);
+ if (rc)
goto error;
- }
/*
* After reconnecting to a different server, unique ids won't
* match anymore, so we disable serverino. This prevents
@@ -4976,9 +4969,11 @@
return mount_setup_tlink(cifs_sb, ses, tcon);
error:
+ kfree(ref_path);
kfree(full_path);
- kfree(root_path);
- kfree(origin_mountdata);
+ kfree(mntdata);
+ kfree(cifs_sb->origin_fullpath);
+ put_root_ses(root_ses);
mount_put_conns(cifs_sb, xid, server, ses, tcon);
return rc;
}
@@ -5114,8 +5109,7 @@
bcc_ptr += strlen("?????");
bcc_ptr += 1;
count = bcc_ptr - &pSMB->Password[0];
- pSMB->hdr.smb_buf_length = cpu_to_be32(be32_to_cpu(
- pSMB->hdr.smb_buf_length) + count);
+ be32_add_cpu(&pSMB->hdr.smb_buf_length, count);
pSMB->ByteCount = cpu_to_le16(count);
rc = SendReceive(xid, ses, smb_buffer, smb_buffer_response, &length,
@@ -5533,3 +5527,115 @@
queue_delayed_work(cifsiod_wq, &cifs_sb->prune_tlinks,
TLINK_IDLE_EXPIRE);
}
+
+#ifdef CONFIG_CIFS_DFS_UPCALL
+int cifs_tree_connect(const unsigned int xid, struct cifs_tcon *tcon, const struct nls_table *nlsc)
+{
+ int rc;
+ struct TCP_Server_Info *server = tcon->ses->server;
+ const struct smb_version_operations *ops = server->ops;
+ struct dfs_cache_tgt_list tl;
+ struct dfs_cache_tgt_iterator *it = NULL;
+ char *tree;
+ const char *tcp_host;
+ size_t tcp_host_len;
+ const char *dfs_host;
+ size_t dfs_host_len;
+ char *share = NULL, *prefix = NULL;
+ struct dfs_info3_param ref = {0};
+ bool isroot;
+
+ tree = kzalloc(MAX_TREE_SIZE, GFP_KERNEL);
+ if (!tree)
+ return -ENOMEM;
+
+ if (!tcon->dfs_path) {
+ if (tcon->ipc) {
+ scnprintf(tree, MAX_TREE_SIZE, "\\\\%s\\IPC$", server->hostname);
+ rc = ops->tree_connect(xid, tcon->ses, tree, tcon, nlsc);
+ } else {
+ rc = ops->tree_connect(xid, tcon->ses, tcon->treeName, tcon, nlsc);
+ }
+ goto out;
+ }
+
+ rc = dfs_cache_noreq_find(tcon->dfs_path + 1, &ref, &tl);
+ if (rc)
+ goto out;
+ isroot = ref.server_type == DFS_TYPE_ROOT;
+ free_dfs_info_param(&ref);
+
+ extract_unc_hostname(server->hostname, &tcp_host, &tcp_host_len);
+
+ for (it = dfs_cache_get_tgt_iterator(&tl); it; it = dfs_cache_get_next_tgt(&tl, it)) {
+ bool target_match;
+
+ kfree(share);
+ kfree(prefix);
+ share = NULL;
+ prefix = NULL;
+
+ rc = dfs_cache_get_tgt_share(tcon->dfs_path + 1, it, &share, &prefix);
+ if (rc) {
+ cifs_dbg(VFS, "%s: failed to parse target share %d\n",
+ __func__, rc);
+ continue;
+ }
+
+ extract_unc_hostname(share, &dfs_host, &dfs_host_len);
+
+ if (dfs_host_len != tcp_host_len
+ || strncasecmp(dfs_host, tcp_host, dfs_host_len) != 0) {
+ cifs_dbg(FYI, "%s: %.*s doesn't match %.*s\n", __func__, (int)dfs_host_len,
+ dfs_host, (int)tcp_host_len, tcp_host);
+
+ rc = match_target_ip(server, dfs_host, dfs_host_len, &target_match);
+ if (rc) {
+ cifs_dbg(VFS, "%s: failed to match target ip: %d\n", __func__, rc);
+ break;
+ }
+
+ if (!target_match) {
+ cifs_dbg(FYI, "%s: skipping target\n", __func__);
+ continue;
+ }
+ }
+
+ if (tcon->ipc) {
+ scnprintf(tree, MAX_TREE_SIZE, "\\\\%s\\IPC$", share);
+ rc = ops->tree_connect(xid, tcon->ses, tree, tcon, nlsc);
+ } else {
+ scnprintf(tree, MAX_TREE_SIZE, "\\%s", share);
+ rc = ops->tree_connect(xid, tcon->ses, tree, tcon, nlsc);
+ /* Only handle prefix paths of DFS link targets */
+ if (!rc && !isroot) {
+ rc = update_super_prepath(tcon, prefix);
+ break;
+ }
+ }
+ if (rc == -EREMOTE)
+ break;
+ }
+
+ kfree(share);
+ kfree(prefix);
+
+ if (!rc) {
+ if (it)
+ rc = dfs_cache_noreq_update_tgthint(tcon->dfs_path + 1, it);
+ else
+ rc = -ENOENT;
+ }
+ dfs_cache_free_tgts(&tl);
+out:
+ kfree(tree);
+ return rc;
+}
+#else
+int cifs_tree_connect(const unsigned int xid, struct cifs_tcon *tcon, const struct nls_table *nlsc)
+{
+ const struct smb_version_operations *ops = tcon->ses->server->ops;
+
+ return ops->tree_connect(xid, tcon->ses, tcon->treeName, tcon, nlsc);
+}
+#endif
diff --git a/fs/cifs/dfs_cache.c b/fs/cifs/dfs_cache.c
index 7a8d414..6ee8496 100644
--- a/fs/cifs/dfs_cache.c
+++ b/fs/cifs/dfs_cache.c
@@ -29,6 +29,7 @@
struct cache_dfs_tgt {
char *name;
+ int path_consumed;
struct list_head list;
};
@@ -350,7 +351,7 @@
}
/* Allocate a new DFS target */
-static struct cache_dfs_tgt *alloc_target(const char *name)
+static struct cache_dfs_tgt *alloc_target(const char *name, int path_consumed)
{
struct cache_dfs_tgt *t;
@@ -362,6 +363,7 @@
kfree(t);
return ERR_PTR(-ENOMEM);
}
+ t->path_consumed = path_consumed;
INIT_LIST_HEAD(&t->list);
return t;
}
@@ -384,7 +386,7 @@
for (i = 0; i < numrefs; i++) {
struct cache_dfs_tgt *t;
- t = alloc_target(refs[i].node_name);
+ t = alloc_target(refs[i].node_name, refs[i].path_consumed);
if (IS_ERR(t)) {
free_tgts(ce);
return PTR_ERR(t);
@@ -490,16 +492,7 @@
return 0;
}
-/*
- * Find a DFS cache entry in hash table and optionally check prefix path against
- * @path.
- * Use whole path components in the match.
- * Must be called with htable_rw_lock held.
- *
- * Return ERR_PTR(-ENOENT) if the entry is not found.
- */
-static struct cache_entry *lookup_cache_entry(const char *path,
- unsigned int *hash)
+static struct cache_entry *__lookup_cache_entry(const char *path)
{
struct cache_entry *ce;
unsigned int h;
@@ -517,9 +510,75 @@
if (!found)
ce = ERR_PTR(-ENOENT);
+ return ce;
+}
+
+/*
+ * Find a DFS cache entry in hash table and optionally check prefix path against
+ * @path.
+ * Use whole path components in the match.
+ * Must be called with htable_rw_lock held.
+ *
+ * Return ERR_PTR(-ENOENT) if the entry is not found.
+ */
+static struct cache_entry *lookup_cache_entry(const char *path, unsigned int *hash)
+{
+ struct cache_entry *ce = ERR_PTR(-ENOENT);
+ unsigned int h;
+ int cnt = 0;
+ char *npath;
+ char *s, *e;
+ char sep;
+
+ npath = kstrndup(path, strlen(path), GFP_KERNEL);
+ if (!npath)
+ return ERR_PTR(-ENOMEM);
+
+ s = npath;
+ sep = *npath;
+ while ((s = strchr(s, sep)) && ++cnt < 3)
+ s++;
+
+ if (cnt < 3) {
+ h = cache_entry_hash(path, strlen(path));
+ ce = __lookup_cache_entry(path);
+ goto out;
+ }
+ /*
+ * Handle paths that have more than two path components and are a complete prefix of the DFS
+ * referral request path (@path).
+ *
+ * See MS-DFSC 3.2.5.5 "Receiving a Root Referral Request or Link Referral Request".
+ */
+ h = cache_entry_hash(npath, strlen(npath));
+ e = npath + strlen(npath) - 1;
+ while (e > s) {
+ char tmp;
+
+ /* skip separators */
+ while (e > s && *e == sep)
+ e--;
+ if (e == s)
+ goto out;
+
+ tmp = *(e+1);
+ *(e+1) = 0;
+
+ ce = __lookup_cache_entry(npath);
+ if (!IS_ERR(ce)) {
+ h = cache_entry_hash(npath, strlen(npath));
+ break;
+ }
+
+ *(e+1) = tmp;
+ /* backward until separator */
+ while (e > s && *e != sep)
+ e--;
+ }
+out:
if (hash)
*hash = h;
-
+ kfree(npath);
return ce;
}
@@ -773,6 +832,7 @@
rc = -ENOMEM;
goto err_free_it;
}
+ it->it_path_consumed = t->path_consumed;
if (ce->tgthint == t)
list_add(&it->it_list, head);
@@ -1263,23 +1323,26 @@
/**
* dfs_cache_get_tgt_share - parse a DFS target
*
+ * @path: DFS full path
* @it: DFS target iterator.
* @share: tree name.
- * @share_len: length of tree name.
* @prefix: prefix path.
- * @prefix_len: length of prefix path.
*
* Return zero if target was parsed correctly, otherwise non-zero.
*/
-int dfs_cache_get_tgt_share(const struct dfs_cache_tgt_iterator *it,
- const char **share, size_t *share_len,
- const char **prefix, size_t *prefix_len)
+int dfs_cache_get_tgt_share(char *path, const struct dfs_cache_tgt_iterator *it,
+ char **share, char **prefix)
{
- char *s, sep;
+ char *s, sep, *p;
+ size_t len;
+ size_t plen1, plen2;
- if (!it || !share || !share_len || !prefix || !prefix_len)
+ if (!it || !path || !share || !prefix || strlen(path) < it->it_path_consumed)
return -EINVAL;
+ *share = NULL;
+ *prefix = NULL;
+
sep = it->it_name[0];
if (sep != '\\' && sep != '/')
return -EINVAL;
@@ -1288,13 +1351,38 @@
if (!s)
return -EINVAL;
+ /* point to prefix in target node */
s = strchrnul(s + 1, sep);
- *share = it->it_name;
- *share_len = s - it->it_name;
- *prefix = *s ? s + 1 : s;
- *prefix_len = &it->it_name[strlen(it->it_name)] - *prefix;
+ /* extract target share */
+ *share = kstrndup(it->it_name, s - it->it_name, GFP_KERNEL);
+ if (!*share)
+ return -ENOMEM;
+ /* skip separator */
+ if (*s)
+ s++;
+ /* point to prefix in DFS path */
+ p = path + it->it_path_consumed;
+ if (*p == sep)
+ p++;
+
+ /* merge prefix paths from DFS path and target node */
+ plen1 = it->it_name + strlen(it->it_name) - s;
+ plen2 = path + strlen(path) - p;
+ if (plen1 || plen2) {
+ len = plen1 + plen2 + 2;
+ *prefix = kmalloc(len, GFP_KERNEL);
+ if (!*prefix) {
+ kfree(*share);
+ *share = NULL;
+ return -ENOMEM;
+ }
+ if (plen1)
+ scnprintf(*prefix, len, "%.*s%c%.*s", (int)plen1, s, sep, (int)plen2, p);
+ else
+ strscpy(*prefix, p, len);
+ }
return 0;
}
diff --git a/fs/cifs/dfs_cache.h b/fs/cifs/dfs_cache.h
index bf94d08..3d7c051 100644
--- a/fs/cifs/dfs_cache.h
+++ b/fs/cifs/dfs_cache.h
@@ -19,6 +19,7 @@
struct dfs_cache_tgt_iterator {
char *it_name;
+ int it_path_consumed;
struct list_head it_list;
};
@@ -48,10 +49,8 @@
extern int dfs_cache_update_vol(const char *fullpath,
struct TCP_Server_Info *server);
extern void dfs_cache_del_vol(const char *fullpath);
-
-extern int dfs_cache_get_tgt_share(const struct dfs_cache_tgt_iterator *it,
- const char **share, size_t *share_len,
- const char **prefix, size_t *prefix_len);
+extern int dfs_cache_get_tgt_share(char *path, const struct dfs_cache_tgt_iterator *it,
+ char **share, char **prefix);
static inline struct dfs_cache_tgt_iterator *
dfs_cache_get_next_tgt(struct dfs_cache_tgt_list *tl,
diff --git a/fs/cifs/inode.c b/fs/cifs/inode.c
index ce95801..3989d08 100644
--- a/fs/cifs/inode.c
+++ b/fs/cifs/inode.c
@@ -1086,7 +1086,6 @@
struct super_block *sb, unsigned int xid)
{
struct cifs_tcon *tcon;
- struct TCP_Server_Info *server;
struct tcon_link *tlink;
struct cifs_sb_info *cifs_sb = CIFS_SB(sb);
bool adjust_tz = false;
@@ -1100,7 +1099,6 @@
if (IS_ERR(tlink))
return PTR_ERR(tlink);
tcon = tlink_tcon(tlink);
- server = tcon->ses->server;
/*
* 1. Fetch file metadata
diff --git a/fs/cifs/misc.c b/fs/cifs/misc.c
index 3e377244..1c14cf0 100644
--- a/fs/cifs/misc.c
+++ b/fs/cifs/misc.c
@@ -1164,8 +1164,7 @@
}
#endif
-int update_super_prepath(struct cifs_tcon *tcon, const char *prefix,
- size_t prefix_len)
+int update_super_prepath(struct cifs_tcon *tcon, char *prefix)
{
struct super_block *sb;
struct cifs_sb_info *cifs_sb;
@@ -1179,8 +1178,8 @@
kfree(cifs_sb->prepath);
- if (*prefix && prefix_len) {
- cifs_sb->prepath = kstrndup(prefix, prefix_len, GFP_ATOMIC);
+ if (prefix && *prefix) {
+ cifs_sb->prepath = kstrndup(prefix, strlen(prefix), GFP_ATOMIC);
if (!cifs_sb->prepath) {
rc = -ENOMEM;
goto out;
diff --git a/fs/cifs/netmisc.c b/fs/cifs/netmisc.c
index b7ca496..0e728aa 100644
--- a/fs/cifs/netmisc.c
+++ b/fs/cifs/netmisc.c
@@ -881,6 +881,33 @@
return rc;
}
+int
+map_and_check_smb_error(struct mid_q_entry *mid, bool logErr)
+{
+ int rc;
+ struct smb_hdr *smb = (struct smb_hdr *)mid->resp_buf;
+
+ rc = map_smb_to_linux_error((char *)smb, logErr);
+ if (rc == -EACCES && !(smb->Flags2 & SMBFLG2_ERR_STATUS)) {
+ /* possible ERRBaduid */
+ __u8 class = smb->Status.DosError.ErrorClass;
+ __u16 code = le16_to_cpu(smb->Status.DosError.Error);
+
+ /* switch can be used to handle different errors */
+ if (class == ERRSRV && code == ERRbaduid) {
+ cifs_dbg(FYI, "Server returned 0x%x, reconnecting session...\n",
+ code);
+ spin_lock(&GlobalMid_Lock);
+ if (mid->server->tcpStatus != CifsExiting)
+ mid->server->tcpStatus = CifsNeedReconnect;
+ spin_unlock(&GlobalMid_Lock);
+ }
+ }
+
+ return rc;
+}
+
+
/*
* calculate the size of the SMB message based on the fixed header
* portion, the number of word parameters and the data portion of the message
diff --git a/fs/cifs/sess.c b/fs/cifs/sess.c
index 5d05bd2..69cd585 100644
--- a/fs/cifs/sess.c
+++ b/fs/cifs/sess.c
@@ -938,8 +938,7 @@
struct kvec rsp_iov = { NULL, 0 };
count = sess_data->iov[1].iov_len + sess_data->iov[2].iov_len;
- smb_buf->smb_buf_length =
- cpu_to_be32(be32_to_cpu(smb_buf->smb_buf_length) + count);
+ be32_add_cpu(&smb_buf->smb_buf_length, count);
put_bcc(count, smb_buf);
rc = SendReceive2(sess_data->xid, sess_data->ses,
@@ -1705,7 +1704,6 @@
#else
cifs_dbg(VFS, "Kerberos negotiated but upcall support disabled!\n");
return -ENOSYS;
- break;
#endif /* CONFIG_CIFS_UPCALL */
case RawNTLMSSP:
sess_data->func = sess_auth_rawntlmssp_negotiate;
diff --git a/fs/cifs/smb1ops.c b/fs/cifs/smb1ops.c
index 197ed45..80287c2 100644
--- a/fs/cifs/smb1ops.c
+++ b/fs/cifs/smb1ops.c
@@ -688,7 +688,7 @@
dosattrs = cifsInode->cifsAttrs|ATTR_READONLY;
info.Attributes = cpu_to_le32(dosattrs);
rc = CIFSSMBSetPathInfo(xid, tcon, full_path, &info, cifs_sb->local_nls,
- cifs_remap(cifs_sb));
+ cifs_sb);
if (rc == 0)
cifsInode->cifsAttrs = dosattrs;
}
@@ -783,7 +783,7 @@
tcon = tlink_tcon(tlink);
rc = CIFSSMBSetPathInfo(xid, tcon, full_path, buf, cifs_sb->local_nls,
- cifs_remap(cifs_sb));
+ cifs_sb);
if (rc == 0) {
cinode->cifsAttrs = le32_to_cpu(buf->Attributes);
goto out;
diff --git a/fs/cifs/smb2misc.c b/fs/cifs/smb2misc.c
index 1579928..d88e2683 100644
--- a/fs/cifs/smb2misc.c
+++ b/fs/cifs/smb2misc.c
@@ -508,15 +508,31 @@
kfree(lw);
}
-static bool
-smb2_tcon_has_lease(struct cifs_tcon *tcon, struct smb2_lease_break *rsp,
- struct smb2_lease_break_work *lw)
+static void
+smb2_queue_pending_open_break(struct tcon_link *tlink, __u8 *lease_key,
+ __le32 new_lease_state)
{
- bool found;
+ struct smb2_lease_break_work *lw;
+
+ lw = kmalloc(sizeof(struct smb2_lease_break_work), GFP_KERNEL);
+ if (!lw) {
+ cifs_put_tlink(tlink);
+ return;
+ }
+
+ INIT_WORK(&lw->lease_break, cifs_ses_oplock_break);
+ lw->tlink = tlink;
+ lw->lease_state = new_lease_state;
+ memcpy(lw->lease_key, lease_key, SMB2_LEASE_KEY_SIZE);
+ queue_work(cifsiod_wq, &lw->lease_break);
+}
+
+static bool
+smb2_tcon_has_lease(struct cifs_tcon *tcon, struct smb2_lease_break *rsp)
+{
__u8 lease_state;
struct list_head *tmp;
struct cifsFileInfo *cfile;
- struct cifs_pending_open *open;
struct cifsInodeInfo *cinode;
int ack_req = le32_to_cpu(rsp->Flags &
SMB2_NOTIFY_BREAK_LEASE_FLAG_ACK_REQUIRED);
@@ -546,22 +562,29 @@
cfile->oplock_level = lease_state;
cifs_queue_oplock_break(cfile);
- kfree(lw);
return true;
}
- found = false;
+ return false;
+}
+
+static struct cifs_pending_open *
+smb2_tcon_find_pending_open_lease(struct cifs_tcon *tcon,
+ struct smb2_lease_break *rsp)
+{
+ __u8 lease_state = le32_to_cpu(rsp->NewLeaseState);
+ int ack_req = le32_to_cpu(rsp->Flags &
+ SMB2_NOTIFY_BREAK_LEASE_FLAG_ACK_REQUIRED);
+ struct cifs_pending_open *open;
+ struct cifs_pending_open *found = NULL;
+
list_for_each_entry(open, &tcon->pending_opens, olist) {
if (memcmp(open->lease_key, rsp->LeaseKey,
SMB2_LEASE_KEY_SIZE))
continue;
if (!found && ack_req) {
- found = true;
- memcpy(lw->lease_key, open->lease_key,
- SMB2_LEASE_KEY_SIZE);
- lw->tlink = cifs_get_tlink(open->tlink);
- queue_work(cifsiod_wq, &lw->lease_break);
+ found = open;
}
cifs_dbg(FYI, "found in the pending open list\n");
@@ -582,14 +605,7 @@
struct TCP_Server_Info *server;
struct cifs_ses *ses;
struct cifs_tcon *tcon;
- struct smb2_lease_break_work *lw;
-
- lw = kmalloc(sizeof(struct smb2_lease_break_work), GFP_KERNEL);
- if (!lw)
- return false;
-
- INIT_WORK(&lw->lease_break, cifs_ses_oplock_break);
- lw->lease_state = rsp->NewLeaseState;
+ struct cifs_pending_open *open;
cifs_dbg(FYI, "Checking for lease break\n");
@@ -607,11 +623,27 @@
spin_lock(&tcon->open_file_lock);
cifs_stats_inc(
&tcon->stats.cifs_stats.num_oplock_brks);
- if (smb2_tcon_has_lease(tcon, rsp, lw)) {
+ if (smb2_tcon_has_lease(tcon, rsp)) {
spin_unlock(&tcon->open_file_lock);
spin_unlock(&cifs_tcp_ses_lock);
return true;
}
+ open = smb2_tcon_find_pending_open_lease(tcon,
+ rsp);
+ if (open) {
+ __u8 lease_key[SMB2_LEASE_KEY_SIZE];
+ struct tcon_link *tlink;
+
+ tlink = cifs_get_tlink(open->tlink);
+ memcpy(lease_key, open->lease_key,
+ SMB2_LEASE_KEY_SIZE);
+ spin_unlock(&tcon->open_file_lock);
+ spin_unlock(&cifs_tcp_ses_lock);
+ smb2_queue_pending_open_break(tlink,
+ lease_key,
+ rsp->NewLeaseState);
+ return true;
+ }
spin_unlock(&tcon->open_file_lock);
if (tcon->crfid.is_valid &&
@@ -629,7 +661,6 @@
}
}
spin_unlock(&cifs_tcp_ses_lock);
- kfree(lw);
cifs_dbg(FYI, "Can not process lease break - no lease matched\n");
return false;
}
diff --git a/fs/cifs/smb2pdu.c b/fs/cifs/smb2pdu.c
index 2f4cdd2..24c2ac3 100644
--- a/fs/cifs/smb2pdu.c
+++ b/fs/cifs/smb2pdu.c
@@ -152,117 +152,6 @@
return;
}
-#ifdef CONFIG_CIFS_DFS_UPCALL
-static int __smb2_reconnect(const struct nls_table *nlsc,
- struct cifs_tcon *tcon)
-{
- int rc;
- struct TCP_Server_Info *server = tcon->ses->server;
- struct dfs_cache_tgt_list tl;
- struct dfs_cache_tgt_iterator *it = NULL;
- char *tree;
- const char *tcp_host;
- size_t tcp_host_len;
- const char *dfs_host;
- size_t dfs_host_len;
-
- tree = kzalloc(MAX_TREE_SIZE, GFP_KERNEL);
- if (!tree)
- return -ENOMEM;
-
- if (!tcon->dfs_path) {
- if (tcon->ipc) {
- scnprintf(tree, MAX_TREE_SIZE, "\\\\%s\\IPC$",
- server->hostname);
- rc = SMB2_tcon(0, tcon->ses, tree, tcon, nlsc);
- } else {
- rc = SMB2_tcon(0, tcon->ses, tcon->treeName, tcon,
- nlsc);
- }
- goto out;
- }
-
- rc = dfs_cache_noreq_find(tcon->dfs_path + 1, NULL, &tl);
- if (rc)
- goto out;
-
- extract_unc_hostname(server->hostname, &tcp_host, &tcp_host_len);
-
- for (it = dfs_cache_get_tgt_iterator(&tl); it;
- it = dfs_cache_get_next_tgt(&tl, it)) {
- const char *share, *prefix;
- size_t share_len, prefix_len;
- bool target_match;
-
- rc = dfs_cache_get_tgt_share(it, &share, &share_len, &prefix,
- &prefix_len);
- if (rc) {
- cifs_dbg(VFS, "%s: failed to parse target share %d\n",
- __func__, rc);
- continue;
- }
-
- extract_unc_hostname(share, &dfs_host, &dfs_host_len);
-
- if (dfs_host_len != tcp_host_len
- || strncasecmp(dfs_host, tcp_host, dfs_host_len) != 0) {
- cifs_dbg(FYI, "%s: %.*s doesn't match %.*s\n",
- __func__,
- (int)dfs_host_len, dfs_host,
- (int)tcp_host_len, tcp_host);
-
- rc = match_target_ip(server, dfs_host, dfs_host_len,
- &target_match);
- if (rc) {
- cifs_dbg(VFS, "%s: failed to match target ip: %d\n",
- __func__, rc);
- break;
- }
-
- if (!target_match) {
- cifs_dbg(FYI, "%s: skipping target\n", __func__);
- continue;
- }
- }
-
- if (tcon->ipc) {
- scnprintf(tree, MAX_TREE_SIZE, "\\\\%.*s\\IPC$",
- (int)share_len, share);
- rc = SMB2_tcon(0, tcon->ses, tree, tcon, nlsc);
- } else {
- scnprintf(tree, MAX_TREE_SIZE, "\\%.*s", (int)share_len,
- share);
- rc = SMB2_tcon(0, tcon->ses, tree, tcon, nlsc);
- if (!rc) {
- rc = update_super_prepath(tcon, prefix,
- prefix_len);
- break;
- }
- }
- if (rc == -EREMOTE)
- break;
- }
-
- if (!rc) {
- if (it)
- rc = dfs_cache_noreq_update_tgthint(tcon->dfs_path + 1,
- it);
- else
- rc = -ENOENT;
- }
- dfs_cache_free_tgts(&tl);
-out:
- kfree(tree);
- return rc;
-}
-#else
-static inline int __smb2_reconnect(const struct nls_table *nlsc,
- struct cifs_tcon *tcon)
-{
- return SMB2_tcon(0, tcon->ses, tcon->treeName, tcon, nlsc);
-}
-#endif
-
static int
smb2_reconnect(__le16 smb2_command, struct cifs_tcon *tcon,
struct TCP_Server_Info *server)
@@ -409,7 +298,7 @@
if (tcon->use_persistent)
tcon->need_reopen_files = true;
- rc = __smb2_reconnect(nls_codepage, tcon);
+ rc = cifs_tree_connect(0, tcon, nls_codepage);
mutex_unlock(&tcon->ses->session_mutex);
cifs_dbg(FYI, "reconnect tcon rc = %d\n", rc);
@@ -1387,6 +1276,8 @@
spnego_key = cifs_get_spnego_key(ses);
if (IS_ERR(spnego_key)) {
rc = PTR_ERR(spnego_key);
+ if (rc == -ENOKEY)
+ cifs_dbg(VFS, "Verify user has a krb5 ticket and keyutils is installed\n");
spnego_key = NULL;
goto out;
}
diff --git a/fs/cifs/smb2pdu.h b/fs/cifs/smb2pdu.h
index 236814d..c3f1baf 100644
--- a/fs/cifs/smb2pdu.h
+++ b/fs/cifs/smb2pdu.h
@@ -31,7 +31,7 @@
* Note that, due to trying to use names similar to the protocol specifications,
* there are many mixed case field names in the structures below. Although
* this does not match typical Linux kernel style, it is necessary to be
- * be able to match against the protocol specfication.
+ * able to match against the protocol specfication.
*
* SMB2 commands
* Some commands have minimal (wct=0,bcc=0), or uninteresting, responses
diff --git a/fs/cifs/transport.c b/fs/cifs/transport.c
index 84433d0..ac76324 100644
--- a/fs/cifs/transport.c
+++ b/fs/cifs/transport.c
@@ -936,7 +936,7 @@
}
/* BB special case reconnect tid and uid here? */
- return map_smb_to_linux_error(mid->resp_buf, log_error);
+ return map_and_check_smb_error(mid, log_error);
}
struct mid_q_entry *
diff --git a/fs/dlm/config.c b/fs/dlm/config.c
index 3b21082..47f0b98 100644
--- a/fs/dlm/config.c
+++ b/fs/dlm/config.c
@@ -73,6 +73,7 @@
unsigned int cl_log_debug;
unsigned int cl_log_info;
unsigned int cl_protocol;
+ unsigned int cl_mark;
unsigned int cl_timewarn_cs;
unsigned int cl_waitwarn_us;
unsigned int cl_new_rsb_count;
@@ -96,6 +97,7 @@
CLUSTER_ATTR_LOG_DEBUG,
CLUSTER_ATTR_LOG_INFO,
CLUSTER_ATTR_PROTOCOL,
+ CLUSTER_ATTR_MARK,
CLUSTER_ATTR_TIMEWARN_CS,
CLUSTER_ATTR_WAITWARN_US,
CLUSTER_ATTR_NEW_RSB_COUNT,
@@ -168,6 +170,7 @@
CLUSTER_ATTR(log_debug, 0);
CLUSTER_ATTR(log_info, 0);
CLUSTER_ATTR(protocol, 0);
+CLUSTER_ATTR(mark, 0);
CLUSTER_ATTR(timewarn_cs, 1);
CLUSTER_ATTR(waitwarn_us, 0);
CLUSTER_ATTR(new_rsb_count, 0);
@@ -183,6 +186,7 @@
[CLUSTER_ATTR_LOG_DEBUG] = &cluster_attr_log_debug,
[CLUSTER_ATTR_LOG_INFO] = &cluster_attr_log_info,
[CLUSTER_ATTR_PROTOCOL] = &cluster_attr_protocol,
+ [CLUSTER_ATTR_MARK] = &cluster_attr_mark,
[CLUSTER_ATTR_TIMEWARN_CS] = &cluster_attr_timewarn_cs,
[CLUSTER_ATTR_WAITWARN_US] = &cluster_attr_waitwarn_us,
[CLUSTER_ATTR_NEW_RSB_COUNT] = &cluster_attr_new_rsb_count,
@@ -196,6 +200,7 @@
COMM_ATTR_LOCAL,
COMM_ATTR_ADDR,
COMM_ATTR_ADDR_LIST,
+ COMM_ATTR_MARK,
};
enum {
@@ -228,6 +233,7 @@
int nodeid;
int local;
int addr_count;
+ unsigned int mark;
struct sockaddr_storage *addr[DLM_MAX_ADDR_COUNT];
};
@@ -465,6 +471,7 @@
cm->nodeid = -1;
cm->local = 0;
cm->addr_count = 0;
+ cm->mark = 0;
return &cm->item;
}
@@ -660,8 +667,28 @@
return 4096 - allowance;
}
+static ssize_t comm_mark_show(struct config_item *item, char *buf)
+{
+ return sprintf(buf, "%u\n", config_item_to_comm(item)->mark);
+}
+
+static ssize_t comm_mark_store(struct config_item *item, const char *buf,
+ size_t len)
+{
+ unsigned int mark;
+ int rc;
+
+ rc = kstrtouint(buf, 0, &mark);
+ if (rc)
+ return rc;
+
+ config_item_to_comm(item)->mark = mark;
+ return len;
+}
+
CONFIGFS_ATTR(comm_, nodeid);
CONFIGFS_ATTR(comm_, local);
+CONFIGFS_ATTR(comm_, mark);
CONFIGFS_ATTR_WO(comm_, addr);
CONFIGFS_ATTR_RO(comm_, addr_list);
@@ -670,6 +697,7 @@
[COMM_ATTR_LOCAL] = &comm_attr_local,
[COMM_ATTR_ADDR] = &comm_attr_addr,
[COMM_ATTR_ADDR_LIST] = &comm_attr_addr_list,
+ [COMM_ATTR_MARK] = &comm_attr_mark,
NULL,
};
@@ -829,6 +857,20 @@
return 0;
}
+int dlm_comm_mark(int nodeid, unsigned int *mark)
+{
+ struct dlm_comm *cm;
+
+ cm = get_comm(nodeid);
+ if (!cm)
+ return -ENOENT;
+
+ *mark = cm->mark;
+ put_comm(cm);
+
+ return 0;
+}
+
int dlm_our_nodeid(void)
{
return local_comm ? local_comm->nodeid : 0;
@@ -855,6 +897,7 @@
#define DEFAULT_LOG_DEBUG 0
#define DEFAULT_LOG_INFO 1
#define DEFAULT_PROTOCOL 0
+#define DEFAULT_MARK 0
#define DEFAULT_TIMEWARN_CS 500 /* 5 sec = 500 centiseconds */
#define DEFAULT_WAITWARN_US 0
#define DEFAULT_NEW_RSB_COUNT 128
@@ -871,6 +914,7 @@
.ci_log_debug = DEFAULT_LOG_DEBUG,
.ci_log_info = DEFAULT_LOG_INFO,
.ci_protocol = DEFAULT_PROTOCOL,
+ .ci_mark = DEFAULT_MARK,
.ci_timewarn_cs = DEFAULT_TIMEWARN_CS,
.ci_waitwarn_us = DEFAULT_WAITWARN_US,
.ci_new_rsb_count = DEFAULT_NEW_RSB_COUNT,
diff --git a/fs/dlm/config.h b/fs/dlm/config.h
index 2b471aa..f62996c 100644
--- a/fs/dlm/config.h
+++ b/fs/dlm/config.h
@@ -31,6 +31,7 @@
int ci_log_debug;
int ci_log_info;
int ci_protocol;
+ int ci_mark;
int ci_timewarn_cs;
int ci_waitwarn_us;
int ci_new_rsb_count;
@@ -45,6 +46,7 @@
int dlm_config_nodes(char *lsname, struct dlm_config_node **nodes_out,
int *count_out);
int dlm_comm_seq(int nodeid, uint32_t *seq);
+int dlm_comm_mark(int nodeid, unsigned int *mark);
int dlm_our_nodeid(void);
int dlm_our_addr(struct sockaddr_storage *addr, int num);
diff --git a/fs/dlm/lockspace.c b/fs/dlm/lockspace.c
index e93670e..624617c 100644
--- a/fs/dlm/lockspace.c
+++ b/fs/dlm/lockspace.c
@@ -622,6 +622,9 @@
wait_event(ls->ls_recover_lock_wait,
test_bit(LSFL_RECOVER_LOCK, &ls->ls_flags));
+ /* let kobject handle freeing of ls if there's an error */
+ do_unreg = 1;
+
ls->ls_kobj.kset = dlm_kset;
error = kobject_init_and_add(&ls->ls_kobj, &dlm_ktype, NULL,
"%s", ls->ls_name);
@@ -629,9 +632,6 @@
goto out_recoverd;
kobject_uevent(&ls->ls_kobj, KOBJ_ADD);
- /* let kobject handle freeing of ls if there's an error */
- do_unreg = 1;
-
/* This uevent triggers dlm_controld in userspace to add us to the
group of nodes that are members of this lockspace (managed by the
cluster infrastructure.) Once it's done that, it tells us who the
diff --git a/fs/dlm/lowcomms.c b/fs/dlm/lowcomms.c
index 3543a8f..5050fe0 100644
--- a/fs/dlm/lowcomms.c
+++ b/fs/dlm/lowcomms.c
@@ -63,6 +63,7 @@
/* Number of messages to send before rescheduling */
#define MAX_SEND_MSG_COUNT 25
+#define DLM_SHUTDOWN_WAIT_TIMEOUT msecs_to_jiffies(10000)
struct cbuf {
unsigned int base;
@@ -110,10 +111,12 @@
#define CF_CLOSE 6
#define CF_APP_LIMITED 7
#define CF_CLOSING 8
+#define CF_SHUTDOWN 9
struct list_head writequeue; /* List of outgoing writequeue_entries */
spinlock_t writequeue_lock;
int (*rx_action) (struct connection *); /* What to do when active */
void (*connect_action) (struct connection *); /* What to do to connect */
+ void (*shutdown_action)(struct connection *con); /* What to do to shutdown */
struct page *rx_page;
struct cbuf cb;
int retries;
@@ -122,6 +125,7 @@
struct connection *othercon;
struct work_struct rwork; /* Receive workqueue */
struct work_struct swork; /* Send workqueue */
+ wait_queue_head_t shutdown_wait; /* wait for graceful shutdown */
};
#define sock2con(x) ((struct connection *)(x)->sk_user_data)
@@ -218,6 +222,7 @@
spin_lock_init(&con->writequeue_lock);
INIT_WORK(&con->swork, process_send_sockets);
INIT_WORK(&con->rwork, process_recv_sockets);
+ init_waitqueue_head(&con->shutdown_wait);
/* Setup action pointers for child sockets */
if (con->nodeid) {
@@ -619,6 +624,54 @@
clear_bit(CF_CLOSING, &con->flags);
}
+static void shutdown_connection(struct connection *con)
+{
+ int ret;
+
+ if (cancel_work_sync(&con->swork)) {
+ log_print("canceled swork for node %d", con->nodeid);
+ clear_bit(CF_WRITE_PENDING, &con->flags);
+ }
+
+ mutex_lock(&con->sock_mutex);
+ /* nothing to shutdown */
+ if (!con->sock) {
+ mutex_unlock(&con->sock_mutex);
+ return;
+ }
+
+ set_bit(CF_SHUTDOWN, &con->flags);
+ ret = kernel_sock_shutdown(con->sock, SHUT_WR);
+ mutex_unlock(&con->sock_mutex);
+ if (ret) {
+ log_print("Connection %p failed to shutdown: %d will force close",
+ con, ret);
+ goto force_close;
+ } else {
+ ret = wait_event_timeout(con->shutdown_wait,
+ !test_bit(CF_SHUTDOWN, &con->flags),
+ DLM_SHUTDOWN_WAIT_TIMEOUT);
+ if (ret == 0) {
+ log_print("Connection %p shutdown timed out, will force close",
+ con);
+ goto force_close;
+ }
+ }
+
+ return;
+
+force_close:
+ clear_bit(CF_SHUTDOWN, &con->flags);
+ close_connection(con, false, true, true);
+}
+
+static void dlm_tcp_shutdown(struct connection *con)
+{
+ if (con->othercon)
+ shutdown_connection(con->othercon);
+ shutdown_connection(con);
+}
+
/* Data received from remote end */
static int receive_from_sock(struct connection *con)
{
@@ -685,14 +738,14 @@
page_address(con->rx_page),
con->cb.base, con->cb.len,
PAGE_SIZE);
- if (ret == -EBADMSG) {
- log_print("lowcomms: addr=%p, base=%u, len=%u, read=%d",
- page_address(con->rx_page), con->cb.base,
+ if (ret < 0) {
+ log_print("lowcomms err %d: addr=%p, base=%u, len=%u, read=%d",
+ ret, page_address(con->rx_page), con->cb.base,
con->cb.len, r);
+ cbuf_eat(&con->cb, r);
+ } else {
+ cbuf_eat(&con->cb, ret);
}
- if (ret < 0)
- goto out_close;
- cbuf_eat(&con->cb, ret);
if (cbuf_empty(&con->cb) && !call_again_soon) {
__free_page(con->rx_page);
@@ -713,13 +766,18 @@
out_close:
mutex_unlock(&con->sock_mutex);
if (ret != -EAGAIN) {
- close_connection(con, true, true, false);
/* Reconnect when there is something to send */
+ close_connection(con, false, true, false);
+ if (ret == 0) {
+ log_print("connection %p got EOF from %d",
+ con, con->nodeid);
+ /* handling for tcp shutdown */
+ clear_bit(CF_SHUTDOWN, &con->flags);
+ wake_up(&con->shutdown_wait);
+ /* signal to breaking receive worker */
+ ret = -1;
+ }
}
- /* Don't return success if we really got EOF */
- if (ret == 0)
- ret = -EAGAIN;
-
return ret;
}
@@ -803,22 +861,18 @@
spin_lock_init(&othercon->writequeue_lock);
INIT_WORK(&othercon->swork, process_send_sockets);
INIT_WORK(&othercon->rwork, process_recv_sockets);
+ init_waitqueue_head(&othercon->shutdown_wait);
set_bit(CF_IS_OTHERCON, &othercon->flags);
+ } else {
+ /* close other sock con if we have something new */
+ close_connection(othercon, false, true, false);
}
+
mutex_lock_nested(&othercon->sock_mutex, 2);
- if (!othercon->sock) {
- newcon->othercon = othercon;
- add_sock(newsock, othercon);
- addcon = othercon;
- mutex_unlock(&othercon->sock_mutex);
- }
- else {
- printk("Extra connection from node %d attempted\n", nodeid);
- result = -EAGAIN;
- mutex_unlock(&othercon->sock_mutex);
- mutex_unlock(&newcon->sock_mutex);
- goto accept_err;
- }
+ newcon->othercon = othercon;
+ add_sock(newsock, othercon);
+ addcon = othercon;
+ mutex_unlock(&othercon->sock_mutex);
}
else {
newcon->rx_action = receive_from_sock;
@@ -914,6 +968,7 @@
int result;
int addr_len;
struct socket *sock;
+ unsigned int mark;
if (con->nodeid == 0) {
log_print("attempt to connect sock 0 foiled");
@@ -944,6 +999,13 @@
if (result < 0)
goto socket_err;
+ /* set skb mark */
+ result = dlm_comm_mark(con->nodeid, &mark);
+ if (result < 0)
+ goto bind_err;
+
+ sock_set_mark(sock->sk, mark);
+
con->rx_action = receive_from_sock;
con->connect_action = sctp_connect_to_sock;
add_sock(sock, con);
@@ -1006,6 +1068,7 @@
struct sockaddr_storage saddr, src_addr;
int addr_len;
struct socket *sock = NULL;
+ unsigned int mark;
int result;
if (con->nodeid == 0) {
@@ -1027,6 +1090,13 @@
if (result < 0)
goto out_err;
+ /* set skb mark */
+ result = dlm_comm_mark(con->nodeid, &mark);
+ if (result < 0)
+ goto out_err;
+
+ sock_set_mark(sock->sk, mark);
+
memset(&saddr, 0, sizeof(saddr));
result = nodeid_to_addr(con->nodeid, &saddr, NULL, false);
if (result < 0) {
@@ -1036,6 +1106,7 @@
con->rx_action = receive_from_sock;
con->connect_action = tcp_connect_to_sock;
+ con->shutdown_action = dlm_tcp_shutdown;
add_sock(sock, con);
/* Bind to our cluster-known address connecting to avoid
@@ -1111,6 +1182,8 @@
goto create_out;
}
+ sock_set_mark(sock->sk, dlm_config.ci_mark);
+
/* Turn off Nagle's algorithm */
tcp_sock_set_nodelay(sock->sk);
@@ -1185,6 +1258,7 @@
}
sock_set_rcvbuf(sock->sk, NEEDED_RMEM);
+ sock_set_mark(sock->sk, dlm_config.ci_mark);
sctp_sock_set_nodelay(sock->sk);
write_lock_bh(&sock->sk->sk_callback_lock);
@@ -1396,7 +1470,7 @@
send_error:
mutex_unlock(&con->sock_mutex);
- close_connection(con, true, false, true);
+ close_connection(con, false, false, true);
/* Requeue the send work. When the work daemon runs again, it will try
a new connection, then call this function again. */
queue_work(send_workqueue, &con->swork);
@@ -1528,6 +1602,12 @@
_stop_conn(con, true);
}
+static void shutdown_conn(struct connection *con)
+{
+ if (con->shutdown_action)
+ con->shutdown_action(con);
+}
+
static void free_conn(struct connection *con)
{
close_connection(con, true, true, true);
@@ -1579,6 +1659,7 @@
mutex_lock(&connections_lock);
dlm_allow_conn = 0;
mutex_unlock(&connections_lock);
+ foreach_conn(shutdown_conn);
work_flush();
clean_writequeues();
foreach_conn(free_conn);
diff --git a/fs/erofs/compress.h b/fs/erofs/compress.h
index 07d279f..3d45244 100644
--- a/fs/erofs/compress.h
+++ b/fs/erofs/compress.h
@@ -1,7 +1,7 @@
/* SPDX-License-Identifier: GPL-2.0-only */
/*
* Copyright (C) 2019 HUAWEI, Inc.
- * http://www.huawei.com/
+ * https://www.huawei.com/
* Created by Gao Xiang <gaoxiang25@huawei.com>
*/
#ifndef __EROFS_FS_COMPRESS_H
diff --git a/fs/erofs/data.c b/fs/erofs/data.c
index d054215..459ecb4 100644
--- a/fs/erofs/data.c
+++ b/fs/erofs/data.c
@@ -1,7 +1,7 @@
// SPDX-License-Identifier: GPL-2.0-only
/*
* Copyright (C) 2017-2018 HUAWEI, Inc.
- * http://www.huawei.com/
+ * https://www.huawei.com/
* Created by Gao Xiang <gaoxiang25@huawei.com>
*/
#include "internal.h"
diff --git a/fs/erofs/decompressor.c b/fs/erofs/decompressor.c
index 7628816..cbadbf5 100644
--- a/fs/erofs/decompressor.c
+++ b/fs/erofs/decompressor.c
@@ -1,7 +1,7 @@
// SPDX-License-Identifier: GPL-2.0-only
/*
* Copyright (C) 2019 HUAWEI, Inc.
- * http://www.huawei.com/
+ * https://www.huawei.com/
* Created by Gao Xiang <gaoxiang25@huawei.com>
*/
#include "compress.h"
diff --git a/fs/erofs/dir.c b/fs/erofs/dir.c
index d28c623..2776bb8 100644
--- a/fs/erofs/dir.c
+++ b/fs/erofs/dir.c
@@ -1,7 +1,7 @@
// SPDX-License-Identifier: GPL-2.0-only
/*
* Copyright (C) 2017-2018 HUAWEI, Inc.
- * http://www.huawei.com/
+ * https://www.huawei.com/
* Created by Gao Xiang <gaoxiang25@huawei.com>
*/
#include "internal.h"
diff --git a/fs/erofs/erofs_fs.h b/fs/erofs/erofs_fs.h
index 385fa49c..9ad1615f 100644
--- a/fs/erofs/erofs_fs.h
+++ b/fs/erofs/erofs_fs.h
@@ -3,7 +3,7 @@
* EROFS (Enhanced ROM File System) on-disk format definition
*
* Copyright (C) 2017-2018 HUAWEI, Inc.
- * http://www.huawei.com/
+ * https://www.huawei.com/
* Created by Gao Xiang <gaoxiang25@huawei.com>
*/
#ifndef __EROFS_FS_H
diff --git a/fs/erofs/inode.c b/fs/erofs/inode.c
index 7dd4bbe..139d0be 100644
--- a/fs/erofs/inode.c
+++ b/fs/erofs/inode.c
@@ -1,38 +1,87 @@
// SPDX-License-Identifier: GPL-2.0-only
/*
* Copyright (C) 2017-2018 HUAWEI, Inc.
- * http://www.huawei.com/
+ * https://www.huawei.com/
* Created by Gao Xiang <gaoxiang25@huawei.com>
*/
#include "xattr.h"
#include <trace/events/erofs.h>
-/* no locking */
-static int erofs_read_inode(struct inode *inode, void *data)
+/*
+ * if inode is successfully read, return its inode page (or sometimes
+ * the inode payload page if it's an extended inode) in order to fill
+ * inline data if possible.
+ */
+static struct page *erofs_read_inode(struct inode *inode,
+ unsigned int *ofs)
{
+ struct super_block *sb = inode->i_sb;
+ struct erofs_sb_info *sbi = EROFS_SB(sb);
struct erofs_inode *vi = EROFS_I(inode);
- struct erofs_inode_compact *dic = data;
- struct erofs_inode_extended *die;
+ const erofs_off_t inode_loc = iloc(sbi, vi->nid);
- const unsigned int ifmt = le16_to_cpu(dic->i_format);
- struct erofs_sb_info *sbi = EROFS_SB(inode->i_sb);
- erofs_blk_t nblks = 0;
+ erofs_blk_t blkaddr, nblks = 0;
+ struct page *page;
+ struct erofs_inode_compact *dic;
+ struct erofs_inode_extended *die, *copied = NULL;
+ unsigned int ifmt;
+ int err;
+
+ blkaddr = erofs_blknr(inode_loc);
+ *ofs = erofs_blkoff(inode_loc);
+
+ erofs_dbg("%s, reading inode nid %llu at %u of blkaddr %u",
+ __func__, vi->nid, *ofs, blkaddr);
+
+ page = erofs_get_meta_page(sb, blkaddr);
+ if (IS_ERR(page)) {
+ erofs_err(sb, "failed to get inode (nid: %llu) page, err %ld",
+ vi->nid, PTR_ERR(page));
+ return page;
+ }
+
+ dic = page_address(page) + *ofs;
+ ifmt = le16_to_cpu(dic->i_format);
vi->datalayout = erofs_inode_datalayout(ifmt);
-
if (vi->datalayout >= EROFS_INODE_DATALAYOUT_MAX) {
erofs_err(inode->i_sb, "unsupported datalayout %u of nid %llu",
vi->datalayout, vi->nid);
- DBG_BUGON(1);
- return -EOPNOTSUPP;
+ err = -EOPNOTSUPP;
+ goto err_out;
}
switch (erofs_inode_version(ifmt)) {
case EROFS_INODE_LAYOUT_EXTENDED:
- die = data;
-
vi->inode_isize = sizeof(struct erofs_inode_extended);
+ /* check if the inode acrosses page boundary */
+ if (*ofs + vi->inode_isize <= PAGE_SIZE) {
+ *ofs += vi->inode_isize;
+ die = (struct erofs_inode_extended *)dic;
+ } else {
+ const unsigned int gotten = PAGE_SIZE - *ofs;
+
+ copied = kmalloc(vi->inode_isize, GFP_NOFS);
+ if (!copied) {
+ err = -ENOMEM;
+ goto err_out;
+ }
+ memcpy(copied, dic, gotten);
+ unlock_page(page);
+ put_page(page);
+
+ page = erofs_get_meta_page(sb, blkaddr + 1);
+ if (IS_ERR(page)) {
+ erofs_err(sb, "failed to get inode payload page (nid: %llu), err %ld",
+ vi->nid, PTR_ERR(page));
+ kfree(copied);
+ return page;
+ }
+ *ofs = vi->inode_isize - gotten;
+ memcpy((u8 *)copied + gotten, page_address(page), *ofs);
+ die = copied;
+ }
vi->xattr_isize = erofs_xattr_ibody_size(die->i_xattr_icount);
inode->i_mode = le16_to_cpu(die->i_mode);
@@ -69,9 +118,12 @@
/* total blocks for compressed files */
if (erofs_inode_is_data_compressed(vi->datalayout))
nblks = le32_to_cpu(die->i_u.compressed_blocks);
+
+ kfree(copied);
break;
case EROFS_INODE_LAYOUT_COMPACT:
vi->inode_isize = sizeof(struct erofs_inode_compact);
+ *ofs += vi->inode_isize;
vi->xattr_isize = erofs_xattr_ibody_size(dic->i_xattr_icount);
inode->i_mode = le16_to_cpu(dic->i_mode);
@@ -111,8 +163,8 @@
erofs_err(inode->i_sb,
"unsupported on-disk inode version %u of nid %llu",
erofs_inode_version(ifmt), vi->nid);
- DBG_BUGON(1);
- return -EOPNOTSUPP;
+ err = -EOPNOTSUPP;
+ goto err_out;
}
if (!nblks)
@@ -120,13 +172,18 @@
inode->i_blocks = roundup(inode->i_size, EROFS_BLKSIZ) >> 9;
else
inode->i_blocks = nblks << LOG_SECTORS_PER_BLOCK;
- return 0;
+ return page;
bogusimode:
erofs_err(inode->i_sb, "bogus i_mode (%o) @ nid %llu",
inode->i_mode, vi->nid);
+ err = -EFSCORRUPTED;
+err_out:
DBG_BUGON(1);
- return -EFSCORRUPTED;
+ kfree(copied);
+ unlock_page(page);
+ put_page(page);
+ return ERR_PTR(err);
}
static int erofs_fill_symlink(struct inode *inode, void *data,
@@ -146,7 +203,7 @@
if (!lnk)
return -ENOMEM;
- m_pofs += vi->inode_isize + vi->xattr_isize;
+ m_pofs += vi->xattr_isize;
/* inline symlink data shouldn't cross page boundary as well */
if (m_pofs + inode->i_size > PAGE_SIZE) {
kfree(lnk);
@@ -167,37 +224,17 @@
static int erofs_fill_inode(struct inode *inode, int isdir)
{
- struct super_block *sb = inode->i_sb;
struct erofs_inode *vi = EROFS_I(inode);
struct page *page;
- void *data;
- int err;
- erofs_blk_t blkaddr;
unsigned int ofs;
- erofs_off_t inode_loc;
+ int err = 0;
trace_erofs_fill_inode(inode, isdir);
- inode_loc = iloc(EROFS_SB(sb), vi->nid);
- blkaddr = erofs_blknr(inode_loc);
- ofs = erofs_blkoff(inode_loc);
- erofs_dbg("%s, reading inode nid %llu at %u of blkaddr %u",
- __func__, vi->nid, ofs, blkaddr);
-
- page = erofs_get_meta_page(sb, blkaddr);
-
- if (IS_ERR(page)) {
- erofs_err(sb, "failed to get inode (nid: %llu) page, err %ld",
- vi->nid, PTR_ERR(page));
+ /* read inode base data from disk */
+ page = erofs_read_inode(inode, &ofs);
+ if (IS_ERR(page))
return PTR_ERR(page);
- }
-
- DBG_BUGON(!PageUptodate(page));
- data = page_address(page);
-
- err = erofs_read_inode(inode, data + ofs);
- if (err)
- goto out_unlock;
/* setup the new inode */
switch (inode->i_mode & S_IFMT) {
@@ -210,7 +247,7 @@
inode->i_fop = &erofs_dir_fops;
break;
case S_IFLNK:
- err = erofs_fill_symlink(inode, data, ofs);
+ err = erofs_fill_symlink(inode, page_address(page), ofs);
if (err)
goto out_unlock;
inode_nohighmem(inode);
diff --git a/fs/erofs/internal.h b/fs/erofs/internal.h
index 1c077b7..67a7ec9 100644
--- a/fs/erofs/internal.h
+++ b/fs/erofs/internal.h
@@ -1,7 +1,7 @@
/* SPDX-License-Identifier: GPL-2.0-only */
/*
* Copyright (C) 2017-2018 HUAWEI, Inc.
- * http://www.huawei.com/
+ * https://www.huawei.com/
* Created by Gao Xiang <gaoxiang25@huawei.com>
*/
#ifndef __EROFS_INTERNAL_H
diff --git a/fs/erofs/namei.c b/fs/erofs/namei.c
index 52f201e0..5f8cc73 100644
--- a/fs/erofs/namei.c
+++ b/fs/erofs/namei.c
@@ -1,7 +1,7 @@
// SPDX-License-Identifier: GPL-2.0-only
/*
* Copyright (C) 2017-2018 HUAWEI, Inc.
- * http://www.huawei.com/
+ * https://www.huawei.com/
* Created by Gao Xiang <gaoxiang25@huawei.com>
*/
#include "xattr.h"
diff --git a/fs/erofs/super.c b/fs/erofs/super.c
index 7a13ffb..ddaa516 100644
--- a/fs/erofs/super.c
+++ b/fs/erofs/super.c
@@ -1,7 +1,7 @@
// SPDX-License-Identifier: GPL-2.0-only
/*
* Copyright (C) 2017-2018 HUAWEI, Inc.
- * http://www.huawei.com/
+ * https://www.huawei.com/
* Created by Gao Xiang <gaoxiang25@huawei.com>
*/
#include <linux/module.h>
diff --git a/fs/erofs/utils.c b/fs/erofs/utils.c
index 52d0be1..de9986d 100644
--- a/fs/erofs/utils.c
+++ b/fs/erofs/utils.c
@@ -1,7 +1,7 @@
// SPDX-License-Identifier: GPL-2.0-only
/*
* Copyright (C) 2018 HUAWEI, Inc.
- * http://www.huawei.com/
+ * https://www.huawei.com/
* Created by Gao Xiang <gaoxiang25@huawei.com>
*/
#include "internal.h"
@@ -127,12 +127,6 @@
return count;
}
-static void erofs_workgroup_unfreeze_final(struct erofs_workgroup *grp)
-{
- erofs_workgroup_unfreeze(grp, 0);
- __erofs_workgroup_free(grp);
-}
-
static bool erofs_try_to_release_workgroup(struct erofs_sb_info *sbi,
struct erofs_workgroup *grp)
{
@@ -162,11 +156,9 @@
*/
DBG_BUGON(xa_erase(&sbi->managed_pslots, grp->index) != grp);
- /*
- * If managed cache is on, last refcount should indicate
- * the related workstation.
- */
- erofs_workgroup_unfreeze_final(grp);
+ /* last refcount should be connected with its managed pslot. */
+ erofs_workgroup_unfreeze(grp, 0);
+ __erofs_workgroup_free(grp);
return true;
}
diff --git a/fs/erofs/xattr.c b/fs/erofs/xattr.c
index 87e437e..c8c381e 100644
--- a/fs/erofs/xattr.c
+++ b/fs/erofs/xattr.c
@@ -1,7 +1,7 @@
// SPDX-License-Identifier: GPL-2.0-only
/*
* Copyright (C) 2017-2018 HUAWEI, Inc.
- * http://www.huawei.com/
+ * https://www.huawei.com/
* Created by Gao Xiang <gaoxiang25@huawei.com>
*/
#include <linux/security.h>
diff --git a/fs/erofs/xattr.h b/fs/erofs/xattr.h
index e4e5093..815304b 100644
--- a/fs/erofs/xattr.h
+++ b/fs/erofs/xattr.h
@@ -1,7 +1,7 @@
/* SPDX-License-Identifier: GPL-2.0-only */
/*
* Copyright (C) 2017-2018 HUAWEI, Inc.
- * http://www.huawei.com/
+ * https://www.huawei.com/
* Created by Gao Xiang <gaoxiang25@huawei.com>
*/
#ifndef __EROFS_XATTR_H
diff --git a/fs/erofs/zdata.c b/fs/erofs/zdata.c
index 24a26aa..6c939de 100644
--- a/fs/erofs/zdata.c
+++ b/fs/erofs/zdata.c
@@ -1,7 +1,7 @@
// SPDX-License-Identifier: GPL-2.0-only
/*
* Copyright (C) 2018 HUAWEI, Inc.
- * http://www.huawei.com/
+ * https://www.huawei.com/
* Created by Gao Xiang <gaoxiang25@huawei.com>
*/
#include "zdata.h"
@@ -43,13 +43,13 @@
static inline int z_erofs_init_workqueue(void)
{
const unsigned int onlinecpus = num_possible_cpus();
- const unsigned int flags = WQ_UNBOUND | WQ_HIGHPRI | WQ_CPU_INTENSIVE;
/*
* no need to spawn too many threads, limiting threads could minimum
* scheduling overhead, perhaps per-CPU threads should be better?
*/
- z_erofs_workqueue = alloc_workqueue("erofs_unzipd", flags,
+ z_erofs_workqueue = alloc_workqueue("erofs_unzipd",
+ WQ_UNBOUND | WQ_HIGHPRI,
onlinecpus + onlinecpus / 4);
return z_erofs_workqueue ? 0 : -ENOMEM;
}
diff --git a/fs/erofs/zdata.h b/fs/erofs/zdata.h
index 9b66c28..68c9b29 100644
--- a/fs/erofs/zdata.h
+++ b/fs/erofs/zdata.h
@@ -1,7 +1,7 @@
/* SPDX-License-Identifier: GPL-2.0-only */
/*
* Copyright (C) 2018 HUAWEI, Inc.
- * http://www.huawei.com/
+ * https://www.huawei.com/
* Created by Gao Xiang <gaoxiang25@huawei.com>
*/
#ifndef __EROFS_FS_ZDATA_H
diff --git a/fs/erofs/zmap.c b/fs/erofs/zmap.c
index 736db3a..7d40d78 100644
--- a/fs/erofs/zmap.c
+++ b/fs/erofs/zmap.c
@@ -1,7 +1,7 @@
// SPDX-License-Identifier: GPL-2.0-only
/*
* Copyright (C) 2018-2019 HUAWEI, Inc.
- * http://www.huawei.com/
+ * https://www.huawei.com/
* Created by Gao Xiang <gaoxiang25@huawei.com>
*/
#include "internal.h"
diff --git a/fs/erofs/zpvec.h b/fs/erofs/zpvec.h
index 5855690..1d67cbd 100644
--- a/fs/erofs/zpvec.h
+++ b/fs/erofs/zpvec.h
@@ -1,7 +1,7 @@
/* SPDX-License-Identifier: GPL-2.0-only */
/*
* Copyright (C) 2018 HUAWEI, Inc.
- * http://www.huawei.com/
+ * https://www.huawei.com/
* Created by Gao Xiang <gaoxiang25@huawei.com>
*/
#ifndef __EROFS_FS_ZPVEC_H
diff --git a/fs/ext2/dir.c b/fs/ext2/dir.c
index 13318e2..70355ab 100644
--- a/fs/ext2/dir.c
+++ b/fs/ext2/dir.c
@@ -348,7 +348,6 @@
struct page *page = NULL;
struct ext2_inode_info *ei = EXT2_I(dir);
ext2_dirent * de;
- int dir_has_error = 0;
if (npages == 0)
goto out;
@@ -362,25 +361,25 @@
n = start;
do {
char *kaddr;
- page = ext2_get_page(dir, n, dir_has_error);
- if (!IS_ERR(page)) {
- kaddr = page_address(page);
- de = (ext2_dirent *) kaddr;
- kaddr += ext2_last_byte(dir, n) - reclen;
- while ((char *) de <= kaddr) {
- if (de->rec_len == 0) {
- ext2_error(dir->i_sb, __func__,
- "zero-length directory entry");
- ext2_put_page(page);
- goto out;
- }
- if (ext2_match (namelen, name, de))
- goto found;
- de = ext2_next_entry(de);
+ page = ext2_get_page(dir, n, 0);
+ if (IS_ERR(page))
+ return ERR_CAST(page);
+
+ kaddr = page_address(page);
+ de = (ext2_dirent *) kaddr;
+ kaddr += ext2_last_byte(dir, n) - reclen;
+ while ((char *) de <= kaddr) {
+ if (de->rec_len == 0) {
+ ext2_error(dir->i_sb, __func__,
+ "zero-length directory entry");
+ ext2_put_page(page);
+ goto out;
}
- ext2_put_page(page);
- } else
- dir_has_error = 1;
+ if (ext2_match(namelen, name, de))
+ goto found;
+ de = ext2_next_entry(de);
+ }
+ ext2_put_page(page);
if (++n >= npages)
n = 0;
@@ -394,7 +393,7 @@
}
} while (n != start);
out:
- return NULL;
+ return ERR_PTR(-ENOENT);
found:
*res_page = page;
@@ -414,18 +413,18 @@
return de;
}
-ino_t ext2_inode_by_name(struct inode *dir, const struct qstr *child)
+int ext2_inode_by_name(struct inode *dir, const struct qstr *child, ino_t *ino)
{
- ino_t res = 0;
struct ext2_dir_entry_2 *de;
struct page *page;
- de = ext2_find_entry (dir, child, &page);
- if (de) {
- res = le32_to_cpu(de->inode);
- ext2_put_page(page);
- }
- return res;
+ de = ext2_find_entry(dir, child, &page);
+ if (IS_ERR(de))
+ return PTR_ERR(de);
+
+ *ino = le32_to_cpu(de->inode);
+ ext2_put_page(page);
+ return 0;
}
static int ext2_prepare_chunk(struct page *page, loff_t pos, unsigned len)
diff --git a/fs/ext2/ext2.h b/fs/ext2/ext2.h
index 8178bd3..5136b72 100644
--- a/fs/ext2/ext2.h
+++ b/fs/ext2/ext2.h
@@ -52,8 +52,8 @@
/*
* Was i_next_alloc_goal in ext2_inode_info
* is the *physical* companion to i_next_alloc_block.
- * it the the physical block number of the block which was most-recentl
- * allocated to this file. This give us the goal (target) for the next
+ * it is the physical block number of the block which was most-recently
+ * allocated to this file. This gives us the goal (target) for the next
* allocation when we detect linearly ascending requests.
*/
ext2_fsblk_t last_alloc_physical_block;
@@ -374,7 +374,6 @@
/*
* Mount flags
*/
-#define EXT2_MOUNT_CHECK 0x000001 /* Do mount-time checks */
#define EXT2_MOUNT_OLDALLOC 0x000002 /* Don't use the new Orlov allocator */
#define EXT2_MOUNT_GRPID 0x000004 /* Create files with directory's group */
#define EXT2_MOUNT_DEBUG 0x000008 /* Some debugging messages */
@@ -738,7 +737,8 @@
/* dir.c */
extern int ext2_add_link (struct dentry *, struct inode *);
-extern ino_t ext2_inode_by_name(struct inode *, const struct qstr *);
+extern int ext2_inode_by_name(struct inode *dir,
+ const struct qstr *child, ino_t *ino);
extern int ext2_make_empty(struct inode *, struct inode *);
extern struct ext2_dir_entry_2 * ext2_find_entry (struct inode *,const struct qstr *, struct page **);
extern int ext2_delete_entry (struct ext2_dir_entry_2 *, struct page *);
diff --git a/fs/ext2/ialloc.c b/fs/ext2/ialloc.c
index fda7d3f..432c3fe 100644
--- a/fs/ext2/ialloc.c
+++ b/fs/ext2/ialloc.c
@@ -80,6 +80,7 @@
if (dir)
le16_add_cpu(&desc->bg_used_dirs_count, -1);
spin_unlock(sb_bgl_lock(EXT2_SB(sb), group));
+ percpu_counter_inc(&EXT2_SB(sb)->s_freeinodes_counter);
if (dir)
percpu_counter_dec(&EXT2_SB(sb)->s_dirs_counter);
mark_buffer_dirty(bh);
@@ -528,7 +529,7 @@
goto fail;
}
- percpu_counter_add(&sbi->s_freeinodes_counter, -1);
+ percpu_counter_dec(&sbi->s_freeinodes_counter);
if (S_ISDIR(mode))
percpu_counter_inc(&sbi->s_dirs_counter);
diff --git a/fs/ext2/inode.c b/fs/ext2/inode.c
index c8b371c..80662e1 100644
--- a/fs/ext2/inode.c
+++ b/fs/ext2/inode.c
@@ -356,8 +356,7 @@
* @blks: number of data blocks to be mapped.
* @blocks_to_boundary: the offset in the indirect block
*
- * return the total number of blocks to be allocate, including the
- * direct and indirect blocks.
+ * return the number of direct blocks to allocate.
*/
static int
ext2_blks_to_allocate(Indirect * branch, int k, unsigned long blks,
@@ -390,11 +389,9 @@
* ext2_alloc_blocks: multiple allocate blocks needed for a branch
* @indirect_blks: the number of blocks need to allocate for indirect
* blocks
- *
+ * @blks: the number of blocks need to allocate for direct blocks
* @new_blocks: on return it will store the new block numbers for
* the indirect blocks(if needed) and the first direct block,
- * @blks: on return it will store the total number of allocated
- * direct blocks
*/
static int ext2_alloc_blocks(struct inode *inode,
ext2_fsblk_t goal, int indirect_blks, int blks,
diff --git a/fs/ext2/namei.c b/fs/ext2/namei.c
index ba3e3e0..5bf2c14 100644
--- a/fs/ext2/namei.c
+++ b/fs/ext2/namei.c
@@ -57,13 +57,17 @@
{
struct inode * inode;
ino_t ino;
+ int res;
if (dentry->d_name.len > EXT2_NAME_LEN)
return ERR_PTR(-ENAMETOOLONG);
- ino = ext2_inode_by_name(dir, &dentry->d_name);
- inode = NULL;
- if (ino) {
+ res = ext2_inode_by_name(dir, &dentry->d_name, &ino);
+ if (res) {
+ if (res != -ENOENT)
+ return ERR_PTR(res);
+ inode = NULL;
+ } else {
inode = ext2_iget(dir->i_sb, ino);
if (inode == ERR_PTR(-ESTALE)) {
ext2_error(dir->i_sb, __func__,
@@ -78,9 +82,13 @@
struct dentry *ext2_get_parent(struct dentry *child)
{
struct qstr dotdot = QSTR_INIT("..", 2);
- unsigned long ino = ext2_inode_by_name(d_inode(child), &dotdot);
- if (!ino)
- return ERR_PTR(-ENOENT);
+ ino_t ino;
+ int res;
+
+ res = ext2_inode_by_name(d_inode(child), &dotdot, &ino);
+ if (res)
+ return ERR_PTR(res);
+
return d_obtain_alias(ext2_iget(child->d_sb, ino));
}
@@ -274,9 +282,9 @@
if (err)
goto out;
- de = ext2_find_entry (dir, &dentry->d_name, &page);
- if (!de) {
- err = -ENOENT;
+ de = ext2_find_entry(dir, &dentry->d_name, &page);
+ if (IS_ERR(de)) {
+ err = PTR_ERR(de);
goto out;
}
@@ -330,9 +338,9 @@
if (err)
goto out;
- old_de = ext2_find_entry (old_dir, &old_dentry->d_name, &old_page);
- if (!old_de) {
- err = -ENOENT;
+ old_de = ext2_find_entry(old_dir, &old_dentry->d_name, &old_page);
+ if (IS_ERR(old_de)) {
+ err = PTR_ERR(old_de);
goto out;
}
@@ -351,10 +359,11 @@
if (dir_de && !ext2_empty_dir (new_inode))
goto out_dir;
- err = -ENOENT;
- new_de = ext2_find_entry (new_dir, &new_dentry->d_name, &new_page);
- if (!new_de)
+ new_de = ext2_find_entry(new_dir, &new_dentry->d_name, &new_page);
+ if (IS_ERR(new_de)) {
+ err = PTR_ERR(new_de);
goto out_dir;
+ }
ext2_set_link(new_dir, new_de, new_page, old_inode, 1);
new_inode->i_ctime = current_time(new_inode);
if (dir_de)
diff --git a/fs/ext2/super.c b/fs/ext2/super.c
index 4a4ab68..dda8605 100644
--- a/fs/ext2/super.c
+++ b/fs/ext2/super.c
@@ -431,7 +431,7 @@
enum {
Opt_bsd_df, Opt_minix_df, Opt_grpid, Opt_nogrpid,
Opt_resgid, Opt_resuid, Opt_sb, Opt_err_cont, Opt_err_panic,
- Opt_err_ro, Opt_nouid32, Opt_nocheck, Opt_debug,
+ Opt_err_ro, Opt_nouid32, Opt_debug,
Opt_oldalloc, Opt_orlov, Opt_nobh, Opt_user_xattr, Opt_nouser_xattr,
Opt_acl, Opt_noacl, Opt_xip, Opt_dax, Opt_ignore, Opt_err, Opt_quota,
Opt_usrquota, Opt_grpquota, Opt_reservation, Opt_noreservation
@@ -451,8 +451,6 @@
{Opt_err_panic, "errors=panic"},
{Opt_err_ro, "errors=remount-ro"},
{Opt_nouid32, "nouid32"},
- {Opt_nocheck, "check=none"},
- {Opt_nocheck, "nocheck"},
{Opt_debug, "debug"},
{Opt_oldalloc, "oldalloc"},
{Opt_orlov, "orlov"},
@@ -546,12 +544,6 @@
case Opt_nouid32:
set_opt (opts->s_mount_opt, NO_UID32);
break;
- case Opt_nocheck:
- ext2_msg(sb, KERN_WARNING,
- "Option nocheck/check=none is deprecated and"
- " will be removed in June 2020.");
- clear_opt (opts->s_mount_opt, CHECK);
- break;
case Opt_debug:
set_opt (opts->s_mount_opt, DEBUG);
break;
diff --git a/fs/ext2/xattr.c b/fs/ext2/xattr.c
index 943cc46..841fa6d 100644
--- a/fs/ext2/xattr.c
+++ b/fs/ext2/xattr.c
@@ -437,6 +437,9 @@
name_len = strlen(name);
if (name_len > 255 || value_len > sb->s_blocksize)
return -ERANGE;
+ error = dquot_initialize(inode);
+ if (error)
+ return error;
down_write(&EXT2_I(inode)->xattr_sem);
if (EXT2_I(inode)->i_file_acl) {
/* The inode already has an extended attribute block. */
@@ -588,7 +591,6 @@
/* Remove the old value. */
memmove(first_val + size, first_val, val - first_val);
memset(first_val, 0, size);
- here->e_value_offs = 0;
min_offs += size;
/* Adjust all value offsets. */
@@ -600,6 +602,8 @@
cpu_to_le16(o + size);
last = EXT2_XATTR_NEXT(last);
}
+
+ here->e_value_offs = 0;
}
if (value == NULL) {
/* Remove the old name. */
diff --git a/fs/ext4/file.c b/fs/ext4/file.c
index 2a01e31..129cc1d 100644
--- a/fs/ext4/file.c
+++ b/fs/ext4/file.c
@@ -544,6 +544,8 @@
iomap_ops = &ext4_iomap_overwrite_ops;
ret = iomap_dio_rw(iocb, from, iomap_ops, &ext4_dio_write_ops,
is_sync_kiocb(iocb) || unaligned_io || extend);
+ if (ret == -ENOTBLK)
+ ret = 0;
if (extend)
ret = ext4_handle_inode_extension(inode, offset, ret, count);
diff --git a/fs/file.c b/fs/file.c
index 4cb9ef4..21c0893 100644
--- a/fs/file.c
+++ b/fs/file.c
@@ -1145,7 +1145,7 @@
return ksys_dup3(oldfd, newfd, 0);
}
-int ksys_dup(unsigned int fildes)
+SYSCALL_DEFINE1(dup, unsigned int, fildes)
{
int ret = -EBADF;
struct file *file = fget_raw(fildes);
@@ -1160,11 +1160,6 @@
return ret;
}
-SYSCALL_DEFINE1(dup, unsigned int, fildes)
-{
- return ksys_dup(fildes);
-}
-
int f_dupfd(unsigned int from, struct file *file, unsigned flags)
{
int err;
diff --git a/fs/gfs2/file.c b/fs/gfs2/file.c
index bebde537..b085a3b 100644
--- a/fs/gfs2/file.c
+++ b/fs/gfs2/file.c
@@ -835,7 +835,8 @@
ret = iomap_dio_rw(iocb, from, &gfs2_iomap_ops, NULL,
is_sync_kiocb(iocb));
-
+ if (ret == -ENOTBLK)
+ ret = 0;
out:
gfs2_glock_dq(&gh);
out_uninit:
diff --git a/fs/init.c b/fs/init.c
new file mode 100644
index 0000000..e9c320a
--- /dev/null
+++ b/fs/init.c
@@ -0,0 +1,265 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Routines that mimic syscalls, but don't use the user address space or file
+ * descriptors. Only for init/ and related early init code.
+ */
+#include <linux/init.h>
+#include <linux/mount.h>
+#include <linux/namei.h>
+#include <linux/fs.h>
+#include <linux/fs_struct.h>
+#include <linux/file.h>
+#include <linux/init_syscalls.h>
+#include <linux/security.h>
+#include "internal.h"
+
+int __init init_mount(const char *dev_name, const char *dir_name,
+ const char *type_page, unsigned long flags, void *data_page)
+{
+ struct path path;
+ int ret;
+
+ ret = kern_path(dir_name, LOOKUP_FOLLOW, &path);
+ if (ret)
+ return ret;
+ ret = path_mount(dev_name, &path, type_page, flags, data_page);
+ path_put(&path);
+ return ret;
+}
+
+int __init init_umount(const char *name, int flags)
+{
+ int lookup_flags = LOOKUP_MOUNTPOINT;
+ struct path path;
+ int ret;
+
+ if (!(flags & UMOUNT_NOFOLLOW))
+ lookup_flags |= LOOKUP_FOLLOW;
+ ret = kern_path(name, lookup_flags, &path);
+ if (ret)
+ return ret;
+ return path_umount(&path, flags);
+}
+
+int __init init_chdir(const char *filename)
+{
+ struct path path;
+ int error;
+
+ error = kern_path(filename, LOOKUP_FOLLOW | LOOKUP_DIRECTORY, &path);
+ if (error)
+ return error;
+ error = inode_permission(path.dentry->d_inode, MAY_EXEC | MAY_CHDIR);
+ if (!error)
+ set_fs_pwd(current->fs, &path);
+ path_put(&path);
+ return error;
+}
+
+int __init init_chroot(const char *filename)
+{
+ struct path path;
+ int error;
+
+ error = kern_path(filename, LOOKUP_FOLLOW | LOOKUP_DIRECTORY, &path);
+ if (error)
+ return error;
+ error = inode_permission(path.dentry->d_inode, MAY_EXEC | MAY_CHDIR);
+ if (error)
+ goto dput_and_out;
+ error = -EPERM;
+ if (!ns_capable(current_user_ns(), CAP_SYS_CHROOT))
+ goto dput_and_out;
+ error = security_path_chroot(&path);
+ if (error)
+ goto dput_and_out;
+ set_fs_root(current->fs, &path);
+dput_and_out:
+ path_put(&path);
+ return error;
+}
+
+int __init init_chown(const char *filename, uid_t user, gid_t group, int flags)
+{
+ int lookup_flags = (flags & AT_SYMLINK_NOFOLLOW) ? 0 : LOOKUP_FOLLOW;
+ struct path path;
+ int error;
+
+ error = kern_path(filename, lookup_flags, &path);
+ if (error)
+ return error;
+ error = mnt_want_write(path.mnt);
+ if (!error) {
+ error = chown_common(&path, user, group);
+ mnt_drop_write(path.mnt);
+ }
+ path_put(&path);
+ return error;
+}
+
+int __init init_chmod(const char *filename, umode_t mode)
+{
+ struct path path;
+ int error;
+
+ error = kern_path(filename, LOOKUP_FOLLOW, &path);
+ if (error)
+ return error;
+ error = chmod_common(&path, mode);
+ path_put(&path);
+ return error;
+}
+
+int __init init_eaccess(const char *filename)
+{
+ struct path path;
+ int error;
+
+ error = kern_path(filename, LOOKUP_FOLLOW, &path);
+ if (error)
+ return error;
+ error = inode_permission(d_inode(path.dentry), MAY_ACCESS);
+ path_put(&path);
+ return error;
+}
+
+int __init init_stat(const char *filename, struct kstat *stat, int flags)
+{
+ int lookup_flags = (flags & AT_SYMLINK_NOFOLLOW) ? 0 : LOOKUP_FOLLOW;
+ struct path path;
+ int error;
+
+ error = kern_path(filename, lookup_flags, &path);
+ if (error)
+ return error;
+ error = vfs_getattr(&path, stat, STATX_BASIC_STATS,
+ flags | AT_NO_AUTOMOUNT);
+ path_put(&path);
+ return error;
+}
+
+int __init init_mknod(const char *filename, umode_t mode, unsigned int dev)
+{
+ struct dentry *dentry;
+ struct path path;
+ int error;
+
+ if (S_ISFIFO(mode) || S_ISSOCK(mode))
+ dev = 0;
+ else if (!(S_ISBLK(mode) || S_ISCHR(mode)))
+ return -EINVAL;
+
+ dentry = kern_path_create(AT_FDCWD, filename, &path, 0);
+ if (IS_ERR(dentry))
+ return PTR_ERR(dentry);
+
+ if (!IS_POSIXACL(path.dentry->d_inode))
+ mode &= ~current_umask();
+ error = security_path_mknod(&path, dentry, mode, dev);
+ if (!error)
+ error = vfs_mknod(path.dentry->d_inode, dentry, mode,
+ new_decode_dev(dev));
+ done_path_create(&path, dentry);
+ return error;
+}
+
+int __init init_link(const char *oldname, const char *newname)
+{
+ struct dentry *new_dentry;
+ struct path old_path, new_path;
+ int error;
+
+ error = kern_path(oldname, 0, &old_path);
+ if (error)
+ return error;
+
+ new_dentry = kern_path_create(AT_FDCWD, newname, &new_path, 0);
+ error = PTR_ERR(new_dentry);
+ if (IS_ERR(new_dentry))
+ goto out;
+
+ error = -EXDEV;
+ if (old_path.mnt != new_path.mnt)
+ goto out_dput;
+ error = may_linkat(&old_path);
+ if (unlikely(error))
+ goto out_dput;
+ error = security_path_link(old_path.dentry, &new_path, new_dentry);
+ if (error)
+ goto out_dput;
+ error = vfs_link(old_path.dentry, new_path.dentry->d_inode, new_dentry,
+ NULL);
+out_dput:
+ done_path_create(&new_path, new_dentry);
+out:
+ path_put(&old_path);
+ return error;
+}
+
+int __init init_symlink(const char *oldname, const char *newname)
+{
+ struct dentry *dentry;
+ struct path path;
+ int error;
+
+ dentry = kern_path_create(AT_FDCWD, newname, &path, 0);
+ if (IS_ERR(dentry))
+ return PTR_ERR(dentry);
+ error = security_path_symlink(&path, dentry, oldname);
+ if (!error)
+ error = vfs_symlink(path.dentry->d_inode, dentry, oldname);
+ done_path_create(&path, dentry);
+ return error;
+}
+
+int __init init_unlink(const char *pathname)
+{
+ return do_unlinkat(AT_FDCWD, getname_kernel(pathname));
+}
+
+int __init init_mkdir(const char *pathname, umode_t mode)
+{
+ struct dentry *dentry;
+ struct path path;
+ int error;
+
+ dentry = kern_path_create(AT_FDCWD, pathname, &path, LOOKUP_DIRECTORY);
+ if (IS_ERR(dentry))
+ return PTR_ERR(dentry);
+ if (!IS_POSIXACL(path.dentry->d_inode))
+ mode &= ~current_umask();
+ error = security_path_mkdir(&path, dentry, mode);
+ if (!error)
+ error = vfs_mkdir(path.dentry->d_inode, dentry, mode);
+ done_path_create(&path, dentry);
+ return error;
+}
+
+int __init init_rmdir(const char *pathname)
+{
+ return do_rmdir(AT_FDCWD, getname_kernel(pathname));
+}
+
+int __init init_utimes(char *filename, struct timespec64 *ts)
+{
+ struct path path;
+ int error;
+
+ error = kern_path(filename, 0, &path);
+ if (error)
+ return error;
+ error = vfs_utimes(&path, ts);
+ path_put(&path);
+ return error;
+}
+
+int __init init_dup(struct file *file)
+{
+ int fd;
+
+ fd = get_unused_fd_flags(0);
+ if (fd < 0)
+ return fd;
+ fd_install(fd, get_file(file));
+ return 0;
+}
diff --git a/fs/internal.h b/fs/internal.h
index 969988d..10517ec 100644
--- a/fs/internal.h
+++ b/fs/internal.h
@@ -75,15 +75,9 @@
struct path *path, struct path *root);
extern int vfs_path_lookup(struct dentry *, struct vfsmount *,
const char *, unsigned int, struct path *);
-long do_mknodat(int dfd, const char __user *filename, umode_t mode,
- unsigned int dev);
-long do_mkdirat(int dfd, const char __user *pathname, umode_t mode);
-long do_rmdir(int dfd, const char __user *pathname);
+long do_rmdir(int dfd, struct filename *name);
long do_unlinkat(int dfd, struct filename *name);
-long do_symlinkat(const char __user *oldname, int newdfd,
- const char __user *newname);
-int do_linkat(int olddfd, const char __user *oldname, int newdfd,
- const char __user *newname, int flags);
+int may_linkat(struct path *link);
/*
* namespace.c
@@ -102,6 +96,11 @@
extern void __mnt_drop_write_file(struct file *);
extern void dissolve_on_fput(struct vfsmount *);
+
+int path_mount(const char *dev_name, struct path *path,
+ const char *type_page, unsigned long flags, void *data_page);
+int path_umount(struct path *path, int flags);
+
/*
* fs_struct.c
*/
@@ -139,10 +138,10 @@
extern int build_open_flags(const struct open_how *how, struct open_flags *op);
long do_sys_ftruncate(unsigned int fd, loff_t length, int small);
-int do_fchmodat(int dfd, const char __user *filename, umode_t mode);
+int chmod_common(const struct path *path, umode_t mode);
int do_fchownat(int dfd, const char __user *filename, uid_t user, gid_t group,
int flag);
-
+int chown_common(const struct path *path, uid_t user, gid_t group);
extern int vfs_open(const struct path *, struct file *);
/*
diff --git a/fs/ioctl.c b/fs/ioctl.c
index d69786d..4e6cc0a 100644
--- a/fs/ioctl.c
+++ b/fs/ioctl.c
@@ -736,7 +736,7 @@
return -ENOIOCTLCMD;
}
-int ksys_ioctl(unsigned int fd, unsigned int cmd, unsigned long arg)
+SYSCALL_DEFINE3(ioctl, unsigned int, fd, unsigned int, cmd, unsigned long, arg)
{
struct fd f = fdget(fd);
int error;
@@ -757,11 +757,6 @@
return error;
}
-SYSCALL_DEFINE3(ioctl, unsigned int, fd, unsigned int, cmd, unsigned long, arg)
-{
- return ksys_ioctl(fd, cmd, arg);
-}
-
#ifdef CONFIG_COMPAT
/**
* compat_ptr_ioctl - generic implementation of .compat_ioctl file operation
diff --git a/fs/iomap/apply.c b/fs/iomap/apply.c
index 76925b4..26ab656 100644
--- a/fs/iomap/apply.c
+++ b/fs/iomap/apply.c
@@ -46,10 +46,14 @@
ret = ops->iomap_begin(inode, pos, length, flags, &iomap, &srcmap);
if (ret)
return ret;
- if (WARN_ON(iomap.offset > pos))
- return -EIO;
- if (WARN_ON(iomap.length == 0))
- return -EIO;
+ if (WARN_ON(iomap.offset > pos)) {
+ written = -EIO;
+ goto out;
+ }
+ if (WARN_ON(iomap.length == 0)) {
+ written = -EIO;
+ goto out;
+ }
trace_iomap_apply_dstmap(inode, &iomap);
if (srcmap.type != IOMAP_HOLE)
@@ -80,6 +84,7 @@
written = actor(inode, pos, length, data, &iomap,
srcmap.type != IOMAP_HOLE ? &srcmap : &iomap);
+out:
/*
* Now the data has been copied, commit the range we've copied. This
* should not fail unless the filesystem has had a fatal error.
diff --git a/fs/iomap/direct-io.c b/fs/iomap/direct-io.c
index ec7b78e..c1aafb2 100644
--- a/fs/iomap/direct-io.c
+++ b/fs/iomap/direct-io.c
@@ -10,6 +10,7 @@
#include <linux/backing-dev.h>
#include <linux/uio.h>
#include <linux/task_io_accounting_ops.h>
+#include "trace.h"
#include "../internal.h"
@@ -401,6 +402,9 @@
* can be mapped into multiple disjoint IOs and only a subset of the IOs issued
* may be pure data writes. In that case, we still need to do a full data sync
* completion.
+ *
+ * Returns -ENOTBLK In case of a page invalidation invalidation failure for
+ * writes. The callers needs to fall back to buffered I/O in this case.
*/
ssize_t
iomap_dio_rw(struct kiocb *iocb, struct iov_iter *iter,
@@ -475,23 +479,24 @@
if (ret)
goto out_free_dio;
- /*
- * Try to invalidate cache pages for the range we're direct
- * writing. If this invalidation fails, tough, the write will
- * still work, but racing two incompatible write paths is a
- * pretty crazy thing to do, so we don't support it 100%.
- */
- ret = invalidate_inode_pages2_range(mapping,
- pos >> PAGE_SHIFT, end >> PAGE_SHIFT);
- if (ret)
- dio_warn_stale_pagecache(iocb->ki_filp);
- ret = 0;
-
- if (iov_iter_rw(iter) == WRITE && !wait_for_completion &&
- !inode->i_sb->s_dio_done_wq) {
- ret = sb_init_dio_done_wq(inode->i_sb);
- if (ret < 0)
+ if (iov_iter_rw(iter) == WRITE) {
+ /*
+ * Try to invalidate cache pages for the range we are writing.
+ * If this invalidation fails, let the caller fall back to
+ * buffered I/O.
+ */
+ if (invalidate_inode_pages2_range(mapping, pos >> PAGE_SHIFT,
+ end >> PAGE_SHIFT)) {
+ trace_iomap_dio_invalidate_fail(inode, pos, count);
+ ret = -ENOTBLK;
goto out_free_dio;
+ }
+
+ if (!wait_for_completion && !inode->i_sb->s_dio_done_wq) {
+ ret = sb_init_dio_done_wq(inode->i_sb);
+ if (ret < 0)
+ goto out_free_dio;
+ }
}
inode_dio_begin(inode);
diff --git a/fs/iomap/trace.h b/fs/iomap/trace.h
index 5693a39..fdc7ae3 100644
--- a/fs/iomap/trace.h
+++ b/fs/iomap/trace.h
@@ -74,6 +74,7 @@
DEFINE_RANGE_EVENT(iomap_writepage);
DEFINE_RANGE_EVENT(iomap_releasepage);
DEFINE_RANGE_EVENT(iomap_invalidatepage);
+DEFINE_RANGE_EVENT(iomap_dio_invalidate_fail);
#define IOMAP_TYPE_STRINGS \
{ IOMAP_HOLE, "HOLE" }, \
diff --git a/fs/kernfs/file.c b/fs/kernfs/file.c
index 06b342d..f277d02 100644
--- a/fs/kernfs/file.c
+++ b/fs/kernfs/file.c
@@ -883,6 +883,7 @@
list_for_each_entry(info, &kernfs_root(kn)->supers, node) {
struct kernfs_node *parent;
+ struct inode *p_inode = NULL;
struct inode *inode;
struct qstr name;
@@ -899,20 +900,20 @@
name = (struct qstr)QSTR_INIT(kn->name, strlen(kn->name));
parent = kernfs_get_parent(kn);
if (parent) {
- struct inode *p_inode;
-
p_inode = ilookup(info->sb, kernfs_ino(parent));
if (p_inode) {
- fsnotify(p_inode, FS_MODIFY | FS_EVENT_ON_CHILD,
- inode, FSNOTIFY_EVENT_INODE, &name, 0);
+ fsnotify(FS_MODIFY | FS_EVENT_ON_CHILD,
+ inode, FSNOTIFY_EVENT_INODE,
+ p_inode, &name, inode, 0);
iput(p_inode);
}
kernfs_put(parent);
}
- fsnotify(inode, FS_MODIFY, inode, FSNOTIFY_EVENT_INODE,
- &name, 0);
+ if (!p_inode)
+ fsnotify_inode(inode, FS_MODIFY);
+
iput(inode);
}
diff --git a/fs/namei.c b/fs/namei.c
index 72d4219..fde8fe0 100644
--- a/fs/namei.c
+++ b/fs/namei.c
@@ -1024,7 +1024,7 @@
*
* Returns 0 if successful, -ve on error.
*/
-static int may_linkat(struct path *link)
+int may_linkat(struct path *link)
{
struct inode *inode = link->dentry->d_inode;
@@ -3564,7 +3564,7 @@
}
}
-long do_mknodat(int dfd, const char __user *filename, umode_t mode,
+static long do_mknodat(int dfd, const char __user *filename, umode_t mode,
unsigned int dev)
{
struct dentry *dentry;
@@ -3645,7 +3645,7 @@
}
EXPORT_SYMBOL(vfs_mkdir);
-long do_mkdirat(int dfd, const char __user *pathname, umode_t mode)
+static long do_mkdirat(int dfd, const char __user *pathname, umode_t mode)
{
struct dentry *dentry;
struct path path;
@@ -3720,17 +3720,16 @@
}
EXPORT_SYMBOL(vfs_rmdir);
-long do_rmdir(int dfd, const char __user *pathname)
+long do_rmdir(int dfd, struct filename *name)
{
int error = 0;
- struct filename *name;
struct dentry *dentry;
struct path path;
struct qstr last;
int type;
unsigned int lookup_flags = 0;
retry:
- name = filename_parentat(dfd, getname(pathname), lookup_flags,
+ name = filename_parentat(dfd, name, lookup_flags,
&path, &last, &type);
if (IS_ERR(name))
return PTR_ERR(name);
@@ -3781,7 +3780,7 @@
SYSCALL_DEFINE1(rmdir, const char __user *, pathname)
{
- return do_rmdir(AT_FDCWD, pathname);
+ return do_rmdir(AT_FDCWD, getname(pathname));
}
/**
@@ -3926,8 +3925,7 @@
return -EINVAL;
if (flag & AT_REMOVEDIR)
- return do_rmdir(dfd, pathname);
-
+ return do_rmdir(dfd, getname(pathname));
return do_unlinkat(dfd, getname(pathname));
}
@@ -3957,7 +3955,7 @@
}
EXPORT_SYMBOL(vfs_symlink);
-long do_symlinkat(const char __user *oldname, int newdfd,
+static long do_symlinkat(const char __user *oldname, int newdfd,
const char __user *newname)
{
int error;
@@ -4088,7 +4086,7 @@
* with linux 2.0, and to avoid hard-linking to directories
* and other special files. --ADM
*/
-int do_linkat(int olddfd, const char __user *oldname, int newdfd,
+static int do_linkat(int olddfd, const char __user *oldname, int newdfd,
const char __user *newname, int flags)
{
struct dentry *new_dentry;
diff --git a/fs/namespace.c b/fs/namespace.c
index 4a0f600..016553d 100644
--- a/fs/namespace.c
+++ b/fs/namespace.c
@@ -1706,36 +1706,19 @@
}
#endif
-/*
- * Now umount can handle mount points as well as block devices.
- * This is important for filesystems which use unnamed block devices.
- *
- * We now support a flag for forced unmount like the other 'big iron'
- * unixes. Our API is identical to OSF/1 to avoid making a mess of AMD
- */
-
-int ksys_umount(char __user *name, int flags)
+int path_umount(struct path *path, int flags)
{
- struct path path;
struct mount *mnt;
int retval;
- int lookup_flags = LOOKUP_MOUNTPOINT;
if (flags & ~(MNT_FORCE | MNT_DETACH | MNT_EXPIRE | UMOUNT_NOFOLLOW))
return -EINVAL;
-
if (!may_mount())
return -EPERM;
- if (!(flags & UMOUNT_NOFOLLOW))
- lookup_flags |= LOOKUP_FOLLOW;
-
- retval = user_path_at(AT_FDCWD, name, lookup_flags, &path);
- if (retval)
- goto out;
- mnt = real_mount(path.mnt);
+ mnt = real_mount(path->mnt);
retval = -EINVAL;
- if (path.dentry != path.mnt->mnt_root)
+ if (path->dentry != path->mnt->mnt_root)
goto dput_and_out;
if (!check_mnt(mnt))
goto dput_and_out;
@@ -1748,12 +1731,25 @@
retval = do_umount(mnt, flags);
dput_and_out:
/* we mustn't call path_put() as that would clear mnt_expiry_mark */
- dput(path.dentry);
+ dput(path->dentry);
mntput_no_expire(mnt);
-out:
return retval;
}
+static int ksys_umount(char __user *name, int flags)
+{
+ int lookup_flags = LOOKUP_MOUNTPOINT;
+ struct path path;
+ int ret;
+
+ if (!(flags & UMOUNT_NOFOLLOW))
+ lookup_flags |= LOOKUP_FOLLOW;
+ ret = user_path_at(AT_FDCWD, name, lookup_flags, &path);
+ if (ret)
+ return ret;
+ return path_umount(&path, flags);
+}
+
SYSCALL_DEFINE2(umount, char __user *, name, int, flags)
{
return ksys_umount(name, flags);
@@ -3116,12 +3112,11 @@
* Therefore, if this magic number is present, it carries no information
* and must be discarded.
*/
-long do_mount(const char *dev_name, const char __user *dir_name,
+int path_mount(const char *dev_name, struct path *path,
const char *type_page, unsigned long flags, void *data_page)
{
- struct path path;
unsigned int mnt_flags = 0, sb_flags;
- int retval = 0;
+ int ret;
/* Discard magic */
if ((flags & MS_MGC_MSK) == MS_MGC_VAL)
@@ -3134,19 +3129,13 @@
if (flags & MS_NOUSER)
return -EINVAL;
- /* ... and get the mountpoint */
- retval = user_path_at(AT_FDCWD, dir_name, LOOKUP_FOLLOW, &path);
- if (retval)
- return retval;
-
- retval = security_sb_mount(dev_name, &path,
- type_page, flags, data_page);
- if (!retval && !may_mount())
- retval = -EPERM;
- if (!retval && (flags & SB_MANDLOCK) && !may_mandlock())
- retval = -EPERM;
- if (retval)
- goto dput_out;
+ ret = security_sb_mount(dev_name, path, type_page, flags, data_page);
+ if (ret)
+ return ret;
+ if (!may_mount())
+ return -EPERM;
+ if ((flags & SB_MANDLOCK) && !may_mandlock())
+ return -EPERM;
/* Default to relatime unless overriden */
if (!(flags & MS_NOATIME))
@@ -3173,7 +3162,7 @@
((flags & (MS_NOATIME | MS_NODIRATIME | MS_RELATIME |
MS_STRICTATIME)) == 0)) {
mnt_flags &= ~MNT_ATIME_MASK;
- mnt_flags |= path.mnt->mnt_flags & MNT_ATIME_MASK;
+ mnt_flags |= path->mnt->mnt_flags & MNT_ATIME_MASK;
}
sb_flags = flags & (SB_RDONLY |
@@ -3186,22 +3175,32 @@
SB_I_VERSION);
if ((flags & (MS_REMOUNT | MS_BIND)) == (MS_REMOUNT | MS_BIND))
- retval = do_reconfigure_mnt(&path, mnt_flags);
- else if (flags & MS_REMOUNT)
- retval = do_remount(&path, flags, sb_flags, mnt_flags,
- data_page);
- else if (flags & MS_BIND)
- retval = do_loopback(&path, dev_name, flags & MS_REC);
- else if (flags & (MS_SHARED | MS_PRIVATE | MS_SLAVE | MS_UNBINDABLE))
- retval = do_change_type(&path, flags);
- else if (flags & MS_MOVE)
- retval = do_move_mount_old(&path, dev_name);
- else
- retval = do_new_mount(&path, type_page, sb_flags, mnt_flags,
- dev_name, data_page);
-dput_out:
+ return do_reconfigure_mnt(path, mnt_flags);
+ if (flags & MS_REMOUNT)
+ return do_remount(path, flags, sb_flags, mnt_flags, data_page);
+ if (flags & MS_BIND)
+ return do_loopback(path, dev_name, flags & MS_REC);
+ if (flags & (MS_SHARED | MS_PRIVATE | MS_SLAVE | MS_UNBINDABLE))
+ return do_change_type(path, flags);
+ if (flags & MS_MOVE)
+ return do_move_mount_old(path, dev_name);
+
+ return do_new_mount(path, type_page, sb_flags, mnt_flags, dev_name,
+ data_page);
+}
+
+long do_mount(const char *dev_name, const char __user *dir_name,
+ const char *type_page, unsigned long flags, void *data_page)
+{
+ struct path path;
+ int ret;
+
+ ret = user_path_at(AT_FDCWD, dir_name, LOOKUP_FOLLOW, &path);
+ if (ret)
+ return ret;
+ ret = path_mount(dev_name, &path, type_page, flags, data_page);
path_put(&path);
- return retval;
+ return ret;
}
static struct ucounts *inc_mnt_namespaces(struct user_namespace *ns)
diff --git a/fs/nfsd/filecache.c b/fs/nfsd/filecache.c
index 82198d7..c8b9d26 100644
--- a/fs/nfsd/filecache.c
+++ b/fs/nfsd/filecache.c
@@ -598,11 +598,9 @@
};
static int
-nfsd_file_fsnotify_handle_event(struct fsnotify_group *group,
- struct inode *inode,
- u32 mask, const void *data, int data_type,
- const struct qstr *file_name, u32 cookie,
- struct fsnotify_iter_info *iter_info)
+nfsd_file_fsnotify_handle_event(struct fsnotify_mark *mark, u32 mask,
+ struct inode *inode, struct inode *dir,
+ const struct qstr *name)
{
trace_nfsd_file_fsnotify_handle_event(inode, mask);
@@ -624,7 +622,7 @@
static const struct fsnotify_ops nfsd_file_fsnotify_ops = {
- .handle_event = nfsd_file_fsnotify_handle_event,
+ .handle_inode_event = nfsd_file_fsnotify_handle_event,
.free_mark = nfsd_file_mark_free,
};
diff --git a/fs/notify/dnotify/dnotify.c b/fs/notify/dnotify/dnotify.c
index 7a42c2e..5dcda8f 100644
--- a/fs/notify/dnotify/dnotify.c
+++ b/fs/notify/dnotify/dnotify.c
@@ -70,13 +70,10 @@
* destroy the dnotify struct if it was not registered to receive multiple
* events.
*/
-static int dnotify_handle_event(struct fsnotify_group *group,
- struct inode *inode,
- u32 mask, const void *data, int data_type,
- const struct qstr *file_name, u32 cookie,
- struct fsnotify_iter_info *iter_info)
+static int dnotify_handle_event(struct fsnotify_mark *inode_mark, u32 mask,
+ struct inode *inode, struct inode *dir,
+ const struct qstr *name)
{
- struct fsnotify_mark *inode_mark = fsnotify_iter_inode_mark(iter_info);
struct dnotify_mark *dn_mark;
struct dnotify_struct *dn;
struct dnotify_struct **prev;
@@ -84,10 +81,7 @@
__u32 test_mask = mask & ~FS_EVENT_ON_CHILD;
/* not a dir, dnotify doesn't care */
- if (!S_ISDIR(inode->i_mode))
- return 0;
-
- if (WARN_ON(fsnotify_iter_vfsmount_mark(iter_info)))
+ if (!dir && !(mask & FS_ISDIR))
return 0;
dn_mark = container_of(inode_mark, struct dnotify_mark, fsn_mark);
@@ -127,7 +121,7 @@
}
static const struct fsnotify_ops dnotify_fsnotify_ops = {
- .handle_event = dnotify_handle_event,
+ .handle_inode_event = dnotify_handle_event,
.free_mark = dnotify_free_mark,
};
diff --git a/fs/notify/fanotify/fanotify.c b/fs/notify/fanotify/fanotify.c
index 85eda53..c942910 100644
--- a/fs/notify/fanotify/fanotify.c
+++ b/fs/notify/fanotify/fanotify.c
@@ -34,10 +34,6 @@
if (fh1->type != fh2->type || fh1->len != fh2->len)
return false;
- /* Do not merge events if we failed to encode fh */
- if (fh1->type == FILEID_INVALID)
- return false;
-
return !fh1->len ||
!memcmp(fanotify_fh_buf(fh1), fanotify_fh_buf(fh2), fh1->len);
}
@@ -53,25 +49,47 @@
fanotify_fh_equal(&ffe1->object_fh, &ffe2->object_fh);
}
+static bool fanotify_info_equal(struct fanotify_info *info1,
+ struct fanotify_info *info2)
+{
+ if (info1->dir_fh_totlen != info2->dir_fh_totlen ||
+ info1->file_fh_totlen != info2->file_fh_totlen ||
+ info1->name_len != info2->name_len)
+ return false;
+
+ if (info1->dir_fh_totlen &&
+ !fanotify_fh_equal(fanotify_info_dir_fh(info1),
+ fanotify_info_dir_fh(info2)))
+ return false;
+
+ if (info1->file_fh_totlen &&
+ !fanotify_fh_equal(fanotify_info_file_fh(info1),
+ fanotify_info_file_fh(info2)))
+ return false;
+
+ return !info1->name_len ||
+ !memcmp(fanotify_info_name(info1), fanotify_info_name(info2),
+ info1->name_len);
+}
+
static bool fanotify_name_event_equal(struct fanotify_name_event *fne1,
struct fanotify_name_event *fne2)
{
- /*
- * Do not merge name events without dir fh.
- * FAN_DIR_MODIFY does not encode object fh, so it may be empty.
- */
- if (!fne1->dir_fh.len)
+ struct fanotify_info *info1 = &fne1->info;
+ struct fanotify_info *info2 = &fne2->info;
+
+ /* Do not merge name events without dir fh */
+ if (!info1->dir_fh_totlen)
return false;
- if (fne1->name_len != fne2->name_len ||
- !fanotify_fh_equal(&fne1->dir_fh, &fne2->dir_fh))
+ if (!fanotify_fsid_equal(&fne1->fsid, &fne2->fsid))
return false;
- return !memcmp(fne1->name, fne2->name, fne1->name_len);
+ return fanotify_info_equal(info1, info2);
}
static bool fanotify_should_merge(struct fsnotify_event *old_fsn,
- struct fsnotify_event *new_fsn)
+ struct fsnotify_event *new_fsn)
{
struct fanotify_event *old, *new;
@@ -83,22 +101,22 @@
old->type != new->type || old->pid != new->pid)
return false;
+ /*
+ * We want to merge many dirent events in the same dir (i.e.
+ * creates/unlinks/renames), but we do not want to merge dirent
+ * events referring to subdirs with dirent events referring to
+ * non subdirs, otherwise, user won't be able to tell from a
+ * mask FAN_CREATE|FAN_DELETE|FAN_ONDIR if it describes mkdir+
+ * unlink pair or rmdir+create pair of events.
+ */
+ if ((old->mask & FS_ISDIR) != (new->mask & FS_ISDIR))
+ return false;
+
switch (old->type) {
case FANOTIFY_EVENT_TYPE_PATH:
return fanotify_path_equal(fanotify_event_path(old),
fanotify_event_path(new));
case FANOTIFY_EVENT_TYPE_FID:
- /*
- * We want to merge many dirent events in the same dir (i.e.
- * creates/unlinks/renames), but we do not want to merge dirent
- * events referring to subdirs with dirent events referring to
- * non subdirs, otherwise, user won't be able to tell from a
- * mask FAN_CREATE|FAN_DELETE|FAN_ONDIR if it describes mkdir+
- * unlink pair or rmdir+create pair of events.
- */
- if ((old->mask & FS_ISDIR) != (new->mask & FS_ISDIR))
- return false;
-
return fanotify_fid_event_equal(FANOTIFY_FE(old),
FANOTIFY_FE(new));
case FANOTIFY_EVENT_TYPE_FID_NAME:
@@ -208,24 +226,30 @@
static u32 fanotify_group_event_mask(struct fsnotify_group *group,
struct fsnotify_iter_info *iter_info,
u32 event_mask, const void *data,
- int data_type)
+ int data_type, struct inode *dir)
{
__u32 marks_mask = 0, marks_ignored_mask = 0;
- __u32 test_mask, user_mask = FANOTIFY_OUTGOING_EVENTS;
+ __u32 test_mask, user_mask = FANOTIFY_OUTGOING_EVENTS |
+ FANOTIFY_EVENT_FLAGS;
const struct path *path = fsnotify_data_path(data, data_type);
+ unsigned int fid_mode = FAN_GROUP_FLAG(group, FANOTIFY_FID_BITS);
struct fsnotify_mark *mark;
int type;
pr_debug("%s: report_mask=%x mask=%x data=%p data_type=%d\n",
__func__, iter_info->report_mask, event_mask, data, data_type);
- if (!FAN_GROUP_FLAG(group, FAN_REPORT_FID)) {
+ if (!fid_mode) {
/* Do we have path to open a file descriptor? */
if (!path)
return 0;
/* Path type events are only relevant for files and dirs */
if (!d_is_reg(path->dentry) && !d_can_lookup(path->dentry))
return 0;
+ } else if (!(fid_mode & FAN_REPORT_FID)) {
+ /* Do we have a directory inode to report? */
+ if (!dir && !(event_mask & FS_ISDIR))
+ return 0;
}
fsnotify_foreach_obj_type(type) {
@@ -244,12 +268,12 @@
continue;
/*
- * If the event is for a child and this mark doesn't care about
- * events on a child, don't send it!
+ * If the event is for a child and this mark is on a parent not
+ * watching children, don't send it!
*/
if (event_mask & FS_EVENT_ON_CHILD &&
- (type != FSNOTIFY_OBJ_TYPE_INODE ||
- !(mark->mask & FS_EVENT_ON_CHILD)))
+ type == FSNOTIFY_OBJ_TYPE_INODE &&
+ !(mark->mask & FS_EVENT_ON_CHILD))
continue;
marks_mask |= mark->mask;
@@ -264,67 +288,102 @@
*
* For backward compatibility and consistency, do not report FAN_ONDIR
* to user in legacy fanotify mode (reporting fd) and report FAN_ONDIR
- * to user in FAN_REPORT_FID mode for all event types.
+ * to user in fid mode for all event types.
+ *
+ * We never report FAN_EVENT_ON_CHILD to user, but we do pass it in to
+ * fanotify_alloc_event() when group is reporting fid as indication
+ * that event happened on child.
*/
- if (FAN_GROUP_FLAG(group, FAN_REPORT_FID)) {
- /* Do not report FAN_ONDIR without any event */
- if (!(test_mask & ~FAN_ONDIR))
+ if (fid_mode) {
+ /* Do not report event flags without any event */
+ if (!(test_mask & ~FANOTIFY_EVENT_FLAGS))
return 0;
} else {
- user_mask &= ~FAN_ONDIR;
+ user_mask &= ~FANOTIFY_EVENT_FLAGS;
}
return test_mask & user_mask;
}
-static void fanotify_encode_fh(struct fanotify_fh *fh, struct inode *inode,
- gfp_t gfp)
+/*
+ * Check size needed to encode fanotify_fh.
+ *
+ * Return size of encoded fh without fanotify_fh header.
+ * Return 0 on failure to encode.
+ */
+static int fanotify_encode_fh_len(struct inode *inode)
{
- int dwords, type, bytes = 0;
+ int dwords = 0;
+
+ if (!inode)
+ return 0;
+
+ exportfs_encode_inode_fh(inode, NULL, &dwords, NULL);
+
+ return dwords << 2;
+}
+
+/*
+ * Encode fanotify_fh.
+ *
+ * Return total size of encoded fh including fanotify_fh header.
+ * Return 0 on failure to encode.
+ */
+static int fanotify_encode_fh(struct fanotify_fh *fh, struct inode *inode,
+ unsigned int fh_len, gfp_t gfp)
+{
+ int dwords, type = 0;
char *ext_buf = NULL;
void *buf = fh->buf;
int err;
+ fh->type = FILEID_ROOT;
+ fh->len = 0;
+ fh->flags = 0;
if (!inode)
- goto out;
+ return 0;
- dwords = 0;
+ /*
+ * !gpf means preallocated variable size fh, but fh_len could
+ * be zero in that case if encoding fh len failed.
+ */
err = -ENOENT;
- type = exportfs_encode_inode_fh(inode, NULL, &dwords, NULL);
- if (!dwords)
+ if (fh_len < 4 || WARN_ON_ONCE(fh_len % 4))
goto out_err;
- bytes = dwords << 2;
- if (bytes > FANOTIFY_INLINE_FH_LEN) {
- /* Treat failure to allocate fh as failure to allocate event */
+ /* No external buffer in a variable size allocated fh */
+ if (gfp && fh_len > FANOTIFY_INLINE_FH_LEN) {
+ /* Treat failure to allocate fh as failure to encode fh */
err = -ENOMEM;
- ext_buf = kmalloc(bytes, gfp);
+ ext_buf = kmalloc(fh_len, gfp);
if (!ext_buf)
goto out_err;
*fanotify_fh_ext_buf_ptr(fh) = ext_buf;
buf = ext_buf;
+ fh->flags |= FANOTIFY_FH_FLAG_EXT_BUF;
}
+ dwords = fh_len >> 2;
type = exportfs_encode_inode_fh(inode, buf, &dwords, NULL);
err = -EINVAL;
- if (!type || type == FILEID_INVALID || bytes != dwords << 2)
+ if (!type || type == FILEID_INVALID || fh_len != dwords << 2)
goto out_err;
fh->type = type;
- fh->len = bytes;
+ fh->len = fh_len;
- return;
+ return FANOTIFY_FH_HDR_LEN + fh_len;
out_err:
pr_warn_ratelimited("fanotify: failed to encode fid (type=%d, len=%d, err=%i)\n",
- type, bytes, err);
+ type, fh_len, err);
kfree(ext_buf);
*fanotify_fh_ext_buf_ptr(fh) = NULL;
-out:
/* Report the event without a file identifier on encode error */
fh->type = FILEID_INVALID;
fh->len = 0;
+ return 0;
}
/*
@@ -335,27 +394,179 @@
* FS_ATTRIB reports the child inode even if reported on a watched parent.
* FS_CREATE reports the modified dir inode and not the created inode.
*/
-static struct inode *fanotify_fid_inode(struct inode *to_tell, u32 event_mask,
- const void *data, int data_type)
+static struct inode *fanotify_fid_inode(u32 event_mask, const void *data,
+ int data_type, struct inode *dir)
{
if (event_mask & ALL_FSNOTIFY_DIRENT_EVENTS)
- return to_tell;
+ return dir;
- return (struct inode *)fsnotify_data_inode(data, data_type);
+ return fsnotify_data_inode(data, data_type);
}
-struct fanotify_event *fanotify_alloc_event(struct fsnotify_group *group,
- struct inode *inode, u32 mask,
- const void *data, int data_type,
- const struct qstr *file_name,
- __kernel_fsid_t *fsid)
+/*
+ * The inode to use as identifier when reporting dir fid depends on the event.
+ * Report the modified directory inode on dirent modification events.
+ * Report the "victim" inode if "victim" is a directory.
+ * Report the parent inode if "victim" is not a directory and event is
+ * reported to parent.
+ * Otherwise, do not report dir fid.
+ */
+static struct inode *fanotify_dfid_inode(u32 event_mask, const void *data,
+ int data_type, struct inode *dir)
+{
+ struct inode *inode = fsnotify_data_inode(data, data_type);
+
+ if (event_mask & ALL_FSNOTIFY_DIRENT_EVENTS)
+ return dir;
+
+ if (S_ISDIR(inode->i_mode))
+ return inode;
+
+ return dir;
+}
+
+static struct fanotify_event *fanotify_alloc_path_event(const struct path *path,
+ gfp_t gfp)
+{
+ struct fanotify_path_event *pevent;
+
+ pevent = kmem_cache_alloc(fanotify_path_event_cachep, gfp);
+ if (!pevent)
+ return NULL;
+
+ pevent->fae.type = FANOTIFY_EVENT_TYPE_PATH;
+ pevent->path = *path;
+ path_get(path);
+
+ return &pevent->fae;
+}
+
+static struct fanotify_event *fanotify_alloc_perm_event(const struct path *path,
+ gfp_t gfp)
+{
+ struct fanotify_perm_event *pevent;
+
+ pevent = kmem_cache_alloc(fanotify_perm_event_cachep, gfp);
+ if (!pevent)
+ return NULL;
+
+ pevent->fae.type = FANOTIFY_EVENT_TYPE_PATH_PERM;
+ pevent->response = 0;
+ pevent->state = FAN_EVENT_INIT;
+ pevent->path = *path;
+ path_get(path);
+
+ return &pevent->fae;
+}
+
+static struct fanotify_event *fanotify_alloc_fid_event(struct inode *id,
+ __kernel_fsid_t *fsid,
+ gfp_t gfp)
+{
+ struct fanotify_fid_event *ffe;
+
+ ffe = kmem_cache_alloc(fanotify_fid_event_cachep, gfp);
+ if (!ffe)
+ return NULL;
+
+ ffe->fae.type = FANOTIFY_EVENT_TYPE_FID;
+ ffe->fsid = *fsid;
+ fanotify_encode_fh(&ffe->object_fh, id, fanotify_encode_fh_len(id),
+ gfp);
+
+ return &ffe->fae;
+}
+
+static struct fanotify_event *fanotify_alloc_name_event(struct inode *id,
+ __kernel_fsid_t *fsid,
+ const struct qstr *file_name,
+ struct inode *child,
+ gfp_t gfp)
+{
+ struct fanotify_name_event *fne;
+ struct fanotify_info *info;
+ struct fanotify_fh *dfh, *ffh;
+ unsigned int dir_fh_len = fanotify_encode_fh_len(id);
+ unsigned int child_fh_len = fanotify_encode_fh_len(child);
+ unsigned int size;
+
+ size = sizeof(*fne) + FANOTIFY_FH_HDR_LEN + dir_fh_len;
+ if (child_fh_len)
+ size += FANOTIFY_FH_HDR_LEN + child_fh_len;
+ if (file_name)
+ size += file_name->len + 1;
+ fne = kmalloc(size, gfp);
+ if (!fne)
+ return NULL;
+
+ fne->fae.type = FANOTIFY_EVENT_TYPE_FID_NAME;
+ fne->fsid = *fsid;
+ info = &fne->info;
+ fanotify_info_init(info);
+ dfh = fanotify_info_dir_fh(info);
+ info->dir_fh_totlen = fanotify_encode_fh(dfh, id, dir_fh_len, 0);
+ if (child_fh_len) {
+ ffh = fanotify_info_file_fh(info);
+ info->file_fh_totlen = fanotify_encode_fh(ffh, child, child_fh_len, 0);
+ }
+ if (file_name)
+ fanotify_info_copy_name(info, file_name);
+
+ pr_debug("%s: ino=%lu size=%u dir_fh_len=%u child_fh_len=%u name_len=%u name='%.*s'\n",
+ __func__, id->i_ino, size, dir_fh_len, child_fh_len,
+ info->name_len, info->name_len, fanotify_info_name(info));
+
+ return &fne->fae;
+}
+
+static struct fanotify_event *fanotify_alloc_event(struct fsnotify_group *group,
+ u32 mask, const void *data,
+ int data_type, struct inode *dir,
+ const struct qstr *file_name,
+ __kernel_fsid_t *fsid)
{
struct fanotify_event *event = NULL;
- struct fanotify_fid_event *ffe = NULL;
- struct fanotify_name_event *fne = NULL;
gfp_t gfp = GFP_KERNEL_ACCOUNT;
- struct inode *id = fanotify_fid_inode(inode, mask, data, data_type);
+ struct inode *id = fanotify_fid_inode(mask, data, data_type, dir);
+ struct inode *dirid = fanotify_dfid_inode(mask, data, data_type, dir);
const struct path *path = fsnotify_data_path(data, data_type);
+ unsigned int fid_mode = FAN_GROUP_FLAG(group, FANOTIFY_FID_BITS);
+ struct inode *child = NULL;
+ bool name_event = false;
+
+ if ((fid_mode & FAN_REPORT_DIR_FID) && dirid) {
+ /*
+ * With both flags FAN_REPORT_DIR_FID and FAN_REPORT_FID, we
+ * report the child fid for events reported on a non-dir child
+ * in addition to reporting the parent fid and maybe child name.
+ */
+ if ((fid_mode & FAN_REPORT_FID) &&
+ id != dirid && !(mask & FAN_ONDIR))
+ child = id;
+
+ id = dirid;
+
+ /*
+ * We record file name only in a group with FAN_REPORT_NAME
+ * and when we have a directory inode to report.
+ *
+ * For directory entry modification event, we record the fid of
+ * the directory and the name of the modified entry.
+ *
+ * For event on non-directory that is reported to parent, we
+ * record the fid of the parent and the name of the child.
+ *
+ * Even if not reporting name, we need a variable length
+ * fanotify_name_event if reporting both parent and child fids.
+ */
+ if (!(fid_mode & FAN_REPORT_NAME)) {
+ name_event = !!child;
+ file_name = NULL;
+ } else if ((mask & ALL_FSNOTIFY_DIRENT_EVENTS) ||
+ !(mask & FAN_ONDIR)) {
+ name_event = true;
+ }
+ }
/*
* For queues with unlimited length lost events are not expected and
@@ -372,87 +583,30 @@
memalloc_use_memcg(group->memcg);
if (fanotify_is_perm_event(mask)) {
- struct fanotify_perm_event *pevent;
-
- pevent = kmem_cache_alloc(fanotify_perm_event_cachep, gfp);
- if (!pevent)
- goto out;
-
- event = &pevent->fae;
- event->type = FANOTIFY_EVENT_TYPE_PATH_PERM;
- pevent->response = 0;
- pevent->state = FAN_EVENT_INIT;
- goto init;
- }
-
- /*
- * For FAN_DIR_MODIFY event, we report the fid of the directory and
- * the name of the modified entry.
- * Allocate an fanotify_name_event struct and copy the name.
- */
- if (mask & FAN_DIR_MODIFY && !(WARN_ON_ONCE(!file_name))) {
- fne = kmalloc(sizeof(*fne) + file_name->len + 1, gfp);
- if (!fne)
- goto out;
-
- event = &fne->fae;
- event->type = FANOTIFY_EVENT_TYPE_FID_NAME;
- fne->name_len = file_name->len;
- strcpy(fne->name, file_name->name);
- goto init;
- }
-
- if (FAN_GROUP_FLAG(group, FAN_REPORT_FID)) {
- ffe = kmem_cache_alloc(fanotify_fid_event_cachep, gfp);
- if (!ffe)
- goto out;
-
- event = &ffe->fae;
- event->type = FANOTIFY_EVENT_TYPE_FID;
+ event = fanotify_alloc_perm_event(path, gfp);
+ } else if (name_event && (file_name || child)) {
+ event = fanotify_alloc_name_event(id, fsid, file_name, child,
+ gfp);
+ } else if (fid_mode) {
+ event = fanotify_alloc_fid_event(id, fsid, gfp);
} else {
- struct fanotify_path_event *pevent;
-
- pevent = kmem_cache_alloc(fanotify_path_event_cachep, gfp);
- if (!pevent)
- goto out;
-
- event = &pevent->fae;
- event->type = FANOTIFY_EVENT_TYPE_PATH;
+ event = fanotify_alloc_path_event(path, gfp);
}
-init:
+ if (!event)
+ goto out;
+
/*
* Use the victim inode instead of the watching inode as the id for
* event queue, so event reported on parent is merged with event
* reported on child when both directory and child watches exist.
*/
- fsnotify_init_event(&event->fse, (unsigned long)id);
- event->mask = mask;
+ fanotify_init_event(event, (unsigned long)id, mask);
if (FAN_GROUP_FLAG(group, FAN_REPORT_TID))
event->pid = get_pid(task_pid(current));
else
event->pid = get_pid(task_tgid(current));
- if (fsid && fanotify_event_fsid(event))
- *fanotify_event_fsid(event) = *fsid;
-
- if (fanotify_event_object_fh(event))
- fanotify_encode_fh(fanotify_event_object_fh(event), id, gfp);
-
- if (fanotify_event_dir_fh(event))
- fanotify_encode_fh(fanotify_event_dir_fh(event), id, gfp);
-
- if (fanotify_event_has_path(event)) {
- struct path *p = fanotify_event_path(event);
-
- if (path) {
- *p = *path;
- path_get(path);
- } else {
- p->mnt = NULL;
- p->dentry = NULL;
- }
- }
out:
memalloc_unuse_memcg();
return event;
@@ -491,9 +645,9 @@
return fsid;
}
-static int fanotify_handle_event(struct fsnotify_group *group,
- struct inode *inode,
- u32 mask, const void *data, int data_type,
+static int fanotify_handle_event(struct fsnotify_group *group, u32 mask,
+ const void *data, int data_type,
+ struct inode *dir,
const struct qstr *file_name, u32 cookie,
struct fsnotify_iter_info *iter_info)
{
@@ -512,7 +666,6 @@
BUILD_BUG_ON(FAN_MOVED_FROM != FS_MOVED_FROM);
BUILD_BUG_ON(FAN_CREATE != FS_CREATE);
BUILD_BUG_ON(FAN_DELETE != FS_DELETE);
- BUILD_BUG_ON(FAN_DIR_MODIFY != FS_DIR_MODIFY);
BUILD_BUG_ON(FAN_DELETE_SELF != FS_DELETE_SELF);
BUILD_BUG_ON(FAN_MOVE_SELF != FS_MOVE_SELF);
BUILD_BUG_ON(FAN_EVENT_ON_CHILD != FS_EVENT_ON_CHILD);
@@ -526,12 +679,11 @@
BUILD_BUG_ON(HWEIGHT32(ALL_FANOTIFY_EVENT_BITS) != 19);
mask = fanotify_group_event_mask(group, iter_info, mask, data,
- data_type);
+ data_type, dir);
if (!mask)
return 0;
- pr_debug("%s: group=%p inode=%p mask=%x\n", __func__, group, inode,
- mask);
+ pr_debug("%s: group=%p mask=%x\n", __func__, group, mask);
if (fanotify_is_perm_event(mask)) {
/*
@@ -542,14 +694,14 @@
return 0;
}
- if (FAN_GROUP_FLAG(group, FAN_REPORT_FID)) {
+ if (FAN_GROUP_FLAG(group, FANOTIFY_FID_BITS)) {
fsid = fanotify_get_fsid(iter_info);
/* Racing with mark destruction or creation? */
if (!fsid.val[0] && !fsid.val[1])
return 0;
}
- event = fanotify_alloc_event(group, inode, mask, data, data_type,
+ event = fanotify_alloc_event(group, mask, data, data_type, dir,
file_name, &fsid);
ret = -ENOMEM;
if (unlikely(!event)) {
@@ -614,11 +766,7 @@
static void fanotify_free_name_event(struct fanotify_event *event)
{
- struct fanotify_name_event *fne = FANOTIFY_NE(event);
-
- if (fanotify_fh_has_ext_buf(&fne->dir_fh))
- kfree(fanotify_fh_ext_buf(&fne->dir_fh));
- kfree(fne);
+ kfree(FANOTIFY_NE(event));
}
static void fanotify_free_event(struct fsnotify_event *fsn_event)
@@ -640,6 +788,9 @@
case FANOTIFY_EVENT_TYPE_FID_NAME:
fanotify_free_name_event(event);
break;
+ case FANOTIFY_EVENT_TYPE_OVERFLOW:
+ kfree(event);
+ break;
default:
WARN_ON_ONCE(1);
}
diff --git a/fs/notify/fanotify/fanotify.h b/fs/notify/fanotify/fanotify.h
index 8ce7ccf..896c819 100644
--- a/fs/notify/fanotify/fanotify.h
+++ b/fs/notify/fanotify/fanotify.h
@@ -23,20 +23,41 @@
* stored in either the first or last 2 dwords.
*/
#define FANOTIFY_INLINE_FH_LEN (3 << 2)
+#define FANOTIFY_FH_HDR_LEN offsetof(struct fanotify_fh, buf)
+/* Fixed size struct for file handle */
struct fanotify_fh {
- unsigned char buf[FANOTIFY_INLINE_FH_LEN];
u8 type;
u8 len;
+#define FANOTIFY_FH_FLAG_EXT_BUF 1
+ u8 flags;
+ u8 pad;
+ unsigned char buf[];
+} __aligned(4);
+
+/* Variable size struct for dir file handle + child file handle + name */
+struct fanotify_info {
+ /* size of dir_fh/file_fh including fanotify_fh hdr size */
+ u8 dir_fh_totlen;
+ u8 file_fh_totlen;
+ u8 name_len;
+ u8 pad;
+ unsigned char buf[];
+ /*
+ * (struct fanotify_fh) dir_fh starts at buf[0]
+ * (optional) file_fh starts at buf[dir_fh_totlen]
+ * name starts at buf[dir_fh_totlen + file_fh_totlen]
+ */
} __aligned(4);
static inline bool fanotify_fh_has_ext_buf(struct fanotify_fh *fh)
{
- return fh->len > FANOTIFY_INLINE_FH_LEN;
+ return (fh->flags & FANOTIFY_FH_FLAG_EXT_BUF);
}
static inline char **fanotify_fh_ext_buf_ptr(struct fanotify_fh *fh)
{
+ BUILD_BUG_ON(FANOTIFY_FH_HDR_LEN % 4);
BUILD_BUG_ON(__alignof__(char *) - 4 + sizeof(char *) >
FANOTIFY_INLINE_FH_LEN);
return (char **)ALIGN((unsigned long)(fh->buf), __alignof__(char *));
@@ -52,6 +73,56 @@
return fanotify_fh_has_ext_buf(fh) ? fanotify_fh_ext_buf(fh) : fh->buf;
}
+static inline int fanotify_info_dir_fh_len(struct fanotify_info *info)
+{
+ if (!info->dir_fh_totlen ||
+ WARN_ON_ONCE(info->dir_fh_totlen < FANOTIFY_FH_HDR_LEN))
+ return 0;
+
+ return info->dir_fh_totlen - FANOTIFY_FH_HDR_LEN;
+}
+
+static inline struct fanotify_fh *fanotify_info_dir_fh(struct fanotify_info *info)
+{
+ BUILD_BUG_ON(offsetof(struct fanotify_info, buf) % 4);
+
+ return (struct fanotify_fh *)info->buf;
+}
+
+static inline int fanotify_info_file_fh_len(struct fanotify_info *info)
+{
+ if (!info->file_fh_totlen ||
+ WARN_ON_ONCE(info->file_fh_totlen < FANOTIFY_FH_HDR_LEN))
+ return 0;
+
+ return info->file_fh_totlen - FANOTIFY_FH_HDR_LEN;
+}
+
+static inline struct fanotify_fh *fanotify_info_file_fh(struct fanotify_info *info)
+{
+ return (struct fanotify_fh *)(info->buf + info->dir_fh_totlen);
+}
+
+static inline const char *fanotify_info_name(struct fanotify_info *info)
+{
+ return info->buf + info->dir_fh_totlen + info->file_fh_totlen;
+}
+
+static inline void fanotify_info_init(struct fanotify_info *info)
+{
+ info->dir_fh_totlen = 0;
+ info->file_fh_totlen = 0;
+ info->name_len = 0;
+}
+
+static inline void fanotify_info_copy_name(struct fanotify_info *info,
+ const struct qstr *name)
+{
+ info->name_len = name->len;
+ strcpy(info->buf + info->dir_fh_totlen + info->file_fh_totlen,
+ name->name);
+}
+
/*
* Common structure for fanotify events. Concrete structs are allocated in
* fanotify_handle_event() and freed when the information is retrieved by
@@ -63,6 +134,7 @@
FANOTIFY_EVENT_TYPE_FID_NAME, /* variable length */
FANOTIFY_EVENT_TYPE_PATH,
FANOTIFY_EVENT_TYPE_PATH_PERM,
+ FANOTIFY_EVENT_TYPE_OVERFLOW, /* struct fanotify_event */
};
struct fanotify_event {
@@ -72,10 +144,20 @@
struct pid *pid;
};
+static inline void fanotify_init_event(struct fanotify_event *event,
+ unsigned long id, u32 mask)
+{
+ fsnotify_init_event(&event->fse, id);
+ event->mask = mask;
+ event->pid = NULL;
+}
+
struct fanotify_fid_event {
struct fanotify_event fae;
__kernel_fsid_t fsid;
struct fanotify_fh object_fh;
+ /* Reserve space in object_fh.buf[] - access with fanotify_fh_buf() */
+ unsigned char _inline_fh_buf[FANOTIFY_INLINE_FH_LEN];
};
static inline struct fanotify_fid_event *
@@ -87,9 +169,7 @@
struct fanotify_name_event {
struct fanotify_event fae;
__kernel_fsid_t fsid;
- struct fanotify_fh dir_fh;
- u8 name_len;
- char name[];
+ struct fanotify_info info;
};
static inline struct fanotify_name_event *
@@ -113,35 +193,37 @@
{
if (event->type == FANOTIFY_EVENT_TYPE_FID)
return &FANOTIFY_FE(event)->object_fh;
+ else if (event->type == FANOTIFY_EVENT_TYPE_FID_NAME)
+ return fanotify_info_file_fh(&FANOTIFY_NE(event)->info);
else
return NULL;
}
-static inline struct fanotify_fh *fanotify_event_dir_fh(
+static inline struct fanotify_info *fanotify_event_info(
struct fanotify_event *event)
{
if (event->type == FANOTIFY_EVENT_TYPE_FID_NAME)
- return &FANOTIFY_NE(event)->dir_fh;
+ return &FANOTIFY_NE(event)->info;
else
return NULL;
}
static inline int fanotify_event_object_fh_len(struct fanotify_event *event)
{
+ struct fanotify_info *info = fanotify_event_info(event);
struct fanotify_fh *fh = fanotify_event_object_fh(event);
- return fh ? fh->len : 0;
+ if (info)
+ return info->file_fh_totlen ? fh->len : 0;
+ else
+ return fh ? fh->len : 0;
}
-static inline bool fanotify_event_has_name(struct fanotify_event *event)
+static inline int fanotify_event_dir_fh_len(struct fanotify_event *event)
{
- return event->type == FANOTIFY_EVENT_TYPE_FID_NAME;
-}
+ struct fanotify_info *info = fanotify_event_info(event);
-static inline int fanotify_event_name_len(struct fanotify_event *event)
-{
- return fanotify_event_has_name(event) ?
- FANOTIFY_NE(event)->name_len : 0;
+ return info ? fanotify_info_dir_fh_len(info) : 0;
}
struct fanotify_path_event {
@@ -202,9 +284,3 @@
else
return NULL;
}
-
-struct fanotify_event *fanotify_alloc_event(struct fsnotify_group *group,
- struct inode *inode, u32 mask,
- const void *data, int data_type,
- const struct qstr *file_name,
- __kernel_fsid_t *fsid);
diff --git a/fs/notify/fanotify/fanotify_user.c b/fs/notify/fanotify/fanotify_user.c
index 63b5dff..559de31 100644
--- a/fs/notify/fanotify/fanotify_user.c
+++ b/fs/notify/fanotify/fanotify_user.c
@@ -64,20 +64,27 @@
return roundup(FANOTIFY_INFO_HDR_LEN + info_len, FANOTIFY_EVENT_ALIGN);
}
-static int fanotify_event_info_len(struct fanotify_event *event)
+static int fanotify_event_info_len(unsigned int fid_mode,
+ struct fanotify_event *event)
{
- int info_len = 0;
+ struct fanotify_info *info = fanotify_event_info(event);
+ int dir_fh_len = fanotify_event_dir_fh_len(event);
int fh_len = fanotify_event_object_fh_len(event);
+ int info_len = 0;
+ int dot_len = 0;
+
+ if (dir_fh_len) {
+ info_len += fanotify_fid_info_len(dir_fh_len, info->name_len);
+ } else if ((fid_mode & FAN_REPORT_NAME) && (event->mask & FAN_ONDIR)) {
+ /*
+ * With group flag FAN_REPORT_NAME, if name was not recorded in
+ * event on a directory, we will report the name ".".
+ */
+ dot_len = 1;
+ }
if (fh_len)
- info_len += fanotify_fid_info_len(fh_len, 0);
-
- if (fanotify_event_name_len(event)) {
- struct fanotify_name_event *fne = FANOTIFY_NE(event);
-
- info_len += fanotify_fid_info_len(fne->dir_fh.len,
- fne->name_len);
- }
+ info_len += fanotify_fid_info_len(fh_len, dot_len);
return info_len;
}
@@ -93,6 +100,7 @@
{
size_t event_size = FAN_EVENT_METADATA_LEN;
struct fanotify_event *event = NULL;
+ unsigned int fid_mode = FAN_GROUP_FLAG(group, FANOTIFY_FID_BITS);
pr_debug("%s: group=%p count=%zd\n", __func__, group, count);
@@ -100,8 +108,8 @@
if (fsnotify_notify_queue_is_empty(group))
goto out;
- if (FAN_GROUP_FLAG(group, FAN_REPORT_FID)) {
- event_size += fanotify_event_info_len(
+ if (fid_mode) {
+ event_size += fanotify_event_info_len(fid_mode,
FANOTIFY_E(fsnotify_peek_first_event(group)));
}
@@ -218,7 +226,7 @@
}
static int copy_info_to_user(__kernel_fsid_t *fsid, struct fanotify_fh *fh,
- const char *name, size_t name_len,
+ int info_type, const char *name, size_t name_len,
char __user *buf, size_t count)
{
struct fanotify_event_info_fid info = { };
@@ -231,7 +239,7 @@
pr_debug("%s: fh_len=%zu name_len=%zu, info_len=%zu, count=%zu\n",
__func__, fh_len, name_len, info_len, count);
- if (!fh_len || (name && !name_len))
+ if (!fh_len)
return 0;
if (WARN_ON_ONCE(len < sizeof(info) || len > count))
@@ -241,8 +249,21 @@
* Copy event info fid header followed by variable sized file handle
* and optionally followed by variable sized filename.
*/
- info.hdr.info_type = name_len ? FAN_EVENT_INFO_TYPE_DFID_NAME :
- FAN_EVENT_INFO_TYPE_FID;
+ switch (info_type) {
+ case FAN_EVENT_INFO_TYPE_FID:
+ case FAN_EVENT_INFO_TYPE_DFID:
+ if (WARN_ON_ONCE(name_len))
+ return -EFAULT;
+ break;
+ case FAN_EVENT_INFO_TYPE_DFID_NAME:
+ if (WARN_ON_ONCE(!name || !name_len))
+ return -EFAULT;
+ break;
+ default:
+ return -EFAULT;
+ }
+
+ info.hdr.info_type = info_type;
info.hdr.len = len;
info.fsid = *fsid;
if (copy_to_user(buf, &info, sizeof(info)))
@@ -305,13 +326,16 @@
{
struct fanotify_event_metadata metadata;
struct path *path = fanotify_event_path(event);
+ struct fanotify_info *info = fanotify_event_info(event);
+ unsigned int fid_mode = FAN_GROUP_FLAG(group, FANOTIFY_FID_BITS);
struct file *f = NULL;
int ret, fd = FAN_NOFD;
+ int info_type = 0;
pr_debug("%s: group=%p event=%p\n", __func__, group, event);
metadata.event_len = FAN_EVENT_METADATA_LEN +
- fanotify_event_info_len(event);
+ fanotify_event_info_len(fid_mode, event);
metadata.metadata_len = FAN_EVENT_METADATA_LEN;
metadata.vers = FANOTIFY_METADATA_VERSION;
metadata.reserved = 0;
@@ -346,13 +370,13 @@
fd_install(fd, f);
/* Event info records order is: dir fid + name, child fid */
- if (fanotify_event_name_len(event)) {
- struct fanotify_name_event *fne = FANOTIFY_NE(event);
-
+ if (fanotify_event_dir_fh_len(event)) {
+ info_type = info->name_len ? FAN_EVENT_INFO_TYPE_DFID_NAME :
+ FAN_EVENT_INFO_TYPE_DFID;
ret = copy_info_to_user(fanotify_event_fsid(event),
- fanotify_event_dir_fh(event),
- fne->name, fne->name_len,
- buf, count);
+ fanotify_info_dir_fh(info),
+ info_type, fanotify_info_name(info),
+ info->name_len, buf, count);
if (ret < 0)
return ret;
@@ -361,9 +385,46 @@
}
if (fanotify_event_object_fh_len(event)) {
+ const char *dot = NULL;
+ int dot_len = 0;
+
+ if (fid_mode == FAN_REPORT_FID || info_type) {
+ /*
+ * With only group flag FAN_REPORT_FID only type FID is
+ * reported. Second info record type is always FID.
+ */
+ info_type = FAN_EVENT_INFO_TYPE_FID;
+ } else if ((fid_mode & FAN_REPORT_NAME) &&
+ (event->mask & FAN_ONDIR)) {
+ /*
+ * With group flag FAN_REPORT_NAME, if name was not
+ * recorded in an event on a directory, report the
+ * name "." with info type DFID_NAME.
+ */
+ info_type = FAN_EVENT_INFO_TYPE_DFID_NAME;
+ dot = ".";
+ dot_len = 1;
+ } else if ((event->mask & ALL_FSNOTIFY_DIRENT_EVENTS) ||
+ (event->mask & FAN_ONDIR)) {
+ /*
+ * With group flag FAN_REPORT_DIR_FID, a single info
+ * record has type DFID for directory entry modification
+ * event and for event on a directory.
+ */
+ info_type = FAN_EVENT_INFO_TYPE_DFID;
+ } else {
+ /*
+ * With group flags FAN_REPORT_DIR_FID|FAN_REPORT_FID,
+ * a single info record has type FID for event on a
+ * non-directory, when there is no directory to report.
+ * For example, on FAN_DELETE_SELF event.
+ */
+ info_type = FAN_EVENT_INFO_TYPE_FID;
+ }
+
ret = copy_info_to_user(fanotify_event_fsid(event),
fanotify_event_object_fh(event),
- NULL, 0, buf, count);
+ info_type, dot, dot_len, buf, count);
if (ret < 0)
return ret;
@@ -412,6 +473,11 @@
add_wait_queue(&group->notification_waitq, &wait);
while (1) {
+ /*
+ * User can supply arbitrarily large buffer. Avoid softlockups
+ * in case there are lots of available events.
+ */
+ cond_resched();
event = get_one_event(group, count);
if (IS_ERR(event)) {
ret = PTR_ERR(event);
@@ -651,12 +717,13 @@
}
static __u32 fanotify_mark_remove_from_mask(struct fsnotify_mark *fsn_mark,
- __u32 mask,
- unsigned int flags,
- int *destroy)
+ __u32 mask, unsigned int flags,
+ __u32 umask, int *destroy)
{
__u32 oldmask = 0;
+ /* umask bits cannot be removed by user */
+ mask &= ~umask;
spin_lock(&fsn_mark->lock);
if (!(flags & FAN_MARK_IGNORED_MASK)) {
oldmask = fsn_mark->mask;
@@ -664,7 +731,13 @@
} else {
fsn_mark->ignored_mask &= ~mask;
}
- *destroy = !(fsn_mark->mask | fsn_mark->ignored_mask);
+ /*
+ * We need to keep the mark around even if remaining mask cannot
+ * result in any events (e.g. mask == FAN_ONDIR) to support incremenal
+ * changes to the mask.
+ * Destroy mark when only umask bits remain.
+ */
+ *destroy = !((fsn_mark->mask | fsn_mark->ignored_mask) & ~umask);
spin_unlock(&fsn_mark->lock);
return mask & oldmask;
@@ -672,7 +745,7 @@
static int fanotify_remove_mark(struct fsnotify_group *group,
fsnotify_connp_t *connp, __u32 mask,
- unsigned int flags)
+ unsigned int flags, __u32 umask)
{
struct fsnotify_mark *fsn_mark = NULL;
__u32 removed;
@@ -686,7 +759,7 @@
}
removed = fanotify_mark_remove_from_mask(fsn_mark, mask, flags,
- &destroy_mark);
+ umask, &destroy_mark);
if (removed & fsnotify_conn_mask(fsn_mark->connector))
fsnotify_recalc_mask(fsn_mark->connector);
if (destroy_mark)
@@ -702,25 +775,26 @@
static int fanotify_remove_vfsmount_mark(struct fsnotify_group *group,
struct vfsmount *mnt, __u32 mask,
- unsigned int flags)
+ unsigned int flags, __u32 umask)
{
return fanotify_remove_mark(group, &real_mount(mnt)->mnt_fsnotify_marks,
- mask, flags);
+ mask, flags, umask);
}
static int fanotify_remove_sb_mark(struct fsnotify_group *group,
- struct super_block *sb, __u32 mask,
- unsigned int flags)
+ struct super_block *sb, __u32 mask,
+ unsigned int flags, __u32 umask)
{
- return fanotify_remove_mark(group, &sb->s_fsnotify_marks, mask, flags);
+ return fanotify_remove_mark(group, &sb->s_fsnotify_marks, mask,
+ flags, umask);
}
static int fanotify_remove_inode_mark(struct fsnotify_group *group,
struct inode *inode, __u32 mask,
- unsigned int flags)
+ unsigned int flags, __u32 umask)
{
return fanotify_remove_mark(group, &inode->i_fsnotify_marks, mask,
- flags);
+ flags, umask);
}
static __u32 fanotify_mark_add_to_mask(struct fsnotify_mark *fsn_mark,
@@ -831,13 +905,28 @@
FSNOTIFY_OBJ_TYPE_INODE, mask, flags, fsid);
}
+static struct fsnotify_event *fanotify_alloc_overflow_event(void)
+{
+ struct fanotify_event *oevent;
+
+ oevent = kmalloc(sizeof(*oevent), GFP_KERNEL_ACCOUNT);
+ if (!oevent)
+ return NULL;
+
+ fanotify_init_event(oevent, 0, FS_Q_OVERFLOW);
+ oevent->type = FANOTIFY_EVENT_TYPE_OVERFLOW;
+
+ return &oevent->fse;
+}
+
/* fanotify syscalls */
SYSCALL_DEFINE2(fanotify_init, unsigned int, flags, unsigned int, event_f_flags)
{
struct fsnotify_group *group;
int f_flags, fd;
struct user_struct *user;
- struct fanotify_event *oevent;
+ unsigned int fid_mode = flags & FANOTIFY_FID_BITS;
+ unsigned int class = flags & FANOTIFY_CLASS_BITS;
pr_debug("%s: flags=%x event_f_flags=%x\n",
__func__, flags, event_f_flags);
@@ -864,8 +953,14 @@
return -EINVAL;
}
- if ((flags & FAN_REPORT_FID) &&
- (flags & FANOTIFY_CLASS_BITS) != FAN_CLASS_NOTIF)
+ if (fid_mode && class != FAN_CLASS_NOTIF)
+ return -EINVAL;
+
+ /*
+ * Child name is reported with parent fid so requires dir fid.
+ * We can report both child fid and dir fid with or without name.
+ */
+ if ((fid_mode & FAN_REPORT_NAME) && !(fid_mode & FAN_REPORT_DIR_FID))
return -EINVAL;
user = get_current_user();
@@ -892,20 +987,18 @@
atomic_inc(&user->fanotify_listeners);
group->memcg = get_mem_cgroup_from_mm(current->mm);
- oevent = fanotify_alloc_event(group, NULL, FS_Q_OVERFLOW, NULL,
- FSNOTIFY_EVENT_NONE, NULL, NULL);
- if (unlikely(!oevent)) {
+ group->overflow_event = fanotify_alloc_overflow_event();
+ if (unlikely(!group->overflow_event)) {
fd = -ENOMEM;
goto out_destroy_group;
}
- group->overflow_event = &oevent->fse;
if (force_o_largefile())
event_f_flags |= O_LARGEFILE;
group->fanotify_data.f_flags = event_f_flags;
init_waitqueue_head(&group->fanotify_data.access_waitq);
INIT_LIST_HEAD(&group->fanotify_data.access_list);
- switch (flags & FANOTIFY_CLASS_BITS) {
+ switch (class) {
case FAN_CLASS_NOTIF:
group->priority = FS_PRIO_0;
break;
@@ -1024,7 +1117,9 @@
__kernel_fsid_t __fsid, *fsid = NULL;
u32 valid_mask = FANOTIFY_EVENTS | FANOTIFY_EVENT_FLAGS;
unsigned int mark_type = flags & FANOTIFY_MARK_TYPE_BITS;
- unsigned int obj_type;
+ bool ignored = flags & FAN_MARK_IGNORED_MASK;
+ unsigned int obj_type, fid_mode;
+ u32 umask = 0;
int ret;
pr_debug("%s: fanotify_fd=%d flags=%x dfd=%d pathname=%p mask=%llx\n",
@@ -1071,6 +1166,10 @@
if (mask & ~valid_mask)
return -EINVAL;
+ /* Event flags (ONDIR, ON_CHILD) are meaningless in ignored mask */
+ if (ignored)
+ mask &= ~FANOTIFY_EVENT_FLAGS;
+
f = fdget(fanotify_fd);
if (unlikely(!f.file))
return -EBADF;
@@ -1097,9 +1196,9 @@
* inode events are not supported on a mount mark, because they do not
* carry enough information (i.e. path) to be filtered by mount point.
*/
+ fid_mode = FAN_GROUP_FLAG(group, FANOTIFY_FID_BITS);
if (mask & FANOTIFY_INODE_EVENTS &&
- (!FAN_GROUP_FLAG(group, FAN_REPORT_FID) ||
- mark_type == FAN_MARK_MOUNT))
+ (!fid_mode || mark_type == FAN_MARK_MOUNT))
goto fput_and_out;
if (flags & FAN_MARK_FLUSH) {
@@ -1124,7 +1223,7 @@
goto path_put_and_out;
}
- if (FAN_GROUP_FLAG(group, FAN_REPORT_FID)) {
+ if (fid_mode) {
ret = fanotify_test_fid(&path, &__fsid);
if (ret)
goto path_put_and_out;
@@ -1138,6 +1237,19 @@
else
mnt = path.mnt;
+ /* Mask out FAN_EVENT_ON_CHILD flag for sb/mount/non-dir marks */
+ if (mnt || !S_ISDIR(inode->i_mode)) {
+ mask &= ~FAN_EVENT_ON_CHILD;
+ umask = FAN_EVENT_ON_CHILD;
+ /*
+ * If group needs to report parent fid, register for getting
+ * events with parent/name info for non-directory.
+ */
+ if ((fid_mode & FAN_REPORT_DIR_FID) &&
+ (flags & FAN_MARK_ADD) && !ignored)
+ mask |= FAN_EVENT_ON_CHILD;
+ }
+
/* create/update an inode mark */
switch (flags & (FAN_MARK_ADD | FAN_MARK_REMOVE)) {
case FAN_MARK_ADD:
@@ -1154,13 +1266,13 @@
case FAN_MARK_REMOVE:
if (mark_type == FAN_MARK_MOUNT)
ret = fanotify_remove_vfsmount_mark(group, mnt, mask,
- flags);
+ flags, umask);
else if (mark_type == FAN_MARK_FILESYSTEM)
ret = fanotify_remove_sb_mark(group, mnt->mnt_sb, mask,
- flags);
+ flags, umask);
else
ret = fanotify_remove_inode_mark(group, inode, mask,
- flags);
+ flags, umask);
break;
default:
ret = -EINVAL;
@@ -1203,7 +1315,7 @@
*/
static int __init fanotify_user_setup(void)
{
- BUILD_BUG_ON(HWEIGHT32(FANOTIFY_INIT_FLAGS) != 8);
+ BUILD_BUG_ON(HWEIGHT32(FANOTIFY_INIT_FLAGS) != 10);
BUILD_BUG_ON(HWEIGHT32(FANOTIFY_MARK_FLAGS) != 9);
fanotify_mark_cache = KMEM_CACHE(fsnotify_mark,
diff --git a/fs/notify/fsnotify.c b/fs/notify/fsnotify.c
index 72d332c..a960ec3 100644
--- a/fs/notify/fsnotify.c
+++ b/fs/notify/fsnotify.c
@@ -74,7 +74,7 @@
iput(iput_inode);
/* for each watch, send FS_UNMOUNT and then remove it */
- fsnotify(inode, FS_UNMOUNT, inode, FSNOTIFY_EVENT_INODE, NULL, 0);
+ fsnotify_inode(inode, FS_UNMOUNT);
fsnotify_inode_delete(inode);
@@ -142,45 +142,140 @@
spin_unlock(&inode->i_lock);
}
-/* Notify this dentry's parent about a child's events. */
-int fsnotify_parent(struct dentry *dentry, __u32 mask, const void *data,
- int data_type)
+/* Are inode/sb/mount interested in parent and name info with this event? */
+static bool fsnotify_event_needs_parent(struct inode *inode, struct mount *mnt,
+ __u32 mask)
{
+ __u32 marks_mask = 0;
+
+ /* We only send parent/name to inode/sb/mount for events on non-dir */
+ if (mask & FS_ISDIR)
+ return false;
+
+ /* Did either inode/sb/mount subscribe for events with parent/name? */
+ marks_mask |= fsnotify_parent_needed_mask(inode->i_fsnotify_mask);
+ marks_mask |= fsnotify_parent_needed_mask(inode->i_sb->s_fsnotify_mask);
+ if (mnt)
+ marks_mask |= fsnotify_parent_needed_mask(mnt->mnt_fsnotify_mask);
+
+ /* Did they subscribe for this event with parent/name info? */
+ return mask & marks_mask;
+}
+
+/*
+ * Notify this dentry's parent about a child's events with child name info
+ * if parent is watching or if inode/sb/mount are interested in events with
+ * parent and name info.
+ *
+ * Notify only the child without name info if parent is not watching and
+ * inode/sb/mount are not interested in events with parent and name info.
+ */
+int __fsnotify_parent(struct dentry *dentry, __u32 mask, const void *data,
+ int data_type)
+{
+ const struct path *path = fsnotify_data_path(data, data_type);
+ struct mount *mnt = path ? real_mount(path->mnt) : NULL;
+ struct inode *inode = d_inode(dentry);
struct dentry *parent;
- struct inode *p_inode;
+ bool parent_watched = dentry->d_flags & DCACHE_FSNOTIFY_PARENT_WATCHED;
+ __u32 p_mask;
+ struct inode *p_inode = NULL;
+ struct name_snapshot name;
+ struct qstr *file_name = NULL;
int ret = 0;
- if (!(dentry->d_flags & DCACHE_FSNOTIFY_PARENT_WATCHED))
+ /*
+ * Do inode/sb/mount care about parent and name info on non-dir?
+ * Do they care about any event at all?
+ */
+ if (!inode->i_fsnotify_marks && !inode->i_sb->s_fsnotify_marks &&
+ (!mnt || !mnt->mnt_fsnotify_marks) && !parent_watched)
return 0;
+ parent = NULL;
+ if (!parent_watched && !fsnotify_event_needs_parent(inode, mnt, mask))
+ goto notify;
+
+ /* Does parent inode care about events on children? */
parent = dget_parent(dentry);
p_inode = parent->d_inode;
-
- if (unlikely(!fsnotify_inode_watches_children(p_inode))) {
+ p_mask = fsnotify_inode_watches_children(p_inode);
+ if (unlikely(parent_watched && !p_mask))
__fsnotify_update_child_dentry_flags(p_inode);
- } else if (p_inode->i_fsnotify_mask & mask & ALL_FSNOTIFY_EVENTS) {
- struct name_snapshot name;
- /* we are notifying a parent so come up with the new mask which
- * specifies these are events which came from a child. */
- mask |= FS_EVENT_ON_CHILD;
+ /*
+ * Include parent/name in notification either if some notification
+ * groups require parent info (!parent_watched case) or the parent is
+ * interested in this event.
+ */
+ if (!parent_watched || (mask & p_mask & ALL_FSNOTIFY_EVENTS)) {
+ /* When notifying parent, child should be passed as data */
+ WARN_ON_ONCE(inode != fsnotify_data_inode(data, data_type));
+ /* Notify both parent and child with child name info */
take_dentry_name_snapshot(&name, dentry);
- ret = fsnotify(p_inode, mask, data, data_type, &name.name, 0);
- release_dentry_name_snapshot(&name);
+ file_name = &name.name;
+ if (parent_watched)
+ mask |= FS_EVENT_ON_CHILD;
}
+notify:
+ ret = fsnotify(mask, data, data_type, p_inode, file_name, inode, 0);
+
+ if (file_name)
+ release_dentry_name_snapshot(&name);
dput(parent);
return ret;
}
-EXPORT_SYMBOL_GPL(fsnotify_parent);
+EXPORT_SYMBOL_GPL(__fsnotify_parent);
-static int send_to_group(struct inode *to_tell,
- __u32 mask, const void *data,
- int data_is, u32 cookie,
- const struct qstr *file_name,
- struct fsnotify_iter_info *iter_info)
+static int fsnotify_handle_event(struct fsnotify_group *group, __u32 mask,
+ const void *data, int data_type,
+ struct inode *dir, const struct qstr *name,
+ u32 cookie, struct fsnotify_iter_info *iter_info)
+{
+ struct fsnotify_mark *inode_mark = fsnotify_iter_inode_mark(iter_info);
+ struct fsnotify_mark *child_mark = fsnotify_iter_child_mark(iter_info);
+ struct inode *inode = fsnotify_data_inode(data, data_type);
+ const struct fsnotify_ops *ops = group->ops;
+ int ret;
+
+ if (WARN_ON_ONCE(!ops->handle_inode_event))
+ return 0;
+
+ if (WARN_ON_ONCE(fsnotify_iter_sb_mark(iter_info)) ||
+ WARN_ON_ONCE(fsnotify_iter_vfsmount_mark(iter_info)))
+ return 0;
+
+ /*
+ * An event can be sent on child mark iterator instead of inode mark
+ * iterator because of other groups that have interest of this inode
+ * and have marks on both parent and child. We can simplify this case.
+ */
+ if (!inode_mark) {
+ inode_mark = child_mark;
+ child_mark = NULL;
+ dir = NULL;
+ name = NULL;
+ }
+
+ ret = ops->handle_inode_event(inode_mark, mask, inode, dir, name);
+ if (ret || !child_mark)
+ return ret;
+
+ /*
+ * Some events can be sent on both parent dir and child marks
+ * (e.g. FS_ATTRIB). If both parent dir and child are watching,
+ * report the event once to parent dir with name and once to child
+ * without name.
+ */
+ return ops->handle_inode_event(child_mark, mask, inode, NULL, NULL);
+}
+
+static int send_to_group(__u32 mask, const void *data, int data_type,
+ struct inode *dir, const struct qstr *file_name,
+ u32 cookie, struct fsnotify_iter_info *iter_info)
{
struct fsnotify_group *group = NULL;
__u32 test_mask = (mask & ALL_FSNOTIFY_EVENTS);
@@ -216,16 +311,20 @@
}
}
- pr_debug("%s: group=%p to_tell=%p mask=%x marks_mask=%x marks_ignored_mask=%x"
- " data=%p data_is=%d cookie=%d\n",
- __func__, group, to_tell, mask, marks_mask, marks_ignored_mask,
- data, data_is, cookie);
+ pr_debug("%s: group=%p mask=%x marks_mask=%x marks_ignored_mask=%x data=%p data_type=%d dir=%p cookie=%d\n",
+ __func__, group, mask, marks_mask, marks_ignored_mask,
+ data, data_type, dir, cookie);
if (!(test_mask & marks_mask & ~marks_ignored_mask))
return 0;
- return group->ops->handle_event(group, to_tell, mask, data, data_is,
- file_name, cookie, iter_info);
+ if (group->ops->handle_event) {
+ return group->ops->handle_event(group, mask, data, data_type, dir,
+ file_name, cookie, iter_info);
+ }
+
+ return fsnotify_handle_event(group, mask, data, data_type, dir,
+ file_name, cookie, iter_info);
}
static struct fsnotify_mark *fsnotify_first_mark(struct fsnotify_mark_connector **connp)
@@ -303,29 +402,51 @@
}
/*
- * This is the main call to fsnotify. The VFS calls into hook specific functions
- * in linux/fsnotify.h. Those functions then in turn call here. Here will call
- * out to all of the registered fsnotify_group. Those groups can then use the
- * notification event in whatever means they feel necessary.
+ * fsnotify - This is the main call to fsnotify.
+ *
+ * The VFS calls into hook specific functions in linux/fsnotify.h.
+ * Those functions then in turn call here. Here will call out to all of the
+ * registered fsnotify_group. Those groups can then use the notification event
+ * in whatever means they feel necessary.
+ *
+ * @mask: event type and flags
+ * @data: object that event happened on
+ * @data_type: type of object for fanotify_data_XXX() accessors
+ * @dir: optional directory associated with event -
+ * if @file_name is not NULL, this is the directory that
+ * @file_name is relative to
+ * @file_name: optional file name associated with event
+ * @inode: optional inode associated with event -
+ * either @dir or @inode must be non-NULL.
+ * if both are non-NULL event may be reported to both.
+ * @cookie: inotify rename cookie
*/
-int fsnotify(struct inode *to_tell, __u32 mask, const void *data, int data_is,
- const struct qstr *file_name, u32 cookie)
+int fsnotify(__u32 mask, const void *data, int data_type, struct inode *dir,
+ const struct qstr *file_name, struct inode *inode, u32 cookie)
{
- const struct path *path = fsnotify_data_path(data, data_is);
+ const struct path *path = fsnotify_data_path(data, data_type);
struct fsnotify_iter_info iter_info = {};
- struct super_block *sb = to_tell->i_sb;
+ struct super_block *sb;
struct mount *mnt = NULL;
- __u32 mnt_or_sb_mask = sb->s_fsnotify_mask;
+ struct inode *child = NULL;
int ret = 0;
- __u32 test_mask = (mask & ALL_FSNOTIFY_EVENTS);
+ __u32 test_mask, marks_mask;
- if (path) {
+ if (path)
mnt = real_mount(path->mnt);
- mnt_or_sb_mask |= mnt->mnt_fsnotify_mask;
+
+ if (!inode) {
+ /* Dirent event - report on TYPE_INODE to dir */
+ inode = dir;
+ } else if (mask & FS_EVENT_ON_CHILD) {
+ /*
+ * Event on child - report on TYPE_INODE to dir if it is
+ * watching children and on TYPE_CHILD to child.
+ */
+ child = inode;
+ inode = dir;
}
- /* An event "on child" is not intended for a mount/sb mark */
- if (mask & FS_EVENT_ON_CHILD)
- mnt_or_sb_mask = 0;
+ sb = inode->i_sb;
/*
* Optimization: srcu_read_lock() has a memory barrier which can
@@ -334,28 +455,45 @@
* SRCU because we have no references to any objects and do not
* need SRCU to keep them "alive".
*/
- if (!to_tell->i_fsnotify_marks && !sb->s_fsnotify_marks &&
- (!mnt || !mnt->mnt_fsnotify_marks))
+ if (!sb->s_fsnotify_marks &&
+ (!mnt || !mnt->mnt_fsnotify_marks) &&
+ (!inode || !inode->i_fsnotify_marks) &&
+ (!child || !child->i_fsnotify_marks))
return 0;
+
+ marks_mask = sb->s_fsnotify_mask;
+ if (mnt)
+ marks_mask |= mnt->mnt_fsnotify_mask;
+ if (inode)
+ marks_mask |= inode->i_fsnotify_mask;
+ if (child)
+ marks_mask |= child->i_fsnotify_mask;
+
+
/*
* if this is a modify event we may need to clear the ignored masks
- * otherwise return if neither the inode nor the vfsmount/sb care about
- * this type of event.
+ * otherwise return if none of the marks care about this type of event.
*/
- if (!(mask & FS_MODIFY) &&
- !(test_mask & (to_tell->i_fsnotify_mask | mnt_or_sb_mask)))
+ test_mask = (mask & ALL_FSNOTIFY_EVENTS);
+ if (!(mask & FS_MODIFY) && !(test_mask & marks_mask))
return 0;
iter_info.srcu_idx = srcu_read_lock(&fsnotify_mark_srcu);
- iter_info.marks[FSNOTIFY_OBJ_TYPE_INODE] =
- fsnotify_first_mark(&to_tell->i_fsnotify_marks);
iter_info.marks[FSNOTIFY_OBJ_TYPE_SB] =
fsnotify_first_mark(&sb->s_fsnotify_marks);
if (mnt) {
iter_info.marks[FSNOTIFY_OBJ_TYPE_VFSMOUNT] =
fsnotify_first_mark(&mnt->mnt_fsnotify_marks);
}
+ if (inode) {
+ iter_info.marks[FSNOTIFY_OBJ_TYPE_INODE] =
+ fsnotify_first_mark(&inode->i_fsnotify_marks);
+ }
+ if (child) {
+ iter_info.marks[FSNOTIFY_OBJ_TYPE_CHILD] =
+ fsnotify_first_mark(&child->i_fsnotify_marks);
+ }
/*
* We need to merge inode/vfsmount/sb mark lists so that e.g. inode mark
@@ -363,8 +501,8 @@
* That's why this traversal is so complicated...
*/
while (fsnotify_iter_select_report_types(&iter_info)) {
- ret = send_to_group(to_tell, mask, data, data_is, cookie,
- file_name, &iter_info);
+ ret = send_to_group(mask, data, data_type, dir, file_name,
+ cookie, &iter_info);
if (ret && (mask & ALL_FSNOTIFY_PERM_EVENTS))
goto out;
@@ -383,7 +521,7 @@
{
int ret;
- BUILD_BUG_ON(HWEIGHT32(ALL_FSNOTIFY_BITS) != 26);
+ BUILD_BUG_ON(HWEIGHT32(ALL_FSNOTIFY_BITS) != 25);
ret = init_srcu_struct(&fsnotify_mark_srcu);
if (ret)
diff --git a/fs/notify/inotify/inotify.h b/fs/notify/inotify/inotify.h
index 3f246f7..4327d0e 100644
--- a/fs/notify/inotify/inotify.h
+++ b/fs/notify/inotify/inotify.h
@@ -24,9 +24,9 @@
extern void inotify_ignored_and_remove_idr(struct fsnotify_mark *fsn_mark,
struct fsnotify_group *group);
-extern int inotify_handle_event(struct fsnotify_group *group,
- struct inode *inode,
- u32 mask, const void *data, int data_type,
+extern int inotify_handle_event(struct fsnotify_group *group, u32 mask,
+ const void *data, int data_type,
+ struct inode *dir,
const struct qstr *file_name, u32 cookie,
struct fsnotify_iter_info *iter_info);
diff --git a/fs/notify/inotify/inotify_fsnotify.c b/fs/notify/inotify/inotify_fsnotify.c
index 2ebc890..a65cf8c 100644
--- a/fs/notify/inotify/inotify_fsnotify.c
+++ b/fs/notify/inotify/inotify_fsnotify.c
@@ -39,7 +39,7 @@
if (old->mask & FS_IN_IGNORED)
return false;
if ((old->mask == new->mask) &&
- (old_fsn->objectid == new_fsn->objectid) &&
+ (old->wd == new->wd) &&
(old->name_len == new->name_len) &&
(!old->name_len || !strcmp(old->name, new->name)))
return true;
@@ -55,14 +55,11 @@
return event_compare(last_event, event);
}
-int inotify_handle_event(struct fsnotify_group *group,
- struct inode *inode,
- u32 mask, const void *data, int data_type,
- const struct qstr *file_name, u32 cookie,
- struct fsnotify_iter_info *iter_info)
+static int inotify_one_event(struct fsnotify_group *group, u32 mask,
+ struct fsnotify_mark *inode_mark,
+ const struct path *path,
+ const struct qstr *file_name, u32 cookie)
{
- const struct path *path = fsnotify_data_path(data, data_type);
- struct fsnotify_mark *inode_mark = fsnotify_iter_inode_mark(iter_info);
struct inotify_inode_mark *i_mark;
struct inotify_event_info *event;
struct fsnotify_event *fsn_event;
@@ -70,9 +67,6 @@
int len = 0;
int alloc_len = sizeof(struct inotify_event_info);
- if (WARN_ON(fsnotify_iter_vfsmount_mark(iter_info)))
- return 0;
-
if ((inode_mark->mask & FS_EXCL_UNLINK) &&
path && d_unlinked(path->dentry))
return 0;
@@ -82,7 +76,7 @@
alloc_len += len + 1;
}
- pr_debug("%s: group=%p inode=%p mask=%x\n", __func__, group, inode,
+ pr_debug("%s: group=%p mark=%p mask=%x\n", __func__, group, inode_mark,
mask);
i_mark = container_of(inode_mark, struct inotify_inode_mark,
@@ -116,7 +110,7 @@
mask &= ~IN_ISDIR;
fsn_event = &event->fse;
- fsnotify_init_event(fsn_event, (unsigned long)inode);
+ fsnotify_init_event(fsn_event, 0);
event->mask = mask;
event->wd = i_mark->wd;
event->sync_cookie = cookie;
@@ -136,6 +130,37 @@
return 0;
}
+int inotify_handle_event(struct fsnotify_group *group, u32 mask,
+ const void *data, int data_type, struct inode *dir,
+ const struct qstr *file_name, u32 cookie,
+ struct fsnotify_iter_info *iter_info)
+{
+ const struct path *path = fsnotify_data_path(data, data_type);
+ struct fsnotify_mark *inode_mark = fsnotify_iter_inode_mark(iter_info);
+ struct fsnotify_mark *child_mark = fsnotify_iter_child_mark(iter_info);
+ int ret = 0;
+
+ if (WARN_ON(fsnotify_iter_vfsmount_mark(iter_info)))
+ return 0;
+
+ /*
+ * Some events cannot be sent on both parent and child marks
+ * (e.g. IN_CREATE). Those events are always sent on inode_mark.
+ * For events that are possible on both parent and child (e.g. IN_OPEN),
+ * event is sent on inode_mark with name if the parent is watching and
+ * is sent on child_mark without name if child is watching.
+ * If both parent and child are watching, report the event with child's
+ * name here and report another event without child's name below.
+ */
+ if (inode_mark)
+ ret = inotify_one_event(group, mask, inode_mark, path,
+ file_name, cookie);
+ if (ret || !child_mark)
+ return ret;
+
+ return inotify_one_event(group, mask, child_mark, path, NULL, 0);
+}
+
static void inotify_freeing_mark(struct fsnotify_mark *fsn_mark, struct fsnotify_group *group)
{
inotify_ignored_and_remove_idr(fsn_mark, group);
diff --git a/fs/notify/inotify/inotify_user.c b/fs/notify/inotify/inotify_user.c
index f88bbcc..186722b 100644
--- a/fs/notify/inotify/inotify_user.c
+++ b/fs/notify/inotify/inotify_user.c
@@ -75,15 +75,17 @@
};
#endif /* CONFIG_SYSCTL */
-static inline __u32 inotify_arg_to_mask(u32 arg)
+static inline __u32 inotify_arg_to_mask(struct inode *inode, u32 arg)
{
__u32 mask;
/*
- * everything should accept their own ignored, cares about children,
- * and should receive events when the inode is unmounted
+ * Everything should accept their own ignored and should receive events
+ * when the inode is unmounted. All directories care about children.
*/
- mask = (FS_IN_IGNORED | FS_EVENT_ON_CHILD | FS_UNMOUNT);
+ mask = (FS_IN_IGNORED | FS_UNMOUNT);
+ if (S_ISDIR(inode->i_mode))
+ mask |= FS_EVENT_ON_CHILD;
/* mask off the flags used to open the fd */
mask |= (arg & (IN_ALL_EVENTS | IN_ONESHOT | IN_EXCL_UNLINK));
@@ -490,8 +492,8 @@
fsn_mark);
/* Queue ignore event for the watch */
- inotify_handle_event(group, NULL, FS_IN_IGNORED, NULL,
- FSNOTIFY_EVENT_NONE, NULL, 0, &iter_info);
+ inotify_handle_event(group, FS_IN_IGNORED, NULL, FSNOTIFY_EVENT_NONE,
+ NULL, NULL, 0, &iter_info);
i_mark = container_of(fsn_mark, struct inotify_inode_mark, fsn_mark);
/* remove this mark from the idr */
@@ -512,7 +514,7 @@
int create = (arg & IN_MASK_CREATE);
int ret;
- mask = inotify_arg_to_mask(arg);
+ mask = inotify_arg_to_mask(inode, arg);
fsn_mark = fsnotify_find_mark(&inode->i_fsnotify_marks, group);
if (!fsn_mark)
@@ -565,7 +567,7 @@
struct idr *idr = &group->inotify_data.idr;
spinlock_t *idr_lock = &group->inotify_data.idr_lock;
- mask = inotify_arg_to_mask(arg);
+ mask = inotify_arg_to_mask(inode, arg);
tmp_i_mark = kmem_cache_alloc(inotify_inode_mark_cachep, GFP_KERNEL);
if (unlikely(!tmp_i_mark))
diff --git a/fs/open.c b/fs/open.c
index 5e62f18..c80e9f4 100644
--- a/fs/open.c
+++ b/fs/open.c
@@ -394,7 +394,7 @@
return old_cred;
}
-long do_faccessat(int dfd, const char __user *filename, int mode, int flags)
+static long do_faccessat(int dfd, const char __user *filename, int mode, int flags)
{
struct path path;
struct inode *inode;
@@ -482,7 +482,7 @@
return do_faccessat(AT_FDCWD, filename, mode, 0);
}
-int ksys_chdir(const char __user *filename)
+SYSCALL_DEFINE1(chdir, const char __user *, filename)
{
struct path path;
int error;
@@ -508,11 +508,6 @@
return error;
}
-SYSCALL_DEFINE1(chdir, const char __user *, filename)
-{
- return ksys_chdir(filename);
-}
-
SYSCALL_DEFINE1(fchdir, unsigned int, fd)
{
struct fd f = fdget_raw(fd);
@@ -535,7 +530,7 @@
return error;
}
-int ksys_chroot(const char __user *filename)
+SYSCALL_DEFINE1(chroot, const char __user *, filename)
{
struct path path;
int error;
@@ -568,12 +563,7 @@
return error;
}
-SYSCALL_DEFINE1(chroot, const char __user *, filename)
-{
- return ksys_chroot(filename);
-}
-
-static int chmod_common(const struct path *path, umode_t mode)
+int chmod_common(const struct path *path, umode_t mode)
{
struct inode *inode = path->dentry->d_inode;
struct inode *delegated_inode = NULL;
@@ -602,25 +592,25 @@
return error;
}
-int ksys_fchmod(unsigned int fd, umode_t mode)
+int vfs_fchmod(struct file *file, umode_t mode)
+{
+ audit_file(file);
+ return chmod_common(&file->f_path, mode);
+}
+
+SYSCALL_DEFINE2(fchmod, unsigned int, fd, umode_t, mode)
{
struct fd f = fdget(fd);
int err = -EBADF;
if (f.file) {
- audit_file(f.file);
- err = chmod_common(&f.file->f_path, mode);
+ err = vfs_fchmod(f.file, mode);
fdput(f);
}
return err;
}
-SYSCALL_DEFINE2(fchmod, unsigned int, fd, umode_t, mode)
-{
- return ksys_fchmod(fd, mode);
-}
-
-int do_fchmodat(int dfd, const char __user *filename, umode_t mode)
+static int do_fchmodat(int dfd, const char __user *filename, umode_t mode)
{
struct path path;
int error;
@@ -649,7 +639,7 @@
return do_fchmodat(AT_FDCWD, filename, mode);
}
-static int chown_common(const struct path *path, uid_t user, gid_t group)
+int chown_common(const struct path *path, uid_t user, gid_t group)
{
struct inode *inode = path->dentry->d_inode;
struct inode *delegated_inode = NULL;
@@ -740,23 +730,28 @@
AT_SYMLINK_NOFOLLOW);
}
+int vfs_fchown(struct file *file, uid_t user, gid_t group)
+{
+ int error;
+
+ error = mnt_want_write_file(file);
+ if (error)
+ return error;
+ audit_file(file);
+ error = chown_common(&file->f_path, user, group);
+ mnt_drop_write_file(file);
+ return error;
+}
+
int ksys_fchown(unsigned int fd, uid_t user, gid_t group)
{
struct fd f = fdget(fd);
int error = -EBADF;
- if (!f.file)
- goto out;
-
- error = mnt_want_write_file(f.file);
- if (error)
- goto out_fput;
- audit_file(f.file);
- error = chown_common(&f.file->f_path, user, group);
- mnt_drop_write_file(f.file);
-out_fput:
- fdput(f);
-out:
+ if (f.file) {
+ error = vfs_fchown(f.file, user, group);
+ fdput(f);
+ }
return error;
}
@@ -1198,7 +1193,9 @@
SYSCALL_DEFINE3(open, const char __user *, filename, int, flags, umode_t, mode)
{
- return ksys_open(filename, flags, mode);
+ if (force_o_largefile())
+ flags |= O_LARGEFILE;
+ return do_sys_open(AT_FDCWD, filename, flags, mode);
}
SYSCALL_DEFINE4(openat, int, dfd, const char __user *, filename, int, flags,
@@ -1260,9 +1257,12 @@
*/
SYSCALL_DEFINE2(creat, const char __user *, pathname, umode_t, mode)
{
- return ksys_open(pathname, O_CREAT | O_WRONLY | O_TRUNC, mode);
-}
+ int flags = O_CREAT | O_WRONLY | O_TRUNC;
+ if (force_o_largefile())
+ flags |= O_LARGEFILE;
+ return do_sys_open(AT_FDCWD, pathname, flags, mode);
+}
#endif
/*
diff --git a/fs/quota/Kconfig b/fs/quota/Kconfig
index 7218314..d1ceb76 100644
--- a/fs/quota/Kconfig
+++ b/fs/quota/Kconfig
@@ -15,7 +15,7 @@
Ext3, ext4 and reiserfs also support journaled quotas for which
you don't need to run quotacheck(8) after an unclean shutdown.
For further details, read the Quota mini-HOWTO, available from
- <http://www.tldp.org/docs.html#howto>, or the documentation provided
+ <https://www.tldp.org/docs.html#howto>, or the documentation provided
with the quota tools. Probably the quota support is only useful for
multi user systems. If unsure, say N.
diff --git a/fs/read_write.c b/fs/read_write.c
index 4fb7978..5db58b8 100644
--- a/fs/read_write.c
+++ b/fs/read_write.c
@@ -301,7 +301,7 @@
}
EXPORT_SYMBOL(vfs_llseek);
-off_t ksys_lseek(unsigned int fd, off_t offset, unsigned int whence)
+static off_t ksys_lseek(unsigned int fd, off_t offset, unsigned int whence)
{
off_t retval;
struct fd f = fdget_pos(fd);
diff --git a/fs/readdir.c b/fs/readdir.c
index a49f07c..19434b3 100644
--- a/fs/readdir.c
+++ b/fs/readdir.c
@@ -348,8 +348,8 @@
return -EFAULT;
}
-int ksys_getdents64(unsigned int fd, struct linux_dirent64 __user *dirent,
- unsigned int count)
+SYSCALL_DEFINE3(getdents64, unsigned int, fd,
+ struct linux_dirent64 __user *, dirent, unsigned int, count)
{
struct fd f;
struct getdents_callback64 buf = {
@@ -380,13 +380,6 @@
return error;
}
-
-SYSCALL_DEFINE3(getdents64, unsigned int, fd,
- struct linux_dirent64 __user *, dirent, unsigned int, count)
-{
- return ksys_getdents64(fd, dirent, count);
-}
-
#ifdef CONFIG_COMPAT
struct compat_old_linux_dirent {
compat_ulong_t d_ino;
diff --git a/fs/reiserfs/dir.c b/fs/reiserfs/dir.c
index 5b50689..79ee2b4 100644
--- a/fs/reiserfs/dir.c
+++ b/fs/reiserfs/dir.c
@@ -289,7 +289,7 @@
/* direntry header of "." */
put_deh_offset(dot, DOT_OFFSET);
- /* these two are from make_le_item_head, and are are LE */
+ /* these two are from make_le_item_head, and are LE */
dot->deh_dir_id = dirid;
dot->deh_objectid = objid;
dot->deh_state = 0; /* Endian safe if 0 */
@@ -299,7 +299,7 @@
/* direntry header of ".." */
put_deh_offset(dotdot, DOT_DOT_OFFSET);
/* key of ".." for the root directory */
- /* these two are from the inode, and are are LE */
+ /* these two are from the inode, and are LE */
dotdot->deh_dir_id = par_dirid;
dotdot->deh_objectid = par_objid;
dotdot->deh_state = 0; /* Endian safe if 0 */
@@ -323,7 +323,7 @@
/* direntry header of "." */
put_deh_offset(dot, DOT_OFFSET);
- /* these two are from make_le_item_head, and are are LE */
+ /* these two are from make_le_item_head, and are LE */
dot->deh_dir_id = dirid;
dot->deh_objectid = objid;
dot->deh_state = 0; /* Endian safe if 0 */
@@ -333,7 +333,7 @@
/* direntry header of ".." */
put_deh_offset(dotdot, DOT_DOT_OFFSET);
/* key of ".." for the root directory */
- /* these two are from the inode, and are are LE */
+ /* these two are from the inode, and are LE */
dotdot->deh_dir_id = par_dirid;
dotdot->deh_objectid = par_objid;
dotdot->deh_state = 0; /* Endian safe if 0 */
diff --git a/fs/reiserfs/fix_node.c b/fs/reiserfs/fix_node.c
index 1170922..fefe87e 100644
--- a/fs/reiserfs/fix_node.c
+++ b/fs/reiserfs/fix_node.c
@@ -611,9 +611,9 @@
* blk_num number of blocks that S[h] will be splitted into;
* s012 number of items that fall into splitted nodes.
* lbytes number of bytes which flow to the left neighbor from the
- * item that is not not shifted entirely
+ * item that is not shifted entirely
* rbytes number of bytes which flow to the right neighbor from the
- * item that is not not shifted entirely
+ * item that is not shifted entirely
* s1bytes number of bytes which flow to the first new node when
* S[0] splits (this number is contained in s012 array)
*/
diff --git a/fs/reiserfs/journal.c b/fs/reiserfs/journal.c
index 5c76633..e98f993 100644
--- a/fs/reiserfs/journal.c
+++ b/fs/reiserfs/journal.c
@@ -32,7 +32,7 @@
* to disk for all backgrounded commits that have been
* around too long.
* -- Note, if you call this as an immediate flush from
- * from within kupdate, it will ignore the immediate flag
+ * within kupdate, it will ignore the immediate flag
*/
#include <linux/time.h>
diff --git a/fs/reiserfs/reiserfs.h b/fs/reiserfs/reiserfs.h
index 7265801..f698715 100644
--- a/fs/reiserfs/reiserfs.h
+++ b/fs/reiserfs/reiserfs.h
@@ -1109,7 +1109,7 @@
* ReiserFS leaves the first 64k unused, so that partition labels have
* enough space. If someone wants to write a fancy bootloader that
* needs more than 64k, let us know, and this will be increased in size.
- * This number must be larger than than the largest block size on any
+ * This number must be larger than the largest block size on any
* platform, or code will break. -Hans
*/
#define REISERFS_DISK_OFFSET_IN_BYTES (64 * 1024)
diff --git a/fs/reiserfs/xattr_acl.c b/fs/reiserfs/xattr_acl.c
index 05f6667..ccd40df 100644
--- a/fs/reiserfs/xattr_acl.c
+++ b/fs/reiserfs/xattr_acl.c
@@ -373,7 +373,7 @@
/* Other xattrs can be created during inode creation. We don't
* want to claim too many blocks, so we check to see if we
- * we need to create the tree to the xattrs, and then we
+ * need to create the tree to the xattrs, and then we
* just want two files. */
nblocks = reiserfs_xattr_jcreate_nblocks(inode);
nblocks += JOURNAL_BLOCKS_PER_OBJECT(inode->i_sb);
diff --git a/fs/udf/ecma_167.h b/fs/udf/ecma_167.h
index 736ebc5dc..185c3e2 100644
--- a/fs/udf/ecma_167.h
+++ b/fs/udf/ecma_167.h
@@ -2,7 +2,7 @@
* ecma_167.h
*
* This file is based on ECMA-167 3rd edition (June 1997)
- * http://www.ecma.ch
+ * https://www.ecma.ch
*
* Copyright (c) 2001-2002 Ben Fennema
* Copyright (c) 2017-2019 Pali Rohár <pali@kernel.org>
diff --git a/fs/udf/osta_udf.h b/fs/udf/osta_udf.h
index d5fbfab..22bc4fb 100644
--- a/fs/udf/osta_udf.h
+++ b/fs/udf/osta_udf.h
@@ -226,7 +226,7 @@
#define ICBTAG_FILE_TYPE_MIRROR 0xFB
#define ICBTAG_FILE_TYPE_BITMAP 0xFC
-/* struct struct long_ad ICB - ADImpUse (UDF 2.60 2.2.4.3) */
+/* struct long_ad ICB - ADImpUse (UDF 2.60 2.2.4.3) */
struct allocDescImpUse {
__le16 flags;
uint8_t impUse[4];
diff --git a/fs/udf/super.c b/fs/udf/super.c
index f747bf7..1c42f54 100644
--- a/fs/udf/super.c
+++ b/fs/udf/super.c
@@ -11,8 +11,8 @@
* This code is based on version 2.00 of the UDF specification,
* and revision 3 of the ECMA 167 standard [equivalent to ISO 13346].
* http://www.osta.org/
- * http://www.ecma.ch/
- * http://www.iso.org/
+ * https://www.ecma.ch/
+ * https://www.iso.org/
*
* COPYRIGHT
* This file is distributed under the terms of the GNU General Public
diff --git a/fs/utimes.c b/fs/utimes.c
index b7b9275..fd3cc42 100644
--- a/fs/utimes.c
+++ b/fs/utimes.c
@@ -16,21 +16,26 @@
return nsec >= 0 && nsec <= 999999999;
}
-static int utimes_common(const struct path *path, struct timespec64 *times)
+int vfs_utimes(const struct path *path, struct timespec64 *times)
{
int error;
struct iattr newattrs;
struct inode *inode = path->dentry->d_inode;
struct inode *delegated_inode = NULL;
+ if (times) {
+ if (!nsec_valid(times[0].tv_nsec) ||
+ !nsec_valid(times[1].tv_nsec))
+ return -EINVAL;
+ if (times[0].tv_nsec == UTIME_NOW &&
+ times[1].tv_nsec == UTIME_NOW)
+ times = NULL;
+ }
+
error = mnt_want_write(path->mnt);
if (error)
goto out;
- if (times && times[0].tv_nsec == UTIME_NOW &&
- times[1].tv_nsec == UTIME_NOW)
- times = NULL;
-
newattrs.ia_valid = ATTR_CTIME | ATTR_MTIME | ATTR_ATIME;
if (times) {
if (times[0].tv_nsec == UTIME_OMIT)
@@ -70,6 +75,51 @@
return error;
}
+static int do_utimes_path(int dfd, const char __user *filename,
+ struct timespec64 *times, int flags)
+{
+ struct path path;
+ int lookup_flags = 0, error;
+
+ if (flags & ~(AT_SYMLINK_NOFOLLOW | AT_EMPTY_PATH))
+ return -EINVAL;
+
+ if (!(flags & AT_SYMLINK_NOFOLLOW))
+ lookup_flags |= LOOKUP_FOLLOW;
+ if (flags & AT_EMPTY_PATH)
+ lookup_flags |= LOOKUP_EMPTY;
+
+retry:
+ error = user_path_at(dfd, filename, lookup_flags, &path);
+ if (error)
+ return error;
+
+ error = vfs_utimes(&path, times);
+ path_put(&path);
+ if (retry_estale(error, lookup_flags)) {
+ lookup_flags |= LOOKUP_REVAL;
+ goto retry;
+ }
+
+ return error;
+}
+
+static int do_utimes_fd(int fd, struct timespec64 *times, int flags)
+{
+ struct fd f;
+ int error;
+
+ if (flags)
+ return -EINVAL;
+
+ f = fdget(fd);
+ if (!f.file)
+ return -EBADF;
+ error = vfs_utimes(&f.file->f_path, times);
+ fdput(f);
+ return error;
+}
+
/*
* do_utimes - change times on filename or file descriptor
* @dfd: open file descriptor, -1 or AT_FDCWD
@@ -88,52 +138,9 @@
long do_utimes(int dfd, const char __user *filename, struct timespec64 *times,
int flags)
{
- int error = -EINVAL;
-
- if (times && (!nsec_valid(times[0].tv_nsec) ||
- !nsec_valid(times[1].tv_nsec))) {
- goto out;
- }
-
- if (flags & ~(AT_SYMLINK_NOFOLLOW | AT_EMPTY_PATH))
- goto out;
-
- if (filename == NULL && dfd != AT_FDCWD) {
- struct fd f;
-
- if (flags)
- goto out;
-
- f = fdget(dfd);
- error = -EBADF;
- if (!f.file)
- goto out;
-
- error = utimes_common(&f.file->f_path, times);
- fdput(f);
- } else {
- struct path path;
- int lookup_flags = 0;
-
- if (!(flags & AT_SYMLINK_NOFOLLOW))
- lookup_flags |= LOOKUP_FOLLOW;
- if (flags & AT_EMPTY_PATH)
- lookup_flags |= LOOKUP_EMPTY;
-retry:
- error = user_path_at(dfd, filename, lookup_flags, &path);
- if (error)
- goto out;
-
- error = utimes_common(&path, times);
- path_put(&path);
- if (retry_estale(error, lookup_flags)) {
- lookup_flags |= LOOKUP_REVAL;
- goto retry;
- }
- }
-
-out:
- return error;
+ if (filename == NULL && dfd != AT_FDCWD)
+ return do_utimes_fd(dfd, times, flags);
+ return do_utimes_path(dfd, filename, times, flags);
}
SYSCALL_DEFINE4(utimensat, int, dfd, const char __user *, filename,
diff --git a/fs/xfs/kmem.c b/fs/xfs/kmem.c
index f136647..e841ed7 100644
--- a/fs/xfs/kmem.c
+++ b/fs/xfs/kmem.c
@@ -115,24 +115,3 @@
congestion_wait(BLK_RW_ASYNC, HZ/50);
} while (1);
}
-
-void *
-kmem_zone_alloc(kmem_zone_t *zone, xfs_km_flags_t flags)
-{
- int retries = 0;
- gfp_t lflags = kmem_flags_convert(flags);
- void *ptr;
-
- trace_kmem_zone_alloc(kmem_cache_size(zone), flags, _RET_IP_);
- do {
- ptr = kmem_cache_alloc(zone, lflags);
- if (ptr || (flags & KM_MAYFAIL))
- return ptr;
- if (!(++retries % 100))
- xfs_err(NULL,
- "%s(%u) possible memory allocation deadlock in %s (mode:0x%x)",
- current->comm, current->pid,
- __func__, lflags);
- congestion_wait(BLK_RW_ASYNC, HZ/50);
- } while (1);
-}
diff --git a/fs/xfs/kmem.h b/fs/xfs/kmem.h
index 34cbcfd..8e85558 100644
--- a/fs/xfs/kmem.h
+++ b/fs/xfs/kmem.h
@@ -85,14 +85,6 @@
#define kmem_zone kmem_cache
#define kmem_zone_t struct kmem_cache
-extern void *kmem_zone_alloc(kmem_zone_t *, xfs_km_flags_t);
-
-static inline void *
-kmem_zone_zalloc(kmem_zone_t *zone, xfs_km_flags_t flags)
-{
- return kmem_zone_alloc(zone, flags | KM_ZERO);
-}
-
static inline struct page *
kmem_to_page(void *addr)
{
diff --git a/fs/xfs/libxfs/xfs_ag.c b/fs/xfs/libxfs/xfs_ag.c
index 9d84007..8cf73fe 100644
--- a/fs/xfs/libxfs/xfs_ag.c
+++ b/fs/xfs/libxfs/xfs_ag.c
@@ -563,7 +563,8 @@
error = xfs_alloc_read_agf(mp, NULL, agno, 0, &agf_bp);
if (error)
goto out_agi;
- pag = xfs_perag_get(mp, agno);
+
+ pag = agi_bp->b_pag;
/* Fill out form. */
memset(ageo, 0, sizeof(*ageo));
@@ -583,7 +584,6 @@
xfs_ag_geom_health(pag, ageo);
/* Release resources. */
- xfs_perag_put(pag);
xfs_buf_relse(agf_bp);
out_agi:
xfs_buf_relse(agi_bp);
diff --git a/fs/xfs/libxfs/xfs_ag_resv.h b/fs/xfs/libxfs/xfs_ag_resv.h
index f3fd0ee..8a8eb4b 100644
--- a/fs/xfs/libxfs/xfs_ag_resv.h
+++ b/fs/xfs/libxfs/xfs_ag_resv.h
@@ -37,16 +37,4 @@
xfs_perag_put(pag);
}
-static inline void
-xfs_ag_resv_rmapbt_free(
- struct xfs_mount *mp,
- xfs_agnumber_t agno)
-{
- struct xfs_perag *pag;
-
- pag = xfs_perag_get(mp, agno);
- xfs_ag_resv_free_extent(pag, XFS_AG_RESV_RMAPBT, NULL, 1);
- xfs_perag_put(pag);
-}
-
#endif /* __XFS_AG_RESV_H__ */
diff --git a/fs/xfs/libxfs/xfs_alloc.c b/fs/xfs/libxfs/xfs_alloc.c
index 203e74f..852b536 100644
--- a/fs/xfs/libxfs/xfs_alloc.c
+++ b/fs/xfs/libxfs/xfs_alloc.c
@@ -710,13 +710,12 @@
STATIC int
xfs_alloc_update_counters(
struct xfs_trans *tp,
- struct xfs_perag *pag,
struct xfs_buf *agbp,
long len)
{
struct xfs_agf *agf = agbp->b_addr;
- pag->pagf_freeblks += len;
+ agbp->b_pag->pagf_freeblks += len;
be32_add_cpu(&agf->agf_freeblks, len);
xfs_trans_agblocks_delta(tp, len);
@@ -1175,8 +1174,7 @@
}
if (!args->wasfromfl) {
- error = xfs_alloc_update_counters(args->tp, args->pag,
- args->agbp,
+ error = xfs_alloc_update_counters(args->tp, args->agbp,
-((long)(args->len)));
if (error)
return error;
@@ -1887,7 +1885,6 @@
enum xfs_ag_resv_type type)
{
struct xfs_mount *mp;
- struct xfs_perag *pag;
struct xfs_btree_cur *bno_cur;
struct xfs_btree_cur *cnt_cur;
xfs_agblock_t gtbno; /* start of right neighbor */
@@ -2167,10 +2164,8 @@
/*
* Update the freespace totals in the ag and superblock.
*/
- pag = xfs_perag_get(mp, agno);
- error = xfs_alloc_update_counters(tp, pag, agbp, len);
- xfs_ag_resv_free_extent(pag, type, tp, len);
- xfs_perag_put(pag);
+ error = xfs_alloc_update_counters(tp, agbp, len);
+ xfs_ag_resv_free_extent(agbp->b_pag, type, tp, len);
if (error)
goto error0;
@@ -2467,7 +2462,8 @@
ASSERT(xfs_bmap_free_item_zone != NULL);
ASSERT(oinfo != NULL);
- new = kmem_zone_alloc(xfs_bmap_free_item_zone, 0);
+ new = kmem_cache_alloc(xfs_bmap_free_item_zone,
+ GFP_KERNEL | __GFP_NOFAIL);
new->xefi_startblock = XFS_AGB_TO_FSB(mp, agno, agbno);
new->xefi_blockcount = 1;
new->xefi_oinfo = *oinfo;
@@ -2689,7 +2685,7 @@
if (be32_to_cpu(agf->agf_flfirst) == xfs_agfl_size(mp))
agf->agf_flfirst = 0;
- pag = xfs_perag_get(mp, be32_to_cpu(agf->agf_seqno));
+ pag = agbp->b_pag;
ASSERT(!pag->pagf_agflreset);
be32_add_cpu(&agf->agf_flcount, -1);
xfs_trans_agflist_delta(tp, -1);
@@ -2701,7 +2697,6 @@
pag->pagf_btreeblks++;
logflags |= XFS_AGF_BTREEBLKS;
}
- xfs_perag_put(pag);
xfs_alloc_log_agf(tp, agbp, logflags);
*bnop = bno;
@@ -2797,7 +2792,7 @@
if (be32_to_cpu(agf->agf_fllast) == xfs_agfl_size(mp))
agf->agf_fllast = 0;
- pag = xfs_perag_get(mp, be32_to_cpu(agf->agf_seqno));
+ pag = agbp->b_pag;
ASSERT(!pag->pagf_agflreset);
be32_add_cpu(&agf->agf_flcount, 1);
xfs_trans_agflist_delta(tp, 1);
@@ -2809,7 +2804,6 @@
pag->pagf_btreeblks--;
logflags |= XFS_AGF_BTREEBLKS;
}
- xfs_perag_put(pag);
xfs_alloc_log_agf(tp, agbp, logflags);
@@ -3006,7 +3000,7 @@
ASSERT(!(*bpp)->b_error);
agf = (*bpp)->b_addr;
- pag = xfs_perag_get(mp, agno);
+ pag = (*bpp)->b_pag;
if (!pag->pagf_init) {
pag->pagf_freeblks = be32_to_cpu(agf->agf_freeblks);
pag->pagf_btreeblks = be32_to_cpu(agf->agf_btreeblks);
@@ -3034,7 +3028,6 @@
be32_to_cpu(agf->agf_levels[XFS_BTNUM_CNTi]));
}
#endif
- xfs_perag_put(pag);
return 0;
}
diff --git a/fs/xfs/libxfs/xfs_alloc_btree.c b/fs/xfs/libxfs/xfs_alloc_btree.c
index 60c453c..8e01231 100644
--- a/fs/xfs/libxfs/xfs_alloc_btree.c
+++ b/fs/xfs/libxfs/xfs_alloc_btree.c
@@ -38,16 +38,14 @@
{
struct xfs_buf *agbp = cur->bc_ag.agbp;
struct xfs_agf *agf = agbp->b_addr;
- xfs_agnumber_t seqno = be32_to_cpu(agf->agf_seqno);
int btnum = cur->bc_btnum;
- struct xfs_perag *pag = xfs_perag_get(cur->bc_mp, seqno);
+ struct xfs_perag *pag = agbp->b_pag;
ASSERT(ptr->s != 0);
agf->agf_roots[btnum] = ptr->s;
be32_add_cpu(&agf->agf_levels[btnum], inc);
pag->pagf_levels[btnum] += inc;
- xfs_perag_put(pag);
xfs_alloc_log_agf(cur->bc_tp, agbp, XFS_AGF_ROOTS | XFS_AGF_LEVELS);
}
@@ -115,7 +113,6 @@
int reason)
{
struct xfs_agf *agf = cur->bc_ag.agbp->b_addr;
- xfs_agnumber_t seqno = be32_to_cpu(agf->agf_seqno);
struct xfs_perag *pag;
__be32 len;
int numrecs;
@@ -160,9 +157,8 @@
}
agf->agf_longest = len;
- pag = xfs_perag_get(cur->bc_mp, seqno);
+ pag = cur->bc_ag.agbp->b_pag;
pag->pagf_longest = be32_to_cpu(len);
- xfs_perag_put(pag);
xfs_alloc_log_agf(cur->bc_tp, cur->bc_ag.agbp, XFS_AGF_LONGEST);
}
@@ -484,7 +480,7 @@
ASSERT(btnum == XFS_BTNUM_BNO || btnum == XFS_BTNUM_CNT);
- cur = kmem_zone_zalloc(xfs_btree_cur_zone, KM_NOFS);
+ cur = kmem_cache_zalloc(xfs_btree_cur_zone, GFP_NOFS | __GFP_NOFAIL);
cur->bc_tp = tp;
cur->bc_mp = mp;
diff --git a/fs/xfs/libxfs/xfs_attr.c b/fs/xfs/libxfs/xfs_attr.c
index 3b1bd6e..2e055c0 100644
--- a/fs/xfs/libxfs/xfs_attr.c
+++ b/fs/xfs/libxfs/xfs_attr.c
@@ -46,6 +46,7 @@
STATIC int xfs_attr_leaf_get(xfs_da_args_t *args);
STATIC int xfs_attr_leaf_addname(xfs_da_args_t *args);
STATIC int xfs_attr_leaf_removename(xfs_da_args_t *args);
+STATIC int xfs_attr_leaf_hasname(struct xfs_da_args *args, struct xfs_buf **bp);
/*
* Internal routines when attribute list is more than one block.
@@ -53,6 +54,8 @@
STATIC int xfs_attr_node_get(xfs_da_args_t *args);
STATIC int xfs_attr_node_addname(xfs_da_args_t *args);
STATIC int xfs_attr_node_removename(xfs_da_args_t *args);
+STATIC int xfs_attr_node_hasname(xfs_da_args_t *args,
+ struct xfs_da_state **state);
STATIC int xfs_attr_fillstate(xfs_da_state_t *state);
STATIC int xfs_attr_refillstate(xfs_da_state_t *state);
@@ -175,8 +178,13 @@
struct xfs_da_args *args)
{
- struct xfs_mount *mp = dp->i_mount;
- int error, error2;
+ int error;
+
+ /*
+ * Build initial attribute list (if required).
+ */
+ if (dp->i_afp->if_format == XFS_DINODE_FMT_EXTENTS)
+ xfs_attr_shortform_create(args);
error = xfs_attr_shortform_addname(args);
if (error == -ENOSPC)
@@ -189,12 +197,70 @@
if (!error && !(args->op_flags & XFS_DA_OP_NOTIME))
xfs_trans_ichgtime(args->trans, dp, XFS_ICHGTIME_CHG);
- if (mp->m_flags & XFS_MOUNT_WSYNC)
+ if (dp->i_mount->m_flags & XFS_MOUNT_WSYNC)
xfs_trans_set_sync(args->trans);
- error2 = xfs_trans_commit(args->trans);
- args->trans = NULL;
- return error ? error : error2;
+ return error;
+}
+
+/*
+ * Check to see if the attr should be upgraded from non-existent or shortform to
+ * single-leaf-block attribute list.
+ */
+static inline bool
+xfs_attr_is_shortform(
+ struct xfs_inode *ip)
+{
+ return ip->i_afp->if_format == XFS_DINODE_FMT_LOCAL ||
+ (ip->i_afp->if_format == XFS_DINODE_FMT_EXTENTS &&
+ ip->i_afp->if_nextents == 0);
+}
+
+/*
+ * Attempts to set an attr in shortform, or converts short form to leaf form if
+ * there is not enough room. If the attr is set, the transaction is committed
+ * and set to NULL.
+ */
+STATIC int
+xfs_attr_set_shortform(
+ struct xfs_da_args *args,
+ struct xfs_buf **leaf_bp)
+{
+ struct xfs_inode *dp = args->dp;
+ int error, error2 = 0;
+
+ /*
+ * Try to add the attr to the attribute list in the inode.
+ */
+ error = xfs_attr_try_sf_addname(dp, args);
+ if (error != -ENOSPC) {
+ error2 = xfs_trans_commit(args->trans);
+ args->trans = NULL;
+ return error ? error : error2;
+ }
+ /*
+ * It won't fit in the shortform, transform to a leaf block. GROT:
+ * another possible req'mt for a double-split btree op.
+ */
+ error = xfs_attr_shortform_to_leaf(args, leaf_bp);
+ if (error)
+ return error;
+
+ /*
+ * Prevent the leaf buffer from being unlocked so that a concurrent AIL
+ * push cannot grab the half-baked leaf buffer and run into problems
+ * with the write verifier. Once we're done rolling the transaction we
+ * can release the hold and add the attr to the leaf.
+ */
+ xfs_trans_bhold(args->trans, *leaf_bp);
+ error = xfs_defer_finish(&args->trans);
+ xfs_trans_bhold_release(args->trans, *leaf_bp);
+ if (error) {
+ xfs_trans_brelse(args->trans, *leaf_bp);
+ return error;
+ }
+
+ return 0;
}
/*
@@ -206,61 +272,94 @@
{
struct xfs_inode *dp = args->dp;
struct xfs_buf *leaf_bp = NULL;
- int error;
+ int error = 0;
/*
- * If the attribute list is non-existent or a shortform list,
- * upgrade it to a single-leaf-block attribute list.
+ * If the attribute list is already in leaf format, jump straight to
+ * leaf handling. Otherwise, try to add the attribute to the shortform
+ * list; if there's no room then convert the list to leaf format and try
+ * again.
*/
- if (dp->i_afp->if_format == XFS_DINODE_FMT_LOCAL ||
- (dp->i_afp->if_format == XFS_DINODE_FMT_EXTENTS &&
- dp->i_afp->if_nextents == 0)) {
+ if (xfs_attr_is_shortform(dp)) {
/*
- * Build initial attribute list (if required).
+ * If the attr was successfully set in shortform, the
+ * transaction is committed and set to NULL. Otherwise, is it
+ * converted from shortform to leaf, and the transaction is
+ * retained.
*/
- if (dp->i_afp->if_format == XFS_DINODE_FMT_EXTENTS)
- xfs_attr_shortform_create(args);
+ error = xfs_attr_set_shortform(args, &leaf_bp);
+ if (error || !args->trans)
+ return error;
+ }
- /*
- * Try to add the attr to the attribute list in the inode.
- */
- error = xfs_attr_try_sf_addname(dp, args);
+ if (xfs_bmap_one_block(dp, XFS_ATTR_FORK)) {
+ error = xfs_attr_leaf_addname(args);
if (error != -ENOSPC)
return error;
/*
- * It won't fit in the shortform, transform to a leaf block.
- * GROT: another possible req'mt for a double-split btree op.
+ * Promote the attribute list to the Btree format.
*/
- error = xfs_attr_shortform_to_leaf(args, &leaf_bp);
+ error = xfs_attr3_leaf_to_node(args);
if (error)
return error;
/*
- * Prevent the leaf buffer from being unlocked so that a
- * concurrent AIL push cannot grab the half-baked leaf
- * buffer and run into problems with the write verifier.
- * Once we're done rolling the transaction we can release
- * the hold and add the attr to the leaf.
+ * Finish any deferred work items and roll the transaction once
+ * more. The goal here is to call node_addname with the inode
+ * and transaction in the same state (inode locked and joined,
+ * transaction clean) no matter how we got to this step.
*/
- xfs_trans_bhold(args->trans, leaf_bp);
error = xfs_defer_finish(&args->trans);
- xfs_trans_bhold_release(args->trans, leaf_bp);
- if (error) {
- xfs_trans_brelse(args->trans, leaf_bp);
+ if (error)
return error;
- }
+
+ /*
+ * Commit the current trans (including the inode) and
+ * start a new one.
+ */
+ error = xfs_trans_roll_inode(&args->trans, dp);
+ if (error)
+ return error;
}
- if (xfs_bmap_one_block(dp, XFS_ATTR_FORK))
- error = xfs_attr_leaf_addname(args);
- else
- error = xfs_attr_node_addname(args);
+ error = xfs_attr_node_addname(args);
return error;
}
/*
+ * Return EEXIST if attr is found, or ENOATTR if not
+ */
+int
+xfs_has_attr(
+ struct xfs_da_args *args)
+{
+ struct xfs_inode *dp = args->dp;
+ struct xfs_buf *bp = NULL;
+ int error;
+
+ if (!xfs_inode_hasattr(dp))
+ return -ENOATTR;
+
+ if (dp->i_afp->if_format == XFS_DINODE_FMT_LOCAL) {
+ ASSERT(dp->i_afp->if_flags & XFS_IFINLINE);
+ return xfs_attr_sf_findname(args, NULL, NULL);
+ }
+
+ if (xfs_bmap_one_block(dp, XFS_ATTR_FORK)) {
+ error = xfs_attr_leaf_hasname(args, &bp);
+
+ if (bp)
+ xfs_trans_brelse(args->trans, bp);
+
+ return error;
+ }
+
+ return xfs_attr_node_hasname(args, NULL);
+}
+
+/*
* Remove the attribute specified in @args.
*/
int
@@ -370,6 +469,15 @@
args->total, 0, quota_flags);
if (error)
goto out_trans_cancel;
+
+ error = xfs_has_attr(args);
+ if (error == -EEXIST && (args->attr_flags & XATTR_CREATE))
+ goto out_trans_cancel;
+ if (error == -ENOATTR && (args->attr_flags & XATTR_REPLACE))
+ goto out_trans_cancel;
+ if (error != -ENOATTR && error != -EEXIST)
+ goto out_trans_cancel;
+
error = xfs_attr_set_args(args);
if (error)
goto out_trans_cancel;
@@ -377,6 +485,10 @@
if (!args->trans)
goto out_unlock;
} else {
+ error = xfs_has_attr(args);
+ if (error != -EEXIST)
+ goto out_trans_cancel;
+
error = xfs_attr_remove_args(args);
if (error)
goto out_trans_cancel;
@@ -459,36 +571,54 @@
* External routines when attribute list is one block
*========================================================================*/
-/*
- * Add a name to the leaf attribute list structure
- *
- * This leaf block cannot have a "remote" value, we only call this routine
- * if bmap_one_block() says there is only one block (ie: no remote blks).
- */
-STATIC int
-xfs_attr_leaf_addname(
+/* Store info about a remote block */
+STATIC void
+xfs_attr_save_rmt_blk(
struct xfs_da_args *args)
{
- struct xfs_inode *dp;
- struct xfs_buf *bp;
- int retval, error, forkoff;
+ args->blkno2 = args->blkno;
+ args->index2 = args->index;
+ args->rmtblkno2 = args->rmtblkno;
+ args->rmtblkcnt2 = args->rmtblkcnt;
+ args->rmtvaluelen2 = args->rmtvaluelen;
+}
- trace_xfs_attr_leaf_addname(args);
+/* Set stored info about a remote block */
+STATIC void
+xfs_attr_restore_rmt_blk(
+ struct xfs_da_args *args)
+{
+ args->blkno = args->blkno2;
+ args->index = args->index2;
+ args->rmtblkno = args->rmtblkno2;
+ args->rmtblkcnt = args->rmtblkcnt2;
+ args->rmtvaluelen = args->rmtvaluelen2;
+}
- /*
- * Read the (only) block in the attribute list in.
- */
- dp = args->dp;
- args->blkno = 0;
- error = xfs_attr3_leaf_read(args->trans, args->dp, args->blkno, &bp);
- if (error)
- return error;
+/*
+ * Tries to add an attribute to an inode in leaf form
+ *
+ * This function is meant to execute as part of a delayed operation and leaves
+ * the transaction handling to the caller. On success the attribute is added
+ * and the inode and transaction are left dirty. If there is not enough space,
+ * the attr data is converted to node format and -ENOSPC is returned. Caller is
+ * responsible for handling the dirty inode and transaction or adding the attr
+ * in node format.
+ */
+STATIC int
+xfs_attr_leaf_try_add(
+ struct xfs_da_args *args,
+ struct xfs_buf *bp)
+{
+ int retval;
/*
* Look up the given attribute in the leaf block. Figure out if
* the given flags produce an error or call for an atomic rename.
*/
- retval = xfs_attr3_leaf_lookup_int(bp, args);
+ retval = xfs_attr_leaf_hasname(args, &bp);
+ if (retval != -ENOATTR && retval != -EEXIST)
+ return retval;
if (retval == -ENOATTR && (args->attr_flags & XATTR_REPLACE))
goto out_brelse;
if (retval == -EEXIST) {
@@ -499,11 +629,7 @@
/* save the attribute state for later removal*/
args->op_flags |= XFS_DA_OP_RENAME; /* an atomic rename */
- args->blkno2 = args->blkno; /* set 2nd entry info*/
- args->index2 = args->index;
- args->rmtblkno2 = args->rmtblkno;
- args->rmtblkcnt2 = args->rmtblkcnt;
- args->rmtvaluelen2 = args->rmtvaluelen;
+ xfs_attr_save_rmt_blk(args);
/*
* clear the remote attr state now that it is saved so that the
@@ -516,37 +642,35 @@
}
/*
- * Add the attribute to the leaf block, transitioning to a Btree
- * if required.
+ * Add the attribute to the leaf block
*/
- retval = xfs_attr3_leaf_add(bp, args);
- if (retval == -ENOSPC) {
- /*
- * Promote the attribute list to the Btree format, then
- * Commit that transaction so that the node_addname() call
- * can manage its own transactions.
- */
- error = xfs_attr3_leaf_to_node(args);
- if (error)
- return error;
- error = xfs_defer_finish(&args->trans);
- if (error)
- return error;
+ return xfs_attr3_leaf_add(bp, args);
- /*
- * Commit the current trans (including the inode) and start
- * a new one.
- */
- error = xfs_trans_roll_inode(&args->trans, dp);
- if (error)
- return error;
+out_brelse:
+ xfs_trans_brelse(args->trans, bp);
+ return retval;
+}
- /*
- * Fob the whole rest of the problem off on the Btree code.
- */
- error = xfs_attr_node_addname(args);
+
+/*
+ * Add a name to the leaf attribute list structure
+ *
+ * This leaf block cannot have a "remote" value, we only call this routine
+ * if bmap_one_block() says there is only one block (ie: no remote blks).
+ */
+STATIC int
+xfs_attr_leaf_addname(
+ struct xfs_da_args *args)
+{
+ int error, forkoff;
+ struct xfs_buf *bp = NULL;
+ struct xfs_inode *dp = args->dp;
+
+ trace_xfs_attr_leaf_addname(args);
+
+ error = xfs_attr_leaf_try_add(args, bp);
+ if (error)
return error;
- }
/*
* Commit the transaction that added the attr name so that
@@ -568,75 +692,93 @@
return error;
}
- /*
- * If this is an atomic rename operation, we must "flip" the
- * incomplete flags on the "new" and "old" attribute/value pairs
- * so that one disappears and one appears atomically. Then we
- * must remove the "old" attribute/value pair.
- */
- if (args->op_flags & XFS_DA_OP_RENAME) {
- /*
- * In a separate transaction, set the incomplete flag on the
- * "old" attr and clear the incomplete flag on the "new" attr.
- */
- error = xfs_attr3_leaf_flipflags(args);
- if (error)
- return error;
-
- /*
- * Dismantle the "old" attribute/value pair by removing
- * a "remote" value (if it exists).
- */
- args->index = args->index2;
- args->blkno = args->blkno2;
- args->rmtblkno = args->rmtblkno2;
- args->rmtblkcnt = args->rmtblkcnt2;
- args->rmtvaluelen = args->rmtvaluelen2;
- if (args->rmtblkno) {
- error = xfs_attr_rmtval_remove(args);
- if (error)
- return error;
- }
-
- /*
- * Read in the block containing the "old" attr, then
- * remove the "old" attr from that block (neat, huh!)
- */
- error = xfs_attr3_leaf_read(args->trans, args->dp, args->blkno,
- &bp);
- if (error)
- return error;
-
- xfs_attr3_leaf_remove(bp, args);
-
- /*
- * If the result is small enough, shrink it all into the inode.
- */
- if ((forkoff = xfs_attr_shortform_allfit(bp, dp))) {
- error = xfs_attr3_leaf_to_shortform(bp, args, forkoff);
- /* bp is gone due to xfs_da_shrink_inode */
- if (error)
- return error;
- error = xfs_defer_finish(&args->trans);
- if (error)
- return error;
- }
-
- /*
- * Commit the remove and start the next trans in series.
- */
- error = xfs_trans_roll_inode(&args->trans, dp);
-
- } else if (args->rmtblkno > 0) {
+ if (!(args->op_flags & XFS_DA_OP_RENAME)) {
/*
* Added a "remote" value, just clear the incomplete flag.
*/
- error = xfs_attr3_leaf_clearflag(args);
+ if (args->rmtblkno > 0)
+ error = xfs_attr3_leaf_clearflag(args);
+
+ return error;
}
+
+ /*
+ * If this is an atomic rename operation, we must "flip" the incomplete
+ * flags on the "new" and "old" attribute/value pairs so that one
+ * disappears and one appears atomically. Then we must remove the "old"
+ * attribute/value pair.
+ *
+ * In a separate transaction, set the incomplete flag on the "old" attr
+ * and clear the incomplete flag on the "new" attr.
+ */
+
+ error = xfs_attr3_leaf_flipflags(args);
+ if (error)
+ return error;
+ /*
+ * Commit the flag value change and start the next trans in series.
+ */
+ error = xfs_trans_roll_inode(&args->trans, args->dp);
+ if (error)
+ return error;
+
+ /*
+ * Dismantle the "old" attribute/value pair by removing a "remote" value
+ * (if it exists).
+ */
+ xfs_attr_restore_rmt_blk(args);
+
+ if (args->rmtblkno) {
+ error = xfs_attr_rmtval_invalidate(args);
+ if (error)
+ return error;
+
+ error = xfs_attr_rmtval_remove(args);
+ if (error)
+ return error;
+ }
+
+ /*
+ * Read in the block containing the "old" attr, then remove the "old"
+ * attr from that block (neat, huh!)
+ */
+ error = xfs_attr3_leaf_read(args->trans, args->dp, args->blkno,
+ &bp);
+ if (error)
+ return error;
+
+ xfs_attr3_leaf_remove(bp, args);
+
+ /*
+ * If the result is small enough, shrink it all into the inode.
+ */
+ forkoff = xfs_attr_shortform_allfit(bp, dp);
+ if (forkoff)
+ error = xfs_attr3_leaf_to_shortform(bp, args, forkoff);
+ /* bp is gone due to xfs_da_shrink_inode */
+
return error;
-out_brelse:
- xfs_trans_brelse(args->trans, bp);
- return retval;
+}
+
+/*
+ * Return EEXIST if attr is found, or ENOATTR if not
+ */
+STATIC int
+xfs_attr_leaf_hasname(
+ struct xfs_da_args *args,
+ struct xfs_buf **bp)
+{
+ int error = 0;
+
+ error = xfs_attr3_leaf_read(args->trans, args->dp, 0, bp);
+ if (error)
+ return error;
+
+ error = xfs_attr3_leaf_lookup_int(*bp, args);
+ if (error != -ENOATTR && error != -EEXIST)
+ xfs_trans_brelse(args->trans, *bp);
+
+ return error;
}
/*
@@ -659,31 +801,25 @@
* Remove the attribute.
*/
dp = args->dp;
- args->blkno = 0;
- error = xfs_attr3_leaf_read(args->trans, args->dp, args->blkno, &bp);
- if (error)
- return error;
- error = xfs_attr3_leaf_lookup_int(bp, args);
+ error = xfs_attr_leaf_hasname(args, &bp);
+
if (error == -ENOATTR) {
xfs_trans_brelse(args->trans, bp);
return error;
- }
+ } else if (error != -EEXIST)
+ return error;
xfs_attr3_leaf_remove(bp, args);
/*
* If the result is small enough, shrink it all into the inode.
*/
- if ((forkoff = xfs_attr_shortform_allfit(bp, dp))) {
- error = xfs_attr3_leaf_to_shortform(bp, args, forkoff);
+ forkoff = xfs_attr_shortform_allfit(bp, dp);
+ if (forkoff)
+ return xfs_attr3_leaf_to_shortform(bp, args, forkoff);
/* bp is gone due to xfs_da_shrink_inode */
- if (error)
- return error;
- error = xfs_defer_finish(&args->trans);
- if (error)
- return error;
- }
+
return 0;
}
@@ -703,21 +839,53 @@
trace_xfs_attr_leaf_get(args);
- args->blkno = 0;
- error = xfs_attr3_leaf_read(args->trans, args->dp, args->blkno, &bp);
- if (error)
- return error;
+ error = xfs_attr_leaf_hasname(args, &bp);
- error = xfs_attr3_leaf_lookup_int(bp, args);
- if (error != -EEXIST) {
+ if (error == -ENOATTR) {
xfs_trans_brelse(args->trans, bp);
return error;
- }
+ } else if (error != -EEXIST)
+ return error;
+
+
error = xfs_attr3_leaf_getvalue(bp, args);
xfs_trans_brelse(args->trans, bp);
return error;
}
+/*
+ * Return EEXIST if attr is found, or ENOATTR if not
+ * statep: If not null is set to point at the found state. Caller will
+ * be responsible for freeing the state in this case.
+ */
+STATIC int
+xfs_attr_node_hasname(
+ struct xfs_da_args *args,
+ struct xfs_da_state **statep)
+{
+ struct xfs_da_state *state;
+ int retval, error;
+
+ state = xfs_da_state_alloc(args);
+ if (statep != NULL)
+ *statep = NULL;
+
+ /*
+ * Search to see if name exists, and get back a pointer to it.
+ */
+ error = xfs_da3_node_lookup_int(state, &retval);
+ if (error) {
+ xfs_da_state_free(state);
+ return error;
+ }
+
+ if (statep != NULL)
+ *statep = state;
+ else
+ xfs_da_state_free(state);
+ return retval;
+}
+
/*========================================================================
* External routines when attribute list size > geo->blksize
*========================================================================*/
@@ -739,7 +907,6 @@
struct xfs_da_state *state;
struct xfs_da_state_blk *blk;
struct xfs_inode *dp;
- struct xfs_mount *mp;
int retval, error;
trace_xfs_attr_node_addname(args);
@@ -748,19 +915,15 @@
* Fill in bucket of arguments/results/context to carry around.
*/
dp = args->dp;
- mp = dp->i_mount;
restart:
- state = xfs_da_state_alloc();
- state->args = args;
- state->mp = mp;
-
/*
* Search to see if name already exists, and get back a pointer
* to where it should go.
*/
- error = xfs_da3_node_lookup_int(state, &retval);
- if (error)
+ retval = xfs_attr_node_hasname(args, &state);
+ if (retval != -ENOATTR && retval != -EEXIST)
goto out;
+
blk = &state->path.blk[ state->path.active-1 ];
ASSERT(blk->magic == XFS_ATTR_LEAF_MAGIC);
if (retval == -ENOATTR && (args->attr_flags & XATTR_REPLACE))
@@ -773,11 +936,7 @@
/* save the attribute state for later removal*/
args->op_flags |= XFS_DA_OP_RENAME; /* atomic rename op */
- args->blkno2 = args->blkno; /* set 2nd entry info*/
- args->index2 = args->index;
- args->rmtblkno2 = args->rmtblkno;
- args->rmtblkcnt2 = args->rmtblkcnt;
- args->rmtvaluelen2 = args->rmtvaluelen;
+ xfs_attr_save_rmt_blk(args);
/*
* clear the remote attr state now that it is saved so that the
@@ -863,82 +1022,75 @@
return error;
}
- /*
- * If this is an atomic rename operation, we must "flip" the
- * incomplete flags on the "new" and "old" attribute/value pairs
- * so that one disappears and one appears atomically. Then we
- * must remove the "old" attribute/value pair.
- */
- if (args->op_flags & XFS_DA_OP_RENAME) {
- /*
- * In a separate transaction, set the incomplete flag on the
- * "old" attr and clear the incomplete flag on the "new" attr.
- */
- error = xfs_attr3_leaf_flipflags(args);
- if (error)
- goto out;
-
- /*
- * Dismantle the "old" attribute/value pair by removing
- * a "remote" value (if it exists).
- */
- args->index = args->index2;
- args->blkno = args->blkno2;
- args->rmtblkno = args->rmtblkno2;
- args->rmtblkcnt = args->rmtblkcnt2;
- args->rmtvaluelen = args->rmtvaluelen2;
- if (args->rmtblkno) {
- error = xfs_attr_rmtval_remove(args);
- if (error)
- return error;
- }
-
- /*
- * Re-find the "old" attribute entry after any split ops.
- * The INCOMPLETE flag means that we will find the "old"
- * attr, not the "new" one.
- */
- args->attr_filter |= XFS_ATTR_INCOMPLETE;
- state = xfs_da_state_alloc();
- state->args = args;
- state->mp = mp;
- state->inleaf = 0;
- error = xfs_da3_node_lookup_int(state, &retval);
- if (error)
- goto out;
-
- /*
- * Remove the name and update the hashvals in the tree.
- */
- blk = &state->path.blk[ state->path.active-1 ];
- ASSERT(blk->magic == XFS_ATTR_LEAF_MAGIC);
- error = xfs_attr3_leaf_remove(blk->bp, args);
- xfs_da3_fixhashpath(state, &state->path);
-
- /*
- * Check to see if the tree needs to be collapsed.
- */
- if (retval && (state->path.active > 1)) {
- error = xfs_da3_join(state);
- if (error)
- goto out;
- error = xfs_defer_finish(&args->trans);
- if (error)
- goto out;
- }
-
- /*
- * Commit and start the next trans in the chain.
- */
- error = xfs_trans_roll_inode(&args->trans, dp);
- if (error)
- goto out;
-
- } else if (args->rmtblkno > 0) {
+ if (!(args->op_flags & XFS_DA_OP_RENAME)) {
/*
* Added a "remote" value, just clear the incomplete flag.
*/
- error = xfs_attr3_leaf_clearflag(args);
+ if (args->rmtblkno > 0)
+ error = xfs_attr3_leaf_clearflag(args);
+ retval = error;
+ goto out;
+ }
+
+ /*
+ * If this is an atomic rename operation, we must "flip" the incomplete
+ * flags on the "new" and "old" attribute/value pairs so that one
+ * disappears and one appears atomically. Then we must remove the "old"
+ * attribute/value pair.
+ *
+ * In a separate transaction, set the incomplete flag on the "old" attr
+ * and clear the incomplete flag on the "new" attr.
+ */
+ error = xfs_attr3_leaf_flipflags(args);
+ if (error)
+ goto out;
+ /*
+ * Commit the flag value change and start the next trans in series
+ */
+ error = xfs_trans_roll_inode(&args->trans, args->dp);
+ if (error)
+ goto out;
+
+ /*
+ * Dismantle the "old" attribute/value pair by removing a "remote" value
+ * (if it exists).
+ */
+ xfs_attr_restore_rmt_blk(args);
+
+ if (args->rmtblkno) {
+ error = xfs_attr_rmtval_invalidate(args);
+ if (error)
+ return error;
+
+ error = xfs_attr_rmtval_remove(args);
+ if (error)
+ return error;
+ }
+
+ /*
+ * Re-find the "old" attribute entry after any split ops. The INCOMPLETE
+ * flag means that we will find the "old" attr, not the "new" one.
+ */
+ args->attr_filter |= XFS_ATTR_INCOMPLETE;
+ state = xfs_da_state_alloc(args);
+ state->inleaf = 0;
+ error = xfs_da3_node_lookup_int(state, &retval);
+ if (error)
+ goto out;
+
+ /*
+ * Remove the name and update the hashvals in the tree.
+ */
+ blk = &state->path.blk[state->path.active-1];
+ ASSERT(blk->magic == XFS_ATTR_LEAF_MAGIC);
+ error = xfs_attr3_leaf_remove(blk->bp, args);
+ xfs_da3_fixhashpath(state, &state->path);
+
+ /*
+ * Check to see if the tree needs to be collapsed.
+ */
+ if (retval && (state->path.active > 1)) {
+ error = xfs_da3_join(state);
if (error)
goto out;
}
@@ -953,6 +1105,114 @@
}
/*
+ * Shrink an attribute from leaf to shortform
+ */
+STATIC int
+xfs_attr_node_shrink(
+ struct xfs_da_args *args,
+ struct xfs_da_state *state)
+{
+ struct xfs_inode *dp = args->dp;
+ int error, forkoff;
+ struct xfs_buf *bp;
+
+ /*
+ * Have to get rid of the copy of this dabuf in the state.
+ */
+ ASSERT(state->path.active == 1);
+ ASSERT(state->path.blk[0].bp);
+ state->path.blk[0].bp = NULL;
+
+ error = xfs_attr3_leaf_read(args->trans, args->dp, 0, &bp);
+ if (error)
+ return error;
+
+ forkoff = xfs_attr_shortform_allfit(bp, dp);
+ if (forkoff) {
+ error = xfs_attr3_leaf_to_shortform(bp, args, forkoff);
+ /* bp is gone due to xfs_da_shrink_inode */
+ } else
+ xfs_trans_brelse(args->trans, bp);
+
+ return error;
+}
+
+/*
+ * Mark an attribute entry INCOMPLETE and save pointers to the relevant buffers
+ * for later deletion of the entry.
+ */
+STATIC int
+xfs_attr_leaf_mark_incomplete(
+ struct xfs_da_args *args,
+ struct xfs_da_state *state)
+{
+ int error;
+
+ /*
+ * Fill in disk block numbers in the state structure
+ * so that we can get the buffers back after we commit
+ * several transactions in the following calls.
+ */
+ error = xfs_attr_fillstate(state);
+ if (error)
+ return error;
+
+ /*
+ * Mark the attribute as INCOMPLETE
+ */
+ return xfs_attr3_leaf_setflag(args);
+}
+
+/*
+ * Initial setup for xfs_attr_node_removename. Make sure the attr is there and
+ * the blocks are valid. Attr keys with remote blocks will be marked
+ * incomplete.
+ */
+STATIC
+int xfs_attr_node_removename_setup(
+ struct xfs_da_args *args,
+ struct xfs_da_state **state)
+{
+ int error;
+
+ error = xfs_attr_node_hasname(args, state);
+ if (error != -EEXIST)
+ return error;
+
+ ASSERT((*state)->path.blk[(*state)->path.active - 1].bp != NULL);
+ ASSERT((*state)->path.blk[(*state)->path.active - 1].magic ==
+ XFS_ATTR_LEAF_MAGIC);
+
+ if (args->rmtblkno > 0) {
+ error = xfs_attr_leaf_mark_incomplete(args, *state);
+ if (error)
+ return error;
+
+ return xfs_attr_rmtval_invalidate(args);
+ }
+
+ return 0;
+}
+
+STATIC int
+xfs_attr_node_remove_rmt(
+ struct xfs_da_args *args,
+ struct xfs_da_state *state)
+{
+ int error = 0;
+
+ error = xfs_attr_rmtval_remove(args);
+ if (error)
+ return error;
+
+ /*
+ * Refill the state structure with buffers, the prior calls released our
+ * buffers.
+ */
+ return xfs_attr_refillstate(state);
+}
+
+/*
* Remove a name from a B-tree attribute list.
*
* This will involve walking down the Btree, and may involve joining
@@ -965,64 +1225,22 @@
{
struct xfs_da_state *state;
struct xfs_da_state_blk *blk;
- struct xfs_inode *dp;
- struct xfs_buf *bp;
- int retval, error, forkoff;
+ int retval, error;
+ struct xfs_inode *dp = args->dp;
trace_xfs_attr_node_removename(args);
- /*
- * Tie a string around our finger to remind us where we are.
- */
- dp = args->dp;
- state = xfs_da_state_alloc();
- state->args = args;
- state->mp = dp->i_mount;
-
- /*
- * Search to see if name exists, and get back a pointer to it.
- */
- error = xfs_da3_node_lookup_int(state, &retval);
- if (error || (retval != -EEXIST)) {
- if (error == 0)
- error = retval;
+ error = xfs_attr_node_removename_setup(args, &state);
+ if (error)
goto out;
- }
/*
* If there is an out-of-line value, de-allocate the blocks.
* This is done before we remove the attribute so that we don't
* overflow the maximum size of a transaction and/or hit a deadlock.
*/
- blk = &state->path.blk[ state->path.active-1 ];
- ASSERT(blk->bp != NULL);
- ASSERT(blk->magic == XFS_ATTR_LEAF_MAGIC);
if (args->rmtblkno > 0) {
- /*
- * Fill in disk block numbers in the state structure
- * so that we can get the buffers back after we commit
- * several transactions in the following calls.
- */
- error = xfs_attr_fillstate(state);
- if (error)
- goto out;
-
- /*
- * Mark the attribute as INCOMPLETE, then bunmapi() the
- * remote value.
- */
- error = xfs_attr3_leaf_setflag(args);
- if (error)
- goto out;
- error = xfs_attr_rmtval_remove(args);
- if (error)
- goto out;
-
- /*
- * Refill the state structure with buffers, the prior calls
- * released our buffers.
- */
- error = xfs_attr_refillstate(state);
+ error = xfs_attr_node_remove_rmt(args, state);
if (error)
goto out;
}
@@ -1056,33 +1274,12 @@
/*
* If the result is small enough, push it all into the inode.
*/
- if (xfs_bmap_one_block(dp, XFS_ATTR_FORK)) {
- /*
- * Have to get rid of the copy of this dabuf in the state.
- */
- ASSERT(state->path.active == 1);
- ASSERT(state->path.blk[0].bp);
- state->path.blk[0].bp = NULL;
-
- error = xfs_attr3_leaf_read(args->trans, args->dp, 0, &bp);
- if (error)
- goto out;
-
- if ((forkoff = xfs_attr_shortform_allfit(bp, dp))) {
- error = xfs_attr3_leaf_to_shortform(bp, args, forkoff);
- /* bp is gone due to xfs_da_shrink_inode */
- if (error)
- goto out;
- error = xfs_defer_finish(&args->trans);
- if (error)
- goto out;
- } else
- xfs_trans_brelse(args->trans, bp);
- }
- error = 0;
+ if (xfs_bmap_one_block(dp, XFS_ATTR_FORK))
+ error = xfs_attr_node_shrink(args, state);
out:
- xfs_da_state_free(state);
+ if (state)
+ xfs_da_state_free(state);
return error;
}
@@ -1198,47 +1395,41 @@
* Returns 0 on successful retrieval, otherwise an error.
*/
STATIC int
-xfs_attr_node_get(xfs_da_args_t *args)
+xfs_attr_node_get(
+ struct xfs_da_args *args)
{
- xfs_da_state_t *state;
- xfs_da_state_blk_t *blk;
- int error, retval;
- int i;
+ struct xfs_da_state *state;
+ struct xfs_da_state_blk *blk;
+ int i;
+ int error;
trace_xfs_attr_node_get(args);
- state = xfs_da_state_alloc();
- state->args = args;
- state->mp = args->dp->i_mount;
-
/*
* Search to see if name exists, and get back a pointer to it.
*/
- error = xfs_da3_node_lookup_int(state, &retval);
- if (error) {
- retval = error;
- goto out_release;
- }
- if (retval != -EEXIST)
+ error = xfs_attr_node_hasname(args, &state);
+ if (error != -EEXIST)
goto out_release;
/*
* Get the value, local or "remote"
*/
blk = &state->path.blk[state->path.active - 1];
- retval = xfs_attr3_leaf_getvalue(blk->bp, args);
+ error = xfs_attr3_leaf_getvalue(blk->bp, args);
/*
* If not in a transaction, we have to release all the buffers.
*/
out_release:
- for (i = 0; i < state->path.active; i++) {
+ for (i = 0; state != NULL && i < state->path.active; i++) {
xfs_trans_brelse(args->trans, state->path.blk[i].bp);
state->path.blk[i].bp = NULL;
}
- xfs_da_state_free(state);
- return retval;
+ if (state)
+ xfs_da_state_free(state);
+ return error;
}
/* Returns true if the attribute entry name is valid. */
diff --git a/fs/xfs/libxfs/xfs_attr.h b/fs/xfs/libxfs/xfs_attr.h
index db47176..3e97a93 100644
--- a/fs/xfs/libxfs/xfs_attr.h
+++ b/fs/xfs/libxfs/xfs_attr.h
@@ -89,6 +89,7 @@
int xfs_attr_get(struct xfs_da_args *args);
int xfs_attr_set(struct xfs_da_args *args);
int xfs_attr_set_args(struct xfs_da_args *args);
+int xfs_has_attr(struct xfs_da_args *args);
int xfs_attr_remove_args(struct xfs_da_args *args);
bool xfs_attr_namecheck(const void *name, size_t length);
diff --git a/fs/xfs/libxfs/xfs_attr_leaf.c b/fs/xfs/libxfs/xfs_attr_leaf.c
index 2f7e89e..8623c81 100644
--- a/fs/xfs/libxfs/xfs_attr_leaf.c
+++ b/fs/xfs/libxfs/xfs_attr_leaf.c
@@ -660,18 +660,65 @@
}
/*
+ * Return -EEXIST if attr is found, or -ENOATTR if not
+ * args: args containing attribute name and namelen
+ * sfep: If not null, pointer will be set to the last attr entry found on
+ -EEXIST. On -ENOATTR pointer is left at the last entry in the list
+ * basep: If not null, pointer is set to the byte offset of the entry in the
+ * list on -EEXIST. On -ENOATTR, pointer is left at the byte offset of
+ * the last entry in the list
+ */
+int
+xfs_attr_sf_findname(
+ struct xfs_da_args *args,
+ struct xfs_attr_sf_entry **sfep,
+ unsigned int *basep)
+{
+ struct xfs_attr_shortform *sf;
+ struct xfs_attr_sf_entry *sfe;
+ unsigned int base = sizeof(struct xfs_attr_sf_hdr);
+ int size = 0;
+ int end;
+ int i;
+
+ sf = (struct xfs_attr_shortform *)args->dp->i_afp->if_u1.if_data;
+ sfe = &sf->list[0];
+ end = sf->hdr.count;
+ for (i = 0; i < end; sfe = XFS_ATTR_SF_NEXTENTRY(sfe),
+ base += size, i++) {
+ size = XFS_ATTR_SF_ENTSIZE(sfe);
+ if (!xfs_attr_match(args, sfe->namelen, sfe->nameval,
+ sfe->flags))
+ continue;
+ break;
+ }
+
+ if (sfep != NULL)
+ *sfep = sfe;
+
+ if (basep != NULL)
+ *basep = base;
+
+ if (i == end)
+ return -ENOATTR;
+ return -EEXIST;
+}
+
+/*
* Add a name/value pair to the shortform attribute list.
* Overflow from the inode has already been checked for.
*/
void
-xfs_attr_shortform_add(xfs_da_args_t *args, int forkoff)
+xfs_attr_shortform_add(
+ struct xfs_da_args *args,
+ int forkoff)
{
- xfs_attr_shortform_t *sf;
- xfs_attr_sf_entry_t *sfe;
- int i, offset, size;
- xfs_mount_t *mp;
- xfs_inode_t *dp;
- struct xfs_ifork *ifp;
+ struct xfs_attr_shortform *sf;
+ struct xfs_attr_sf_entry *sfe;
+ int offset, size;
+ struct xfs_mount *mp;
+ struct xfs_inode *dp;
+ struct xfs_ifork *ifp;
trace_xfs_attr_sf_add(args);
@@ -682,11 +729,8 @@
ifp = dp->i_afp;
ASSERT(ifp->if_flags & XFS_IFINLINE);
sf = (xfs_attr_shortform_t *)ifp->if_u1.if_data;
- sfe = &sf->list[0];
- for (i = 0; i < sf->hdr.count; sfe = XFS_ATTR_SF_NEXTENTRY(sfe), i++) {
- ASSERT(!xfs_attr_match(args, sfe->namelen, sfe->nameval,
- sfe->flags));
- }
+ if (xfs_attr_sf_findname(args, &sfe, NULL) == -EEXIST)
+ ASSERT(0);
offset = (char *)sfe - (char *)sf;
size = XFS_ATTR_SF_ENTSIZE_BYNAME(args->namelen, args->valuelen);
@@ -728,31 +772,27 @@
* Remove an attribute from the shortform attribute list structure.
*/
int
-xfs_attr_shortform_remove(xfs_da_args_t *args)
+xfs_attr_shortform_remove(
+ struct xfs_da_args *args)
{
- xfs_attr_shortform_t *sf;
- xfs_attr_sf_entry_t *sfe;
- int base, size=0, end, totsize, i;
- xfs_mount_t *mp;
- xfs_inode_t *dp;
+ struct xfs_attr_shortform *sf;
+ struct xfs_attr_sf_entry *sfe;
+ int size = 0, end, totsize;
+ unsigned int base;
+ struct xfs_mount *mp;
+ struct xfs_inode *dp;
+ int error;
trace_xfs_attr_sf_remove(args);
dp = args->dp;
mp = dp->i_mount;
- base = sizeof(xfs_attr_sf_hdr_t);
sf = (xfs_attr_shortform_t *)dp->i_afp->if_u1.if_data;
- sfe = &sf->list[0];
- end = sf->hdr.count;
- for (i = 0; i < end; sfe = XFS_ATTR_SF_NEXTENTRY(sfe),
- base += size, i++) {
- size = XFS_ATTR_SF_ENTSIZE(sfe);
- if (xfs_attr_match(args, sfe->namelen, sfe->nameval,
- sfe->flags))
- break;
- }
- if (i == end)
- return -ENOATTR;
+
+ error = xfs_attr_sf_findname(args, &sfe, &base);
+ if (error != -EEXIST)
+ return error;
+ size = XFS_ATTR_SF_ENTSIZE(sfe);
/*
* Fix up the attribute fork data, covering the hole
@@ -2742,10 +2782,7 @@
XFS_DA_LOGRANGE(leaf, name_rmt, sizeof(*name_rmt)));
}
- /*
- * Commit the flag value change and start the next trans in series.
- */
- return xfs_trans_roll_inode(&args->trans, args->dp);
+ return 0;
}
/*
@@ -2793,10 +2830,7 @@
XFS_DA_LOGRANGE(leaf, name_rmt, sizeof(*name_rmt)));
}
- /*
- * Commit the flag value change and start the next trans in series.
- */
- return xfs_trans_roll_inode(&args->trans, args->dp);
+ return 0;
}
/*
@@ -2911,10 +2945,5 @@
XFS_DA_LOGRANGE(leaf2, name_rmt, sizeof(*name_rmt)));
}
- /*
- * Commit the flag value change and start the next trans in series.
- */
- error = xfs_trans_roll_inode(&args->trans, args->dp);
-
- return error;
+ return 0;
}
diff --git a/fs/xfs/libxfs/xfs_attr_leaf.h b/fs/xfs/libxfs/xfs_attr_leaf.h
index 5be6be3..9b1c59f 100644
--- a/fs/xfs/libxfs/xfs_attr_leaf.h
+++ b/fs/xfs/libxfs/xfs_attr_leaf.h
@@ -52,6 +52,9 @@
int xfs_attr_shortform_to_leaf(struct xfs_da_args *args,
struct xfs_buf **leaf_bp);
int xfs_attr_shortform_remove(struct xfs_da_args *args);
+int xfs_attr_sf_findname(struct xfs_da_args *args,
+ struct xfs_attr_sf_entry **sfep,
+ unsigned int *basep);
int xfs_attr_shortform_allfit(struct xfs_buf *bp, struct xfs_inode *dp);
int xfs_attr_shortform_bytesfit(struct xfs_inode *dp, int bytes);
xfs_failaddr_t xfs_attr_shortform_verify(struct xfs_inode *ip);
diff --git a/fs/xfs/libxfs/xfs_attr_remote.c b/fs/xfs/libxfs/xfs_attr_remote.c
index 01ad7f3..3f80ced 100644
--- a/fs/xfs/libxfs/xfs_attr_remote.c
+++ b/fs/xfs/libxfs/xfs_attr_remote.c
@@ -440,32 +440,23 @@
}
/*
- * Write the value associated with an attribute into the out-of-line buffer
- * that we have defined for it.
+ * Find a "hole" in the attribute address space large enough for us to drop the
+ * new attribute's value into
*/
-int
-xfs_attr_rmtval_set(
+STATIC int
+xfs_attr_rmt_find_hole(
struct xfs_da_args *args)
{
struct xfs_inode *dp = args->dp;
struct xfs_mount *mp = dp->i_mount;
- struct xfs_bmbt_irec map;
- xfs_dablk_t lblkno;
- xfs_fileoff_t lfileoff = 0;
- uint8_t *src = args->value;
- int blkcnt;
- int valuelen;
- int nmap;
int error;
- int offset = 0;
-
- trace_xfs_attr_rmtval_set(args);
+ int blkcnt;
+ xfs_fileoff_t lfileoff = 0;
/*
- * Find a "hole" in the attribute address space large enough for
- * us to drop the new attribute's value into. Because CRC enable
- * attributes have headers, we can't just do a straight byte to FSB
- * conversion and have to take the header space into account.
+ * Because CRC enable attributes have headers, we can't just do a
+ * straight byte to FSB conversion and have to take the header space
+ * into account.
*/
blkcnt = xfs_attr3_rmt_blocks(mp, args->rmtvaluelen);
error = xfs_bmap_first_unused(args->trans, args->dp, blkcnt, &lfileoff,
@@ -473,48 +464,26 @@
if (error)
return error;
- args->rmtblkno = lblkno = (xfs_dablk_t)lfileoff;
+ args->rmtblkno = (xfs_dablk_t)lfileoff;
args->rmtblkcnt = blkcnt;
- /*
- * Roll through the "value", allocating blocks on disk as required.
- */
- while (blkcnt > 0) {
- /*
- * Allocate a single extent, up to the size of the value.
- *
- * Note that we have to consider this a data allocation as we
- * write the remote attribute without logging the contents.
- * Hence we must ensure that we aren't using blocks that are on
- * the busy list so that we don't overwrite blocks which have
- * recently been freed but their transactions are not yet
- * committed to disk. If we overwrite the contents of a busy
- * extent and then crash then the block may not contain the
- * correct metadata after log recovery occurs.
- */
- nmap = 1;
- error = xfs_bmapi_write(args->trans, dp, (xfs_fileoff_t)lblkno,
- blkcnt, XFS_BMAPI_ATTRFORK, args->total, &map,
- &nmap);
- if (error)
- return error;
- error = xfs_defer_finish(&args->trans);
- if (error)
- return error;
+ return 0;
+}
- ASSERT(nmap == 1);
- ASSERT((map.br_startblock != DELAYSTARTBLOCK) &&
- (map.br_startblock != HOLESTARTBLOCK));
- lblkno += map.br_blockcount;
- blkcnt -= map.br_blockcount;
-
- /*
- * Start the next trans in the chain.
- */
- error = xfs_trans_roll_inode(&args->trans, dp);
- if (error)
- return error;
- }
+STATIC int
+xfs_attr_rmtval_set_value(
+ struct xfs_da_args *args)
+{
+ struct xfs_inode *dp = args->dp;
+ struct xfs_mount *mp = dp->i_mount;
+ struct xfs_bmbt_irec map;
+ xfs_dablk_t lblkno;
+ uint8_t *src = args->value;
+ int blkcnt;
+ int valuelen;
+ int nmap;
+ int error;
+ int offset = 0;
/*
* Roll through the "value", copying the attribute value to the
@@ -595,19 +564,82 @@
}
/*
+ * Write the value associated with an attribute into the out-of-line buffer
+ * that we have defined for it.
+ */
+int
+xfs_attr_rmtval_set(
+ struct xfs_da_args *args)
+{
+ struct xfs_inode *dp = args->dp;
+ struct xfs_bmbt_irec map;
+ xfs_dablk_t lblkno;
+ int blkcnt;
+ int nmap;
+ int error;
+
+ trace_xfs_attr_rmtval_set(args);
+
+ error = xfs_attr_rmt_find_hole(args);
+ if (error)
+ return error;
+
+ blkcnt = args->rmtblkcnt;
+ lblkno = (xfs_dablk_t)args->rmtblkno;
+ /*
+ * Roll through the "value", allocating blocks on disk as required.
+ */
+ while (blkcnt > 0) {
+ /*
+ * Allocate a single extent, up to the size of the value.
+ *
+ * Note that we have to consider this a data allocation as we
+ * write the remote attribute without logging the contents.
+ * Hence we must ensure that we aren't using blocks that are on
+ * the busy list so that we don't overwrite blocks which have
+ * recently been freed but their transactions are not yet
+ * committed to disk. If we overwrite the contents of a busy
+ * extent and then crash then the block may not contain the
+ * correct metadata after log recovery occurs.
+ */
+ nmap = 1;
+ error = xfs_bmapi_write(args->trans, dp, (xfs_fileoff_t)lblkno,
+ blkcnt, XFS_BMAPI_ATTRFORK, args->total, &map,
+ &nmap);
+ if (error)
+ return error;
+ error = xfs_defer_finish(&args->trans);
+ if (error)
+ return error;
+
+ ASSERT(nmap == 1);
+ ASSERT((map.br_startblock != DELAYSTARTBLOCK) &&
+ (map.br_startblock != HOLESTARTBLOCK));
+ lblkno += map.br_blockcount;
+ blkcnt -= map.br_blockcount;
+
+ /*
+ * Start the next trans in the chain.
+ */
+ error = xfs_trans_roll_inode(&args->trans, dp);
+ if (error)
+ return error;
+ }
+
+ return xfs_attr_rmtval_set_value(args);
+}
+
+/*
* Remove the value associated with an attribute by deleting the
* out-of-line buffer that it is stored on.
*/
int
-xfs_attr_rmtval_remove(
+xfs_attr_rmtval_invalidate(
struct xfs_da_args *args)
{
xfs_dablk_t lblkno;
int blkcnt;
int error;
- int done;
-
- trace_xfs_attr_rmtval_remove(args);
/*
* Roll through the "value", invalidating the attribute value's blocks.
@@ -635,21 +667,29 @@
lblkno += map.br_blockcount;
blkcnt -= map.br_blockcount;
}
+ return 0;
+}
+
+/*
+ * Remove the value associated with an attribute by deleting the
+ * out-of-line buffer that it is stored on.
+ */
+int
+xfs_attr_rmtval_remove(
+ struct xfs_da_args *args)
+{
+ int error;
+ int retval;
+
+ trace_xfs_attr_rmtval_remove(args);
/*
* Keep de-allocating extents until the remote-value region is gone.
*/
- lblkno = args->rmtblkno;
- blkcnt = args->rmtblkcnt;
- done = 0;
- while (!done) {
- error = xfs_bunmapi(args->trans, args->dp, lblkno, blkcnt,
- XFS_BMAPI_ATTRFORK, 1, &done);
- if (error)
- return error;
- error = xfs_defer_finish(&args->trans);
- if (error)
- return error;
+ do {
+ retval = __xfs_attr_rmtval_remove(args);
+ if (retval && retval != -EAGAIN)
+ return retval;
/*
* Close out trans and start the next one in the chain.
@@ -657,6 +697,36 @@
error = xfs_trans_roll_inode(&args->trans, args->dp);
if (error)
return error;
- }
+ } while (retval == -EAGAIN);
+
return 0;
}
+
+/*
+ * Remove the value associated with an attribute by deleting the out-of-line
+ * buffer that it is stored on. Returns EAGAIN for the caller to refresh the
+ * transaction and re-call the function
+ */
+int
+__xfs_attr_rmtval_remove(
+ struct xfs_da_args *args)
+{
+ int error, done;
+
+ /*
+ * Unmap value blocks for this attr.
+ */
+ error = xfs_bunmapi(args->trans, args->dp, args->rmtblkno,
+ args->rmtblkcnt, XFS_BMAPI_ATTRFORK, 1, &done);
+ if (error)
+ return error;
+
+ error = xfs_defer_finish(&args->trans);
+ if (error)
+ return error;
+
+ if (!done)
+ return -EAGAIN;
+
+ return error;
+}
diff --git a/fs/xfs/libxfs/xfs_attr_remote.h b/fs/xfs/libxfs/xfs_attr_remote.h
index e1144f2..9eee615 100644
--- a/fs/xfs/libxfs/xfs_attr_remote.h
+++ b/fs/xfs/libxfs/xfs_attr_remote.h
@@ -13,5 +13,6 @@
int xfs_attr_rmtval_remove(struct xfs_da_args *args);
int xfs_attr_rmtval_stale(struct xfs_inode *ip, struct xfs_bmbt_irec *map,
xfs_buf_flags_t incore_flags);
-
+int xfs_attr_rmtval_invalidate(struct xfs_da_args *args);
+int __xfs_attr_rmtval_remove(struct xfs_da_args *args);
#endif /* __XFS_ATTR_REMOTE_H__ */
diff --git a/fs/xfs/libxfs/xfs_bmap.c b/fs/xfs/libxfs/xfs_bmap.c
index 667cdd0..9c40d59 100644
--- a/fs/xfs/libxfs/xfs_bmap.c
+++ b/fs/xfs/libxfs/xfs_bmap.c
@@ -553,7 +553,8 @@
#endif
ASSERT(xfs_bmap_free_item_zone != NULL);
- new = kmem_zone_alloc(xfs_bmap_free_item_zone, 0);
+ new = kmem_cache_alloc(xfs_bmap_free_item_zone,
+ GFP_KERNEL | __GFP_NOFAIL);
new->xefi_startblock = bno;
new->xefi_blockcount = (xfs_extlen_t)len;
if (oinfo)
@@ -1098,7 +1099,10 @@
if (error)
goto trans_cancel;
ASSERT(ip->i_afp == NULL);
- ip->i_afp = kmem_zone_zalloc(xfs_ifork_zone, 0);
+
+ ip->i_afp = kmem_cache_zalloc(xfs_ifork_zone,
+ GFP_KERNEL | __GFP_NOFAIL);
+
ip->i_afp->if_format = XFS_DINODE_FMT_EXTENTS;
ip->i_afp->if_flags = XFS_IFEXTENTS;
logflags = 0;
diff --git a/fs/xfs/libxfs/xfs_bmap.h b/fs/xfs/libxfs/xfs_bmap.h
index 6028a3c..e1bd484 100644
--- a/fs/xfs/libxfs/xfs_bmap.h
+++ b/fs/xfs/libxfs/xfs_bmap.h
@@ -158,17 +158,22 @@
{ BMAP_ATTRFORK, "ATTR" }, \
{ BMAP_COWFORK, "COW" }
+/* Return true if the extent is an allocated extent, written or not. */
+static inline bool xfs_bmap_is_real_extent(struct xfs_bmbt_irec *irec)
+{
+ return irec->br_startblock != HOLESTARTBLOCK &&
+ irec->br_startblock != DELAYSTARTBLOCK &&
+ !isnullstartblock(irec->br_startblock);
+}
/*
* Return true if the extent is a real, allocated extent, or false if it is a
* delayed allocation, and unwritten extent or a hole.
*/
-static inline bool xfs_bmap_is_real_extent(struct xfs_bmbt_irec *irec)
+static inline bool xfs_bmap_is_written_extent(struct xfs_bmbt_irec *irec)
{
- return irec->br_state != XFS_EXT_UNWRITTEN &&
- irec->br_startblock != HOLESTARTBLOCK &&
- irec->br_startblock != DELAYSTARTBLOCK &&
- !isnullstartblock(irec->br_startblock);
+ return xfs_bmap_is_real_extent(irec) &&
+ irec->br_state != XFS_EXT_UNWRITTEN;
}
/*
diff --git a/fs/xfs/libxfs/xfs_bmap_btree.c b/fs/xfs/libxfs/xfs_bmap_btree.c
index d9c63f1..ecec604 100644
--- a/fs/xfs/libxfs/xfs_bmap_btree.c
+++ b/fs/xfs/libxfs/xfs_bmap_btree.c
@@ -552,7 +552,7 @@
struct xfs_btree_cur *cur;
ASSERT(whichfork != XFS_COW_FORK);
- cur = kmem_zone_zalloc(xfs_btree_cur_zone, KM_NOFS);
+ cur = kmem_cache_zalloc(xfs_btree_cur_zone, GFP_NOFS | __GFP_NOFAIL);
cur->bc_tp = tp;
cur->bc_mp = mp;
diff --git a/fs/xfs/libxfs/xfs_btree_staging.h b/fs/xfs/libxfs/xfs_btree_staging.h
index 643f0f9..f0d2976 100644
--- a/fs/xfs/libxfs/xfs_btree_staging.h
+++ b/fs/xfs/libxfs/xfs_btree_staging.h
@@ -18,7 +18,7 @@
unsigned int af_blocks;
};
-/* Cursor interactions with with fake roots for AG-rooted btrees. */
+/* Cursor interactions with fake roots for AG-rooted btrees. */
void xfs_btree_stage_afakeroot(struct xfs_btree_cur *cur,
struct xbtree_afakeroot *afake);
void xfs_btree_commit_afakeroot(struct xfs_btree_cur *cur, struct xfs_trans *tp,
@@ -45,7 +45,7 @@
unsigned int if_extents;
};
-/* Cursor interactions with with fake roots for inode-rooted btrees. */
+/* Cursor interactions with fake roots for inode-rooted btrees. */
void xfs_btree_stage_ifakeroot(struct xfs_btree_cur *cur,
struct xbtree_ifakeroot *ifake,
struct xfs_btree_ops **new_ops);
@@ -90,7 +90,7 @@
/*
* Number of free records to leave in each leaf block. If the caller
- * sets this to -1, the slack value will be calculated to be be halfway
+ * sets this to -1, the slack value will be calculated to be halfway
* between maxrecs and minrecs. This typically leaves the block 75%
* full. Note that slack values are not enforced on inode root blocks.
*/
diff --git a/fs/xfs/libxfs/xfs_da_btree.c b/fs/xfs/libxfs/xfs_da_btree.c
index 897749c..e46bc03 100644
--- a/fs/xfs/libxfs/xfs_da_btree.c
+++ b/fs/xfs/libxfs/xfs_da_btree.c
@@ -78,10 +78,16 @@
* Allocate a dir-state structure.
* We don't put them on the stack since they're large.
*/
-xfs_da_state_t *
-xfs_da_state_alloc(void)
+struct xfs_da_state *
+xfs_da_state_alloc(
+ struct xfs_da_args *args)
{
- return kmem_zone_zalloc(xfs_da_state_zone, KM_NOFS);
+ struct xfs_da_state *state;
+
+ state = kmem_cache_zalloc(xfs_da_state_zone, GFP_NOFS | __GFP_NOFAIL);
+ state->args = args;
+ state->mp = args->dp->i_mount;
+ return state;
}
/*
diff --git a/fs/xfs/libxfs/xfs_da_btree.h b/fs/xfs/libxfs/xfs_da_btree.h
index 6e25de6..ad5dd32 100644
--- a/fs/xfs/libxfs/xfs_da_btree.h
+++ b/fs/xfs/libxfs/xfs_da_btree.h
@@ -219,7 +219,7 @@
const unsigned char *name, int len);
-xfs_da_state_t *xfs_da_state_alloc(void);
+struct xfs_da_state *xfs_da_state_alloc(struct xfs_da_args *args);
void xfs_da_state_free(xfs_da_state_t *state);
void xfs_da3_node_hdr_from_disk(struct xfs_mount *mp,
diff --git a/fs/xfs/libxfs/xfs_dir2_node.c b/fs/xfs/libxfs/xfs_dir2_node.c
index 6ac4aad..5d51265 100644
--- a/fs/xfs/libxfs/xfs_dir2_node.c
+++ b/fs/xfs/libxfs/xfs_dir2_node.c
@@ -2015,9 +2015,7 @@
/*
* Allocate and initialize the state (btree cursor).
*/
- state = xfs_da_state_alloc();
- state->args = args;
- state->mp = args->dp->i_mount;
+ state = xfs_da_state_alloc(args);
/*
* Look up the name. We're not supposed to find it, but
* this gives us the insertion point.
@@ -2086,9 +2084,8 @@
/*
* Allocate and initialize the btree cursor.
*/
- state = xfs_da_state_alloc();
- state->args = args;
- state->mp = args->dp->i_mount;
+ state = xfs_da_state_alloc(args);
+
/*
* Fill in the path to the entry in the cursor.
*/
@@ -2139,9 +2136,7 @@
/*
* Allocate and initialize the btree cursor.
*/
- state = xfs_da_state_alloc();
- state->args = args;
- state->mp = args->dp->i_mount;
+ state = xfs_da_state_alloc(args);
/* Look up the entry we're deleting, set up the cursor. */
error = xfs_da3_node_lookup_int(state, &rval);
@@ -2206,9 +2201,7 @@
/*
* Allocate and initialize the btree cursor.
*/
- state = xfs_da_state_alloc();
- state->args = args;
- state->mp = args->dp->i_mount;
+ state = xfs_da_state_alloc(args);
/*
* We have to save new inode number and ftype since
diff --git a/fs/xfs/libxfs/xfs_dquot_buf.c b/fs/xfs/libxfs/xfs_dquot_buf.c
index bedc1e7..5a2db00b 100644
--- a/fs/xfs/libxfs/xfs_dquot_buf.c
+++ b/fs/xfs/libxfs/xfs_dquot_buf.c
@@ -37,9 +37,10 @@
xfs_dquot_verify(
struct xfs_mount *mp,
struct xfs_disk_dquot *ddq,
- xfs_dqid_t id,
- uint type) /* used only during quotacheck */
+ xfs_dqid_t id) /* used only during quotacheck */
{
+ __u8 ddq_type;
+
/*
* We can encounter an uninitialized dquot buffer for 2 reasons:
* 1. If we crash while deleting the quotainode(s), and those blks got
@@ -60,11 +61,12 @@
if (ddq->d_version != XFS_DQUOT_VERSION)
return __this_address;
- if (type && ddq->d_flags != type)
+ if (ddq->d_type & ~XFS_DQTYPE_ANY)
return __this_address;
- if (ddq->d_flags != XFS_DQ_USER &&
- ddq->d_flags != XFS_DQ_PROJ &&
- ddq->d_flags != XFS_DQ_GROUP)
+ ddq_type = ddq->d_type & XFS_DQTYPE_REC_MASK;
+ if (ddq_type != XFS_DQTYPE_USER &&
+ ddq_type != XFS_DQTYPE_PROJ &&
+ ddq_type != XFS_DQTYPE_GROUP)
return __this_address;
if (id != -1 && id != be32_to_cpu(ddq->d_id))
@@ -95,14 +97,13 @@
xfs_dqblk_verify(
struct xfs_mount *mp,
struct xfs_dqblk *dqb,
- xfs_dqid_t id,
- uint type) /* used only during quotacheck */
+ xfs_dqid_t id) /* used only during quotacheck */
{
if (xfs_sb_version_hascrc(&mp->m_sb) &&
!uuid_equal(&dqb->dd_uuid, &mp->m_sb.sb_meta_uuid))
return __this_address;
- return xfs_dquot_verify(mp, &dqb->dd_diskdq, id, type);
+ return xfs_dquot_verify(mp, &dqb->dd_diskdq, id);
}
/*
@@ -113,7 +114,7 @@
struct xfs_mount *mp,
struct xfs_dqblk *dqb,
xfs_dqid_t id,
- uint type)
+ xfs_dqtype_t type)
{
/*
* Typically, a repair is only requested by quotacheck.
@@ -123,7 +124,7 @@
dqb->dd_diskdq.d_magic = cpu_to_be16(XFS_DQUOT_MAGIC);
dqb->dd_diskdq.d_version = XFS_DQUOT_VERSION;
- dqb->dd_diskdq.d_flags = type;
+ dqb->dd_diskdq.d_type = type;
dqb->dd_diskdq.d_id = cpu_to_be32(id);
if (xfs_sb_version_hascrc(&mp->m_sb)) {
@@ -205,7 +206,7 @@
if (i == 0)
id = be32_to_cpu(ddq->d_id);
- fa = xfs_dqblk_verify(mp, &dqb[i], id + i, 0);
+ fa = xfs_dqblk_verify(mp, &dqb[i], id + i);
if (fa) {
if (!readahead)
xfs_buf_verifier_error(bp, -EFSCORRUPTED,
diff --git a/fs/xfs/libxfs/xfs_format.h b/fs/xfs/libxfs/xfs_format.h
index b42a52b..31b7ece 100644
--- a/fs/xfs/libxfs/xfs_format.h
+++ b/fs/xfs/libxfs/xfs_format.h
@@ -1149,16 +1149,26 @@
#define XFS_DQUOT_MAGIC 0x4451 /* 'DQ' */
#define XFS_DQUOT_VERSION (uint8_t)0x01 /* latest version number */
+#define XFS_DQTYPE_USER 0x01 /* user dquot record */
+#define XFS_DQTYPE_PROJ 0x02 /* project dquot record */
+#define XFS_DQTYPE_GROUP 0x04 /* group dquot record */
+
+/* bitmask to determine if this is a user/group/project dquot */
+#define XFS_DQTYPE_REC_MASK (XFS_DQTYPE_USER | \
+ XFS_DQTYPE_PROJ | \
+ XFS_DQTYPE_GROUP)
+
+#define XFS_DQTYPE_ANY (XFS_DQTYPE_REC_MASK)
+
/*
- * This is the main portion of the on-disk representation of quota
- * information for a user. This is the q_core of the struct xfs_dquot that
- * is kept in kernel memory. We pad this with some more expansion room
- * to construct the on disk structure.
+ * This is the main portion of the on-disk representation of quota information
+ * for a user. We pad this with some more expansion room to construct the on
+ * disk structure.
*/
struct xfs_disk_dquot {
__be16 d_magic; /* dquot magic = XFS_DQUOT_MAGIC */
__u8 d_version; /* dquot version */
- __u8 d_flags; /* XFS_DQ_USER/PROJ/GROUP */
+ __u8 d_type; /* XFS_DQTYPE_USER/PROJ/GROUP */
__be32 d_id; /* user,project,group id */
__be64 d_blk_hardlimit;/* absolute limit on disk blks */
__be64 d_blk_softlimit;/* preferred limit on disk blks */
@@ -1199,6 +1209,22 @@
#define XFS_DQUOT_CRC_OFF offsetof(struct xfs_dqblk, dd_crc)
/*
+ * This defines the unit of allocation of dquots.
+ *
+ * Currently, it is just one file system block, and a 4K blk contains 30
+ * (136 * 30 = 4080) dquots. It's probably not worth trying to make
+ * this more dynamic.
+ *
+ * However, if this number is changed, we have to make sure that we don't
+ * implicitly assume that we do allocations in chunks of a single filesystem
+ * block in the dquot/xqm code.
+ *
+ * This is part of the ondisk format because the structure size is not a power
+ * of two, which leaves slack at the end of the disk block.
+ */
+#define XFS_DQUOT_CLUSTER_SIZE_FSB (xfs_filblks_t)1
+
+/*
* Remote symlink format and access functions.
*/
#define XFS_SYMLINK_MAGIC 0x58534c4d /* XSLM */
diff --git a/fs/xfs/libxfs/xfs_ialloc.c b/fs/xfs/libxfs/xfs_ialloc.c
index 7fcf62b..f742a96 100644
--- a/fs/xfs/libxfs/xfs_ialloc.c
+++ b/fs/xfs/libxfs/xfs_ialloc.c
@@ -888,10 +888,9 @@
*/
be32_add_cpu(&agi->agi_count, newlen);
be32_add_cpu(&agi->agi_freecount, newlen);
- pag = xfs_perag_get(args.mp, agno);
+ pag = agbp->b_pag;
pag->pagi_freecount += newlen;
pag->pagi_count += newlen;
- xfs_perag_put(pag);
agi->agi_newino = cpu_to_be32(newino);
/*
@@ -1134,7 +1133,7 @@
xfs_agnumber_t agno = be32_to_cpu(agi->agi_seqno);
xfs_agnumber_t pagno = XFS_INO_TO_AGNO(mp, parent);
xfs_agino_t pagino = XFS_INO_TO_AGINO(mp, parent);
- struct xfs_perag *pag;
+ struct xfs_perag *pag = agbp->b_pag;
struct xfs_btree_cur *cur, *tcur;
struct xfs_inobt_rec_incore rec, trec;
xfs_ino_t ino;
@@ -1143,8 +1142,6 @@
int i, j;
int searchdistance = 10;
- pag = xfs_perag_get(mp, agno);
-
ASSERT(pag->pagi_init);
ASSERT(pag->pagi_inodeok);
ASSERT(pag->pagi_freecount > 0);
@@ -1384,14 +1381,12 @@
xfs_btree_del_cursor(cur, XFS_BTREE_NOERROR);
xfs_trans_mod_sb(tp, XFS_TRANS_SB_IFREE, -1);
- xfs_perag_put(pag);
*inop = ino;
return 0;
error1:
xfs_btree_del_cursor(tcur, XFS_BTREE_ERROR);
error0:
xfs_btree_del_cursor(cur, XFS_BTREE_ERROR);
- xfs_perag_put(pag);
return error;
}
@@ -1587,7 +1582,6 @@
xfs_agnumber_t agno = be32_to_cpu(agi->agi_seqno);
xfs_agnumber_t pagno = XFS_INO_TO_AGNO(mp, parent);
xfs_agino_t pagino = XFS_INO_TO_AGINO(mp, parent);
- struct xfs_perag *pag;
struct xfs_btree_cur *cur; /* finobt cursor */
struct xfs_btree_cur *icur; /* inobt cursor */
struct xfs_inobt_rec_incore rec;
@@ -1599,8 +1593,6 @@
if (!xfs_sb_version_hasfinobt(&mp->m_sb))
return xfs_dialloc_ag_inobt(tp, agbp, parent, inop);
- pag = xfs_perag_get(mp, agno);
-
/*
* If pagino is 0 (this is the root inode allocation) use newino.
* This must work because we've just allocated some.
@@ -1667,7 +1659,7 @@
*/
be32_add_cpu(&agi->agi_freecount, -1);
xfs_ialloc_log_agi(tp, agbp, XFS_AGI_FREECOUNT);
- pag->pagi_freecount--;
+ agbp->b_pag->pagi_freecount--;
xfs_trans_mod_sb(tp, XFS_TRANS_SB_IFREE, -1);
@@ -1680,7 +1672,6 @@
xfs_btree_del_cursor(icur, XFS_BTREE_NOERROR);
xfs_btree_del_cursor(cur, XFS_BTREE_NOERROR);
- xfs_perag_put(pag);
*inop = ino;
return 0;
@@ -1688,7 +1679,6 @@
xfs_btree_del_cursor(icur, XFS_BTREE_ERROR);
error_cur:
xfs_btree_del_cursor(cur, XFS_BTREE_ERROR);
- xfs_perag_put(pag);
return error;
}
@@ -1945,7 +1935,6 @@
{
struct xfs_agi *agi = agbp->b_addr;
xfs_agnumber_t agno = be32_to_cpu(agi->agi_seqno);
- struct xfs_perag *pag;
struct xfs_btree_cur *cur;
struct xfs_inobt_rec_incore rec;
int ilen;
@@ -2007,6 +1996,8 @@
if (!(mp->m_flags & XFS_MOUNT_IKEEP) &&
rec.ir_free == XFS_INOBT_ALL_FREE &&
mp->m_sb.sb_inopblock <= XFS_INODES_PER_CHUNK) {
+ struct xfs_perag *pag = agbp->b_pag;
+
xic->deleted = true;
xic->first_ino = XFS_AGINO_TO_INO(mp, agno, rec.ir_startino);
xic->alloc = xfs_inobt_irec_to_allocmask(&rec);
@@ -2020,10 +2011,8 @@
be32_add_cpu(&agi->agi_count, -ilen);
be32_add_cpu(&agi->agi_freecount, -(ilen - 1));
xfs_ialloc_log_agi(tp, agbp, XFS_AGI_COUNT | XFS_AGI_FREECOUNT);
- pag = xfs_perag_get(mp, agno);
pag->pagi_freecount -= ilen - 1;
pag->pagi_count -= ilen;
- xfs_perag_put(pag);
xfs_trans_mod_sb(tp, XFS_TRANS_SB_ICOUNT, -ilen);
xfs_trans_mod_sb(tp, XFS_TRANS_SB_IFREE, -(ilen - 1));
@@ -2049,9 +2038,7 @@
*/
be32_add_cpu(&agi->agi_freecount, 1);
xfs_ialloc_log_agi(tp, agbp, XFS_AGI_FREECOUNT);
- pag = xfs_perag_get(mp, agno);
- pag->pagi_freecount++;
- xfs_perag_put(pag);
+ agbp->b_pag->pagi_freecount++;
xfs_trans_mod_sb(tp, XFS_TRANS_SB_IFREE, 1);
}
@@ -2661,7 +2648,7 @@
return error;
agi = (*bpp)->b_addr;
- pag = xfs_perag_get(mp, agno);
+ pag = (*bpp)->b_pag;
if (!pag->pagi_init) {
pag->pagi_freecount = be32_to_cpu(agi->agi_freecount);
pag->pagi_count = be32_to_cpu(agi->agi_count);
@@ -2674,7 +2661,6 @@
*/
ASSERT(pag->pagi_freecount == be32_to_cpu(agi->agi_freecount) ||
XFS_FORCED_SHUTDOWN(mp));
- xfs_perag_put(pag);
return 0;
}
diff --git a/fs/xfs/libxfs/xfs_ialloc_btree.c b/fs/xfs/libxfs/xfs_ialloc_btree.c
index b2c122a..3c8aebc 100644
--- a/fs/xfs/libxfs/xfs_ialloc_btree.c
+++ b/fs/xfs/libxfs/xfs_ialloc_btree.c
@@ -411,7 +411,7 @@
{
struct xfs_btree_cur *cur;
- cur = kmem_zone_zalloc(xfs_btree_cur_zone, KM_NOFS);
+ cur = kmem_cache_zalloc(xfs_btree_cur_zone, GFP_NOFS | __GFP_NOFAIL);
cur->bc_tp = tp;
cur->bc_mp = mp;
cur->bc_btnum = btnum;
diff --git a/fs/xfs/libxfs/xfs_inode_buf.c b/fs/xfs/libxfs/xfs_inode_buf.c
index 6f84ea8..8d5dd08 100644
--- a/fs/xfs/libxfs/xfs_inode_buf.c
+++ b/fs/xfs/libxfs/xfs_inode_buf.c
@@ -21,30 +21,6 @@
#include <linux/iversion.h>
/*
- * Check that none of the inode's in the buffer have a next
- * unlinked field of 0.
- */
-#if defined(DEBUG)
-void
-xfs_inobp_check(
- xfs_mount_t *mp,
- xfs_buf_t *bp)
-{
- int i;
- xfs_dinode_t *dip;
-
- for (i = 0; i < M_IGEO(mp)->inodes_per_cluster; i++) {
- dip = xfs_buf_offset(bp, i * mp->m_sb.sb_inodesize);
- if (!dip->di_next_unlinked) {
- xfs_alert(mp,
- "Detected bogus zero next_unlinked field in inode %d buffer 0x%llx.",
- i, (long long)bp->b_bn);
- }
- }
-}
-#endif
-
-/*
* If we are doing readahead on an inode buffer, we might be in log recovery
* reading an inode allocation buffer that hasn't yet been replayed, and hence
* has not had the inode cores stamped into it. Hence for readahead, the buffer
@@ -53,10 +29,10 @@
* If the readahead buffer is invalid, we need to mark it with an error and
* clear the DONE status of the buffer so that a followup read will re-read it
* from disk. We don't report the error otherwise to avoid warnings during log
- * recovery and we don't get unnecssary panics on debug kernels. We use EIO here
+ * recovery and we don't get unnecessary panics on debug kernels. We use EIO here
* because all we want to do is say readahead failed; there is no-one to report
* the error to, so this will distinguish it from a non-ra verifier failure.
- * Changes to this readahead error behavour also need to be reflected in
+ * Changes to this readahead error behaviour also need to be reflected in
* xfs_dquot_buf_readahead_verify().
*/
static void
@@ -176,7 +152,8 @@
}
*bpp = bp;
- *dipp = xfs_buf_offset(bp, imap->im_boffset);
+ if (dipp)
+ *dipp = xfs_buf_offset(bp, imap->im_boffset);
return 0;
}
@@ -203,7 +180,7 @@
/*
* First get the permanent information that is needed to allocate an
* inode. If the inode is unused, mode is zero and we shouldn't mess
- * with the unitialized part of it.
+ * with the uninitialized part of it.
*/
to->di_flushiter = be16_to_cpu(from->di_flushiter);
inode->i_generation = be32_to_cpu(from->di_gen);
diff --git a/fs/xfs/libxfs/xfs_inode_buf.h b/fs/xfs/libxfs/xfs_inode_buf.h
index 865ac49..6b08b9d 100644
--- a/fs/xfs/libxfs/xfs_inode_buf.h
+++ b/fs/xfs/libxfs/xfs_inode_buf.h
@@ -52,12 +52,6 @@
void xfs_log_dinode_to_disk(struct xfs_log_dinode *from,
struct xfs_dinode *to);
-#if defined(DEBUG)
-void xfs_inobp_check(struct xfs_mount *, struct xfs_buf *);
-#else
-#define xfs_inobp_check(mp, bp)
-#endif /* DEBUG */
-
xfs_failaddr_t xfs_dinode_verify(struct xfs_mount *mp, xfs_ino_t ino,
struct xfs_dinode *dip);
xfs_failaddr_t xfs_inode_validate_extsize(struct xfs_mount *mp,
diff --git a/fs/xfs/libxfs/xfs_inode_fork.c b/fs/xfs/libxfs/xfs_inode_fork.c
index 28b36627..0cf853d 100644
--- a/fs/xfs/libxfs/xfs_inode_fork.c
+++ b/fs/xfs/libxfs/xfs_inode_fork.c
@@ -291,7 +291,7 @@
* Initialize the extent count early, as the per-format routines may
* depend on it.
*/
- ip->i_afp = kmem_zone_zalloc(xfs_ifork_zone, KM_NOFS);
+ ip->i_afp = kmem_cache_zalloc(xfs_ifork_zone, GFP_NOFS | __GFP_NOFAIL);
ip->i_afp->if_format = dip->di_aformat;
if (unlikely(ip->i_afp->if_format == 0)) /* pre IRIX 6.2 file system */
ip->i_afp->if_format = XFS_DINODE_FMT_EXTENTS;
@@ -673,8 +673,8 @@
if (ip->i_cowfp)
return;
- ip->i_cowfp = kmem_zone_zalloc(xfs_ifork_zone,
- KM_NOFS);
+ ip->i_cowfp = kmem_cache_zalloc(xfs_ifork_zone,
+ GFP_NOFS | __GFP_NOFAIL);
ip->i_cowfp->if_flags = XFS_IFEXTENTS;
ip->i_cowfp->if_format = XFS_DINODE_FMT_EXTENTS;
}
diff --git a/fs/xfs/libxfs/xfs_quota_defs.h b/fs/xfs/libxfs/xfs_quota_defs.h
index 56d9dd7..076bdc7 100644
--- a/fs/xfs/libxfs/xfs_quota_defs.h
+++ b/fs/xfs/libxfs/xfs_quota_defs.h
@@ -18,23 +18,22 @@
typedef uint64_t xfs_qcnt_t;
typedef uint16_t xfs_qwarncnt_t;
+typedef uint8_t xfs_dqtype_t;
+
+#define XFS_DQTYPE_STRINGS \
+ { XFS_DQTYPE_USER, "USER" }, \
+ { XFS_DQTYPE_PROJ, "PROJ" }, \
+ { XFS_DQTYPE_GROUP, "GROUP" }
+
/*
* flags for q_flags field in the dquot.
*/
-#define XFS_DQ_USER 0x0001 /* a user quota */
-#define XFS_DQ_PROJ 0x0002 /* project quota */
-#define XFS_DQ_GROUP 0x0004 /* a group quota */
-#define XFS_DQ_DIRTY 0x0008 /* dquot is dirty */
-#define XFS_DQ_FREEING 0x0010 /* dquot is being torn down */
+#define XFS_DQFLAG_DIRTY (1 << 0) /* dquot is dirty */
+#define XFS_DQFLAG_FREEING (1 << 1) /* dquot is being torn down */
-#define XFS_DQ_ALLTYPES (XFS_DQ_USER|XFS_DQ_PROJ|XFS_DQ_GROUP)
-
-#define XFS_DQ_FLAGS \
- { XFS_DQ_USER, "USER" }, \
- { XFS_DQ_PROJ, "PROJ" }, \
- { XFS_DQ_GROUP, "GROUP" }, \
- { XFS_DQ_DIRTY, "DIRTY" }, \
- { XFS_DQ_FREEING, "FREEING" }
+#define XFS_DQFLAG_STRINGS \
+ { XFS_DQFLAG_DIRTY, "DIRTY" }, \
+ { XFS_DQFLAG_FREEING, "FREEING" }
/*
* We have the possibility of all three quota types being active at once, and
@@ -137,11 +136,11 @@
#define XFS_QMOPT_RESBLK_MASK (XFS_QMOPT_RES_REGBLKS | XFS_QMOPT_RES_RTBLKS)
extern xfs_failaddr_t xfs_dquot_verify(struct xfs_mount *mp,
- struct xfs_disk_dquot *ddq, xfs_dqid_t id, uint type);
+ struct xfs_disk_dquot *ddq, xfs_dqid_t id);
extern xfs_failaddr_t xfs_dqblk_verify(struct xfs_mount *mp,
- struct xfs_dqblk *dqb, xfs_dqid_t id, uint type);
+ struct xfs_dqblk *dqb, xfs_dqid_t id);
extern int xfs_calc_dquots_per_chunk(unsigned int nbblks);
extern void xfs_dqblk_repair(struct xfs_mount *mp, struct xfs_dqblk *dqb,
- xfs_dqid_t id, uint type);
+ xfs_dqid_t id, xfs_dqtype_t type);
#endif /* __XFS_QUOTA_H__ */
diff --git a/fs/xfs/libxfs/xfs_refcount_btree.c b/fs/xfs/libxfs/xfs_refcount_btree.c
index 7fd6044..a6ac60a 100644
--- a/fs/xfs/libxfs/xfs_refcount_btree.c
+++ b/fs/xfs/libxfs/xfs_refcount_btree.c
@@ -37,15 +37,13 @@
{
struct xfs_buf *agbp = cur->bc_ag.agbp;
struct xfs_agf *agf = agbp->b_addr;
- xfs_agnumber_t seqno = be32_to_cpu(agf->agf_seqno);
- struct xfs_perag *pag = xfs_perag_get(cur->bc_mp, seqno);
+ struct xfs_perag *pag = agbp->b_pag;
ASSERT(ptr->s != 0);
agf->agf_refcount_root = ptr->s;
be32_add_cpu(&agf->agf_refcount_level, inc);
pag->pagf_refcount_level += inc;
- xfs_perag_put(pag);
xfs_alloc_log_agf(cur->bc_tp, agbp,
XFS_AGF_REFCOUNT_ROOT | XFS_AGF_REFCOUNT_LEVEL);
@@ -325,7 +323,7 @@
ASSERT(agno != NULLAGNUMBER);
ASSERT(agno < mp->m_sb.sb_agcount);
- cur = kmem_zone_zalloc(xfs_btree_cur_zone, KM_NOFS);
+ cur = kmem_cache_zalloc(xfs_btree_cur_zone, GFP_NOFS | __GFP_NOFAIL);
cur->bc_tp = tp;
cur->bc_mp = mp;
cur->bc_btnum = XFS_BTNUM_REFC;
diff --git a/fs/xfs/libxfs/xfs_rmap_btree.c b/fs/xfs/libxfs/xfs_rmap_btree.c
index b7c0531..beb81c8 100644
--- a/fs/xfs/libxfs/xfs_rmap_btree.c
+++ b/fs/xfs/libxfs/xfs_rmap_btree.c
@@ -63,16 +63,14 @@
{
struct xfs_buf *agbp = cur->bc_ag.agbp;
struct xfs_agf *agf = agbp->b_addr;
- xfs_agnumber_t seqno = be32_to_cpu(agf->agf_seqno);
int btnum = cur->bc_btnum;
- struct xfs_perag *pag = xfs_perag_get(cur->bc_mp, seqno);
+ struct xfs_perag *pag = agbp->b_pag;
ASSERT(ptr->s != 0);
agf->agf_roots[btnum] = ptr->s;
be32_add_cpu(&agf->agf_levels[btnum], inc);
pag->pagf_levels[btnum] += inc;
- xfs_perag_put(pag);
xfs_alloc_log_agf(cur->bc_tp, agbp, XFS_AGF_ROOTS | XFS_AGF_LEVELS);
}
@@ -123,6 +121,7 @@
{
struct xfs_buf *agbp = cur->bc_ag.agbp;
struct xfs_agf *agf = agbp->b_addr;
+ struct xfs_perag *pag;
xfs_agblock_t bno;
int error;
@@ -139,8 +138,8 @@
XFS_EXTENT_BUSY_SKIP_DISCARD);
xfs_trans_agbtree_delta(cur->bc_tp, -1);
- xfs_ag_resv_rmapbt_free(cur->bc_mp, cur->bc_ag.agno);
-
+ pag = cur->bc_ag.agbp->b_pag;
+ xfs_ag_resv_free_extent(pag, XFS_AG_RESV_RMAPBT, NULL, 1);
return 0;
}
@@ -457,7 +456,7 @@
{
struct xfs_btree_cur *cur;
- cur = kmem_zone_zalloc(xfs_btree_cur_zone, KM_NOFS);
+ cur = kmem_cache_zalloc(xfs_btree_cur_zone, GFP_NOFS | __GFP_NOFAIL);
cur->bc_tp = tp;
cur->bc_mp = mp;
/* Overlapping btree; 2 keys per pointer. */
diff --git a/fs/xfs/libxfs/xfs_rtbitmap.c b/fs/xfs/libxfs/xfs_rtbitmap.c
index 9498ced..1d9fa8a3 100644
--- a/fs/xfs/libxfs/xfs_rtbitmap.c
+++ b/fs/xfs/libxfs/xfs_rtbitmap.c
@@ -70,7 +70,7 @@
if (error)
return error;
- if (XFS_IS_CORRUPT(mp, nmap == 0 || !xfs_bmap_is_real_extent(&map)))
+ if (XFS_IS_CORRUPT(mp, nmap == 0 || !xfs_bmap_is_written_extent(&map)))
return -EFSCORRUPTED;
ASSERT(map.br_startblock != NULLFSBLOCK);
diff --git a/fs/xfs/libxfs/xfs_shared.h b/fs/xfs/libxfs/xfs_shared.h
index c45acbd..708feb8 100644
--- a/fs/xfs/libxfs/xfs_shared.h
+++ b/fs/xfs/libxfs/xfs_shared.h
@@ -65,6 +65,7 @@
#define XFS_TRANS_DQ_DIRTY 0x10 /* at least one dquot in trx dirty */
#define XFS_TRANS_RESERVE 0x20 /* OK to use reserved data blocks */
#define XFS_TRANS_NO_WRITECOUNT 0x40 /* do not elevate SB writecount */
+#define XFS_TRANS_RES_FDBLKS 0x80 /* reserve newly freed blocks */
/*
* LOWMODE is used by the allocator to activate the lowspace algorithm - when
* free space is running low the extent allocator may choose to allocate an
diff --git a/fs/xfs/libxfs/xfs_trans_inode.c b/fs/xfs/libxfs/xfs_trans_inode.c
index b5dfb66..e151296 100644
--- a/fs/xfs/libxfs/xfs_trans_inode.c
+++ b/fs/xfs/libxfs/xfs_trans_inode.c
@@ -8,6 +8,8 @@
#include "xfs_shared.h"
#include "xfs_format.h"
#include "xfs_log_format.h"
+#include "xfs_trans_resv.h"
+#include "xfs_mount.h"
#include "xfs_inode.h"
#include "xfs_trans.h"
#include "xfs_trans_priv.h"
@@ -36,6 +38,7 @@
ASSERT(iip->ili_lock_flags == 0);
iip->ili_lock_flags = lock_flags;
+ ASSERT(!xfs_iflags_test(ip, XFS_ISTALE));
/*
* Get a log_item_desc to point at the new item.
@@ -71,24 +74,35 @@
}
/*
- * This is called to mark the fields indicated in fieldmask as needing
- * to be logged when the transaction is committed. The inode must
- * already be associated with the given transaction.
+ * This is called to mark the fields indicated in fieldmask as needing to be
+ * logged when the transaction is committed. The inode must already be
+ * associated with the given transaction.
*
- * The values for fieldmask are defined in xfs_inode_item.h. We always
- * log all of the core inode if any of it has changed, and we always log
- * all of the inline data/extents/b-tree root if any of them has changed.
+ * The values for fieldmask are defined in xfs_inode_item.h. We always log all
+ * of the core inode if any of it has changed, and we always log all of the
+ * inline data/extents/b-tree root if any of them has changed.
+ *
+ * Grab and pin the cluster buffer associated with this inode to avoid RMW
+ * cycles at inode writeback time. Avoid the need to add error handling to every
+ * xfs_trans_log_inode() call by shutting down on read error. This will cause
+ * transactions to fail and everything to error out, just like if we return a
+ * read error in a dirty transaction and cancel it.
*/
void
xfs_trans_log_inode(
- xfs_trans_t *tp,
- xfs_inode_t *ip,
- uint flags)
+ struct xfs_trans *tp,
+ struct xfs_inode *ip,
+ uint flags)
{
- struct inode *inode = VFS_I(ip);
+ struct xfs_inode_log_item *iip = ip->i_itemp;
+ struct inode *inode = VFS_I(ip);
+ uint iversion_flags = 0;
- ASSERT(ip->i_itemp != NULL);
+ ASSERT(iip);
ASSERT(xfs_isilocked(ip, XFS_ILOCK_EXCL));
+ ASSERT(!xfs_iflags_test(ip, XFS_ISTALE));
+
+ tp->t_flags |= XFS_TRANS_DIRTY;
/*
* Don't bother with i_lock for the I_DIRTY_TIME check here, as races
@@ -103,15 +117,6 @@
}
/*
- * Record the specific change for fdatasync optimisation. This
- * allows fdatasync to skip log forces for inodes that are only
- * timestamp dirty. We do this before the change count so that
- * the core being logged in this case does not impact on fdatasync
- * behaviour.
- */
- ip->i_itemp->ili_fsync_fields |= flags;
-
- /*
* First time we log the inode in a transaction, bump the inode change
* counter if it is configured for this to occur. While we have the
* inode locked exclusively for metadata modification, we can usually
@@ -120,23 +125,64 @@
* set however, then go ahead and bump the i_version counter
* unconditionally.
*/
- if (!test_and_set_bit(XFS_LI_DIRTY, &ip->i_itemp->ili_item.li_flags) &&
- IS_I_VERSION(VFS_I(ip))) {
- if (inode_maybe_inc_iversion(VFS_I(ip), flags & XFS_ILOG_CORE))
- flags |= XFS_ILOG_CORE;
+ if (!test_and_set_bit(XFS_LI_DIRTY, &iip->ili_item.li_flags)) {
+ if (IS_I_VERSION(inode) &&
+ inode_maybe_inc_iversion(inode, flags & XFS_ILOG_CORE))
+ iversion_flags = XFS_ILOG_CORE;
}
- tp->t_flags |= XFS_TRANS_DIRTY;
+ /*
+ * Record the specific change for fdatasync optimisation. This allows
+ * fdatasync to skip log forces for inodes that are only timestamp
+ * dirty.
+ */
+ spin_lock(&iip->ili_lock);
+ iip->ili_fsync_fields |= flags;
+
+ if (!iip->ili_item.li_buf) {
+ struct xfs_buf *bp;
+ int error;
+
+ /*
+ * We hold the ILOCK here, so this inode is not going to be
+ * flushed while we are here. Further, because there is no
+ * buffer attached to the item, we know that there is no IO in
+ * progress, so nothing will clear the ili_fields while we read
+ * in the buffer. Hence we can safely drop the spin lock and
+ * read the buffer knowing that the state will not change from
+ * here.
+ */
+ spin_unlock(&iip->ili_lock);
+ error = xfs_imap_to_bp(ip->i_mount, tp, &ip->i_imap, NULL,
+ &bp, 0);
+ if (error) {
+ xfs_force_shutdown(ip->i_mount, SHUTDOWN_META_IO_ERROR);
+ return;
+ }
+
+ /*
+ * We need an explicit buffer reference for the log item but
+ * don't want the buffer to remain attached to the transaction.
+ * Hold the buffer but release the transaction reference once
+ * we've attached the inode log item to the buffer log item
+ * list.
+ */
+ xfs_buf_hold(bp);
+ spin_lock(&iip->ili_lock);
+ iip->ili_item.li_buf = bp;
+ bp->b_flags |= _XBF_INODES;
+ list_add_tail(&iip->ili_item.li_bio_list, &bp->b_li_list);
+ xfs_trans_brelse(tp, bp);
+ }
/*
- * Always OR in the bits from the ili_last_fields field.
- * This is to coordinate with the xfs_iflush() and xfs_iflush_done()
- * routines in the eventual clearing of the ili_fields bits.
- * See the big comment in xfs_iflush() for an explanation of
- * this coordination mechanism.
+ * Always OR in the bits from the ili_last_fields field. This is to
+ * coordinate with the xfs_iflush() and xfs_iflush_done() routines in
+ * the eventual clearing of the ili_fields bits. See the big comment in
+ * xfs_iflush() for an explanation of this coordination mechanism.
*/
- flags |= ip->i_itemp->ili_last_fields;
- ip->i_itemp->ili_fields |= flags;
+ iip->ili_fields |= (flags | iip->ili_last_fields | iversion_flags);
+ spin_unlock(&iip->ili_lock);
}
int
diff --git a/fs/xfs/libxfs/xfs_trans_space.h b/fs/xfs/libxfs/xfs_trans_space.h
index 88221c7..c6df01a 100644
--- a/fs/xfs/libxfs/xfs_trans_space.h
+++ b/fs/xfs/libxfs/xfs_trans_space.h
@@ -57,7 +57,7 @@
XFS_DAREMOVE_SPACE_RES(mp, XFS_DATA_FORK)
#define XFS_IALLOC_SPACE_RES(mp) \
(M_IGEO(mp)->ialloc_blks + \
- (xfs_sb_version_hasfinobt(&mp->m_sb) ? 2 : 1 * \
+ ((xfs_sb_version_hasfinobt(&mp->m_sb) ? 2 : 1) * \
(M_IGEO(mp)->inobt_maxlevels - 1)))
/*
diff --git a/fs/xfs/scrub/bmap.c b/fs/xfs/scrub/bmap.c
index 7badd6d..955302e 100644
--- a/fs/xfs/scrub/bmap.c
+++ b/fs/xfs/scrub/bmap.c
@@ -45,9 +45,27 @@
*/
if (S_ISREG(VFS_I(sc->ip)->i_mode) &&
sc->sm->sm_type == XFS_SCRUB_TYPE_BMBTD) {
+ struct address_space *mapping = VFS_I(sc->ip)->i_mapping;
+
inode_dio_wait(VFS_I(sc->ip));
- error = filemap_write_and_wait(VFS_I(sc->ip)->i_mapping);
- if (error)
+
+ /*
+ * Try to flush all incore state to disk before we examine the
+ * space mappings for the data fork. Leave accumulated errors
+ * in the mapping for the writer threads to consume.
+ *
+ * On ENOSPC or EIO writeback errors, we continue into the
+ * extent mapping checks because write failures do not
+ * necessarily imply anything about the correctness of the file
+ * metadata. The metadata and the file data could be on
+ * completely separate devices; a media failure might only
+ * affect a subset of the disk, etc. We can handle delalloc
+ * extents in the scrubber, so leaving them in memory is fine.
+ */
+ error = filemap_fdatawrite(mapping);
+ if (!error)
+ error = filemap_fdatawait_keep_errors(mapping);
+ if (error && (error != -ENOSPC && error != -EIO))
goto out;
}
diff --git a/fs/xfs/scrub/dabtree.c b/fs/xfs/scrub/dabtree.c
index 44b1501..e56786f 100644
--- a/fs/xfs/scrub/dabtree.c
+++ b/fs/xfs/scrub/dabtree.c
@@ -476,9 +476,7 @@
ds.dargs.whichfork = whichfork;
ds.dargs.trans = sc->tp;
ds.dargs.op_flags = XFS_DA_OP_OKNOENT;
- ds.state = xfs_da_state_alloc();
- ds.state->args = &ds.dargs;
- ds.state->mp = mp;
+ ds.state = xfs_da_state_alloc(&ds.dargs);
ds.sc = sc;
ds.private = private;
if (whichfork == XFS_ATTR_FORK) {
diff --git a/fs/xfs/scrub/quota.c b/fs/xfs/scrub/quota.c
index 905a345..e34ca20 100644
--- a/fs/xfs/scrub/quota.c
+++ b/fs/xfs/scrub/quota.c
@@ -18,17 +18,17 @@
#include "scrub/common.h"
/* Convert a scrub type code to a DQ flag, or return 0 if error. */
-static inline uint
+static inline xfs_dqtype_t
xchk_quota_to_dqtype(
struct xfs_scrub *sc)
{
switch (sc->sm->sm_type) {
case XFS_SCRUB_TYPE_UQUOTA:
- return XFS_DQ_USER;
+ return XFS_DQTYPE_USER;
case XFS_SCRUB_TYPE_GQUOTA:
- return XFS_DQ_GROUP;
+ return XFS_DQTYPE_GROUP;
case XFS_SCRUB_TYPE_PQUOTA:
- return XFS_DQ_PROJ;
+ return XFS_DQTYPE_PROJ;
default:
return 0;
}
@@ -40,7 +40,7 @@
struct xfs_scrub *sc,
struct xfs_inode *ip)
{
- uint dqtype;
+ xfs_dqtype_t dqtype;
int error;
if (!XFS_IS_QUOTA_RUNNING(sc->mp) || !XFS_IS_QUOTA_ON(sc->mp))
@@ -73,26 +73,15 @@
STATIC int
xchk_quota_item(
struct xfs_dquot *dq,
- uint dqtype,
+ xfs_dqtype_t dqtype,
void *priv)
{
struct xchk_quota_info *sqi = priv;
struct xfs_scrub *sc = sqi->sc;
struct xfs_mount *mp = sc->mp;
- struct xfs_disk_dquot *d = &dq->q_core;
struct xfs_quotainfo *qi = mp->m_quotainfo;
xfs_fileoff_t offset;
- unsigned long long bsoft;
- unsigned long long isoft;
- unsigned long long rsoft;
- unsigned long long bhard;
- unsigned long long ihard;
- unsigned long long rhard;
- unsigned long long bcount;
- unsigned long long icount;
- unsigned long long rcount;
xfs_ino_t fs_icount;
- xfs_dqid_t id = be32_to_cpu(d->d_id);
int error = 0;
if (xchk_should_terminate(sc, &error))
@@ -102,27 +91,11 @@
* Except for the root dquot, the actual dquot we got must either have
* the same or higher id as we saw before.
*/
- offset = id / qi->qi_dqperchunk;
- if (id && id <= sqi->last_id)
+ offset = dq->q_id / qi->qi_dqperchunk;
+ if (dq->q_id && dq->q_id <= sqi->last_id)
xchk_fblock_set_corrupt(sc, XFS_DATA_FORK, offset);
- sqi->last_id = id;
-
- /* Did we get the dquot type we wanted? */
- if (dqtype != (d->d_flags & XFS_DQ_ALLTYPES))
- xchk_fblock_set_corrupt(sc, XFS_DATA_FORK, offset);
-
- if (d->d_pad0 != cpu_to_be32(0) || d->d_pad != cpu_to_be16(0))
- xchk_fblock_set_corrupt(sc, XFS_DATA_FORK, offset);
-
- /* Check the limits. */
- bhard = be64_to_cpu(d->d_blk_hardlimit);
- ihard = be64_to_cpu(d->d_ino_hardlimit);
- rhard = be64_to_cpu(d->d_rtb_hardlimit);
-
- bsoft = be64_to_cpu(d->d_blk_softlimit);
- isoft = be64_to_cpu(d->d_ino_softlimit);
- rsoft = be64_to_cpu(d->d_rtb_softlimit);
+ sqi->last_id = dq->q_id;
/*
* Warn if the hard limits are larger than the fs.
@@ -132,25 +105,22 @@
* Complain about corruption if the soft limit is greater than
* the hard limit.
*/
- if (bhard > mp->m_sb.sb_dblocks)
+ if (dq->q_blk.hardlimit > mp->m_sb.sb_dblocks)
xchk_fblock_set_warning(sc, XFS_DATA_FORK, offset);
- if (bsoft > bhard)
+ if (dq->q_blk.softlimit > dq->q_blk.hardlimit)
xchk_fblock_set_corrupt(sc, XFS_DATA_FORK, offset);
- if (ihard > M_IGEO(mp)->maxicount)
+ if (dq->q_ino.hardlimit > M_IGEO(mp)->maxicount)
xchk_fblock_set_warning(sc, XFS_DATA_FORK, offset);
- if (isoft > ihard)
+ if (dq->q_ino.softlimit > dq->q_ino.hardlimit)
xchk_fblock_set_corrupt(sc, XFS_DATA_FORK, offset);
- if (rhard > mp->m_sb.sb_rblocks)
+ if (dq->q_rtb.hardlimit > mp->m_sb.sb_rblocks)
xchk_fblock_set_warning(sc, XFS_DATA_FORK, offset);
- if (rsoft > rhard)
+ if (dq->q_rtb.softlimit > dq->q_rtb.hardlimit)
xchk_fblock_set_corrupt(sc, XFS_DATA_FORK, offset);
/* Check the resource counts. */
- bcount = be64_to_cpu(d->d_bcount);
- icount = be64_to_cpu(d->d_icount);
- rcount = be64_to_cpu(d->d_rtbcount);
fs_icount = percpu_counter_sum(&mp->m_icount);
/*
@@ -159,15 +129,15 @@
* if there are no quota limits.
*/
if (xfs_sb_version_hasreflink(&mp->m_sb)) {
- if (mp->m_sb.sb_dblocks < bcount)
+ if (mp->m_sb.sb_dblocks < dq->q_blk.count)
xchk_fblock_set_warning(sc, XFS_DATA_FORK,
offset);
} else {
- if (mp->m_sb.sb_dblocks < bcount)
+ if (mp->m_sb.sb_dblocks < dq->q_blk.count)
xchk_fblock_set_corrupt(sc, XFS_DATA_FORK,
offset);
}
- if (icount > fs_icount || rcount > mp->m_sb.sb_rblocks)
+ if (dq->q_ino.count > fs_icount || dq->q_rtb.count > mp->m_sb.sb_rblocks)
xchk_fblock_set_corrupt(sc, XFS_DATA_FORK, offset);
/*
@@ -175,13 +145,22 @@
* lower limit than the actual usage. However, we flag it for
* admin review.
*/
- if (id != 0 && bhard != 0 && bcount > bhard)
- xchk_fblock_set_warning(sc, XFS_DATA_FORK, offset);
- if (id != 0 && ihard != 0 && icount > ihard)
- xchk_fblock_set_warning(sc, XFS_DATA_FORK, offset);
- if (id != 0 && rhard != 0 && rcount > rhard)
+ if (dq->q_id == 0)
+ goto out;
+
+ if (dq->q_blk.hardlimit != 0 &&
+ dq->q_blk.count > dq->q_blk.hardlimit)
xchk_fblock_set_warning(sc, XFS_DATA_FORK, offset);
+ if (dq->q_ino.hardlimit != 0 &&
+ dq->q_ino.count > dq->q_ino.hardlimit)
+ xchk_fblock_set_warning(sc, XFS_DATA_FORK, offset);
+
+ if (dq->q_rtb.hardlimit != 0 &&
+ dq->q_rtb.count > dq->q_rtb.hardlimit)
+ xchk_fblock_set_warning(sc, XFS_DATA_FORK, offset);
+
+out:
if (sc->sm->sm_flags & XFS_SCRUB_OFLAG_CORRUPT)
return -EFSCORRUPTED;
@@ -235,7 +214,7 @@
struct xchk_quota_info sqi;
struct xfs_mount *mp = sc->mp;
struct xfs_quotainfo *qi = mp->m_quotainfo;
- uint dqtype;
+ xfs_dqtype_t dqtype;
int error = 0;
dqtype = xchk_quota_to_dqtype(sc);
diff --git a/fs/xfs/scrub/repair.c b/fs/xfs/scrub/repair.c
index db3cfd1..25e86c7 100644
--- a/fs/xfs/scrub/repair.c
+++ b/fs/xfs/scrub/repair.c
@@ -899,11 +899,11 @@
void
xrep_force_quotacheck(
struct xfs_scrub *sc,
- uint dqtype)
+ xfs_dqtype_t type)
{
uint flag;
- flag = xfs_quota_chkd_flag(dqtype);
+ flag = xfs_quota_chkd_flag(type);
if (!(flag & sc->mp->m_qflags))
return;
@@ -939,11 +939,11 @@
"inode %llu repair encountered quota error %d, quotacheck forced.",
(unsigned long long)sc->ip->i_ino, error);
if (XFS_IS_UQUOTA_ON(sc->mp) && !sc->ip->i_udquot)
- xrep_force_quotacheck(sc, XFS_DQ_USER);
+ xrep_force_quotacheck(sc, XFS_DQTYPE_USER);
if (XFS_IS_GQUOTA_ON(sc->mp) && !sc->ip->i_gdquot)
- xrep_force_quotacheck(sc, XFS_DQ_GROUP);
+ xrep_force_quotacheck(sc, XFS_DQTYPE_GROUP);
if (XFS_IS_PQUOTA_ON(sc->mp) && !sc->ip->i_pdquot)
- xrep_force_quotacheck(sc, XFS_DQ_PROJ);
+ xrep_force_quotacheck(sc, XFS_DQTYPE_PROJ);
/* fall through */
case -ESRCH:
error = 0;
diff --git a/fs/xfs/scrub/repair.h b/fs/xfs/scrub/repair.h
index 04a47d4..fe77de0 100644
--- a/fs/xfs/scrub/repair.h
+++ b/fs/xfs/scrub/repair.h
@@ -6,6 +6,8 @@
#ifndef __XFS_SCRUB_REPAIR_H__
#define __XFS_SCRUB_REPAIR_H__
+#include "xfs_quota_defs.h"
+
static inline int xrep_notsupported(struct xfs_scrub *sc)
{
return -EOPNOTSUPP;
@@ -49,7 +51,7 @@
int xrep_find_ag_btree_roots(struct xfs_scrub *sc, struct xfs_buf *agf_bp,
struct xrep_find_ag_btree *btree_info, struct xfs_buf *agfl_bp);
-void xrep_force_quotacheck(struct xfs_scrub *sc, uint dqtype);
+void xrep_force_quotacheck(struct xfs_scrub *sc, xfs_dqtype_t type);
int xrep_ino_dqattach(struct xfs_scrub *sc);
/* Metadata repairers */
diff --git a/fs/xfs/scrub/rtbitmap.c b/fs/xfs/scrub/rtbitmap.c
index c642bc2..76e4ffe 100644
--- a/fs/xfs/scrub/rtbitmap.c
+++ b/fs/xfs/scrub/rtbitmap.c
@@ -13,6 +13,7 @@
#include "xfs_trans.h"
#include "xfs_rtalloc.h"
#include "xfs_inode.h"
+#include "xfs_bmap.h"
#include "scrub/scrub.h"
#include "scrub/common.h"
@@ -58,6 +59,41 @@
return 0;
}
+/* Make sure the entire rtbitmap file is mapped with written extents. */
+STATIC int
+xchk_rtbitmap_check_extents(
+ struct xfs_scrub *sc)
+{
+ struct xfs_mount *mp = sc->mp;
+ struct xfs_bmbt_irec map;
+ xfs_rtblock_t off;
+ int nmap;
+ int error = 0;
+
+ for (off = 0; off < mp->m_sb.sb_rbmblocks;) {
+ if (xchk_should_terminate(sc, &error) ||
+ (sc->sm->sm_flags & XFS_SCRUB_OFLAG_CORRUPT))
+ break;
+
+ /* Make sure we have a written extent. */
+ nmap = 1;
+ error = xfs_bmapi_read(mp->m_rbmip, off,
+ mp->m_sb.sb_rbmblocks - off, &map, &nmap,
+ XFS_DATA_FORK);
+ if (!xchk_fblock_process_error(sc, XFS_DATA_FORK, off, &error))
+ break;
+
+ if (nmap != 1 || !xfs_bmap_is_written_extent(&map)) {
+ xchk_fblock_set_corrupt(sc, XFS_DATA_FORK, off);
+ break;
+ }
+
+ off += map.br_blockcount;
+ }
+
+ return error;
+}
+
/* Scrub the realtime bitmap. */
int
xchk_rtbitmap(
@@ -65,11 +101,22 @@
{
int error;
+ /* Is the size of the rtbitmap correct? */
+ if (sc->mp->m_rbmip->i_d.di_size !=
+ XFS_FSB_TO_B(sc->mp, sc->mp->m_sb.sb_rbmblocks)) {
+ xchk_ino_set_corrupt(sc, sc->mp->m_rbmip->i_ino);
+ return 0;
+ }
+
/* Invoke the fork scrubber. */
error = xchk_metadata_inode_forks(sc);
if (error || (sc->sm->sm_flags & XFS_SCRUB_OFLAG_CORRUPT))
return error;
+ error = xchk_rtbitmap_check_extents(sc);
+ if (error || (sc->sm->sm_flags & XFS_SCRUB_OFLAG_CORRUPT))
+ return error;
+
error = xfs_rtalloc_query_all(sc->tp, xchk_rtbitmap_rec, sc);
if (!xchk_fblock_process_error(sc, XFS_DATA_FORK, 0, &error))
goto out;
diff --git a/fs/xfs/xfs_bmap_item.c b/fs/xfs/xfs_bmap_item.c
index 6736c5a..ec36913 100644
--- a/fs/xfs/xfs_bmap_item.c
+++ b/fs/xfs/xfs_bmap_item.c
@@ -138,7 +138,7 @@
{
struct xfs_bui_log_item *buip;
- buip = kmem_zone_zalloc(xfs_bui_zone, 0);
+ buip = kmem_cache_zalloc(xfs_bui_zone, GFP_KERNEL | __GFP_NOFAIL);
xfs_log_item_init(mp, &buip->bui_item, XFS_LI_BUI, &xfs_bui_item_ops);
buip->bui_format.bui_nextents = XFS_BUI_MAX_FAST_EXTENTS;
@@ -215,7 +215,7 @@
{
struct xfs_bud_log_item *budp;
- budp = kmem_zone_zalloc(xfs_bud_zone, 0);
+ budp = kmem_cache_zalloc(xfs_bud_zone, GFP_KERNEL | __GFP_NOFAIL);
xfs_log_item_init(tp->t_mountp, &budp->bud_item, XFS_LI_BUD,
&xfs_bud_item_ops);
budp->bud_buip = buip;
diff --git a/fs/xfs/xfs_bmap_util.c b/fs/xfs/xfs_bmap_util.c
index 3052586..73cafc8 100644
--- a/fs/xfs/xfs_bmap_util.c
+++ b/fs/xfs/xfs_bmap_util.c
@@ -1567,6 +1567,7 @@
int lock_flags;
uint64_t f;
int resblks = 0;
+ unsigned int flags = 0;
/*
* Lock the inodes against other IO, page faults and truncate to
@@ -1630,17 +1631,16 @@
resblks += XFS_SWAP_RMAP_SPACE_RES(mp, tipnext, w);
/*
- * Handle the corner case where either inode might straddle the
- * btree format boundary. If so, the inode could bounce between
- * btree <-> extent format on unmap -> remap cycles, freeing and
- * allocating a bmapbt block each time.
+ * If either inode straddles a bmapbt block allocation boundary,
+ * the rmapbt algorithm triggers repeated allocs and frees as
+ * extents are remapped. This can exhaust the block reservation
+ * prematurely and cause shutdown. Return freed blocks to the
+ * transaction reservation to counter this behavior.
*/
- if (ipnext == (XFS_IFORK_MAXEXT(ip, w) + 1))
- resblks += XFS_IFORK_MAXEXT(ip, w);
- if (tipnext == (XFS_IFORK_MAXEXT(tip, w) + 1))
- resblks += XFS_IFORK_MAXEXT(tip, w);
+ flags |= XFS_TRANS_RES_FDBLKS;
}
- error = xfs_trans_alloc(mp, &M_RES(mp)->tr_write, resblks, 0, 0, &tp);
+ error = xfs_trans_alloc(mp, &M_RES(mp)->tr_write, resblks, 0, flags,
+ &tp);
if (error)
goto out_unlock;
diff --git a/fs/xfs/xfs_buf.c b/fs/xfs/xfs_buf.c
index 20b748f..d4cdcb6 100644
--- a/fs/xfs/xfs_buf.c
+++ b/fs/xfs/xfs_buf.c
@@ -14,6 +14,9 @@
#include "xfs_mount.h"
#include "xfs_trace.h"
#include "xfs_log.h"
+#include "xfs_log_recover.h"
+#include "xfs_trans.h"
+#include "xfs_buf_item.h"
#include "xfs_errortag.h"
#include "xfs_error.h"
@@ -211,9 +214,7 @@
int i;
*bpp = NULL;
- bp = kmem_zone_zalloc(xfs_buf_zone, KM_NOFS);
- if (unlikely(!bp))
- return -ENOMEM;
+ bp = kmem_cache_zalloc(xfs_buf_zone, GFP_NOFS | __GFP_NOFAIL);
/*
* We don't want certain flags to appear in b_flags unless they are
@@ -655,7 +656,6 @@
*/
if (bp->b_flags & XBF_STALE) {
ASSERT((bp->b_flags & _XBF_DELWRI_Q) == 0);
- ASSERT(bp->b_iodone == NULL);
bp->b_flags &= _XBF_KMEM | _XBF_PAGES;
bp->b_ops = NULL;
}
@@ -1191,10 +1191,13 @@
if (!bp->b_error && bp->b_io_error)
xfs_buf_ioerror(bp, bp->b_io_error);
- /* Only validate buffers that were read without errors */
- if (read && !bp->b_error && bp->b_ops) {
- ASSERT(!bp->b_iodone);
- bp->b_ops->verify_read(bp);
+ if (read) {
+ if (!bp->b_error && bp->b_ops)
+ bp->b_ops->verify_read(bp);
+ if (!bp->b_error)
+ bp->b_flags |= XBF_DONE;
+ xfs_buf_ioend_finish(bp);
+ return;
}
if (!bp->b_error) {
@@ -1202,12 +1205,25 @@
bp->b_flags |= XBF_DONE;
}
- if (bp->b_iodone)
- (*(bp->b_iodone))(bp);
- else if (bp->b_flags & XBF_ASYNC)
- xfs_buf_relse(bp);
- else
- complete(&bp->b_iowait);
+ /*
+ * If this is a log recovery buffer, we aren't doing transactional IO
+ * yet so we need to let it handle IO completions.
+ */
+ if (bp->b_flags & _XBF_LOGRECOVERY) {
+ xlog_recover_iodone(bp);
+ return;
+ }
+
+ if (bp->b_flags & _XBF_INODES) {
+ xfs_buf_inode_iodone(bp);
+ return;
+ }
+
+ if (bp->b_flags & _XBF_DQUOTS) {
+ xfs_buf_dquot_iodone(bp);
+ return;
+ }
+ xfs_buf_iodone(bp);
}
static void
diff --git a/fs/xfs/xfs_buf.h b/fs/xfs/xfs_buf.h
index 050c53b..755b652 100644
--- a/fs/xfs/xfs_buf.h
+++ b/fs/xfs/xfs_buf.h
@@ -18,6 +18,7 @@
/*
* Base types
*/
+struct xfs_buf;
#define XFS_BUF_DADDR_NULL ((xfs_daddr_t) (-1LL))
@@ -30,15 +31,20 @@
#define XBF_STALE (1 << 6) /* buffer has been staled, do not find it */
#define XBF_WRITE_FAIL (1 << 7) /* async writes have failed on this buffer */
-/* flags used only as arguments to access routines */
-#define XBF_TRYLOCK (1 << 16)/* lock requested, but do not wait */
-#define XBF_UNMAPPED (1 << 17)/* do not map the buffer */
+/* buffer type flags for write callbacks */
+#define _XBF_INODES (1 << 16)/* inode buffer */
+#define _XBF_DQUOTS (1 << 17)/* dquot buffer */
+#define _XBF_LOGRECOVERY (1 << 18)/* log recovery buffer */
/* flags used only internally */
#define _XBF_PAGES (1 << 20)/* backed by refcounted pages */
#define _XBF_KMEM (1 << 21)/* backed by heap memory */
#define _XBF_DELWRI_Q (1 << 22)/* buffer on a delwri queue */
+/* flags used only as arguments to access routines */
+#define XBF_TRYLOCK (1 << 30)/* lock requested, but do not wait */
+#define XBF_UNMAPPED (1 << 31)/* do not map the buffer */
+
typedef unsigned int xfs_buf_flags_t;
#define XFS_BUF_FLAGS \
@@ -50,12 +56,15 @@
{ XBF_DONE, "DONE" }, \
{ XBF_STALE, "STALE" }, \
{ XBF_WRITE_FAIL, "WRITE_FAIL" }, \
- { XBF_TRYLOCK, "TRYLOCK" }, /* should never be set */\
- { XBF_UNMAPPED, "UNMAPPED" }, /* ditto */\
+ { _XBF_INODES, "INODES" }, \
+ { _XBF_DQUOTS, "DQUOTS" }, \
+ { _XBF_LOGRECOVERY, "LOG_RECOVERY" }, \
{ _XBF_PAGES, "PAGES" }, \
{ _XBF_KMEM, "KMEM" }, \
- { _XBF_DELWRI_Q, "DELWRI_Q" }
-
+ { _XBF_DELWRI_Q, "DELWRI_Q" }, \
+ /* The following interface flags should never be set */ \
+ { XBF_TRYLOCK, "TRYLOCK" }, \
+ { XBF_UNMAPPED, "UNMAPPED" }
/*
* Internal state flags.
@@ -94,10 +103,6 @@
struct ratelimit_state bt_ioerror_rl;
} xfs_buftarg_t;
-struct xfs_buf;
-typedef void (*xfs_buf_iodone_t)(struct xfs_buf *);
-
-
#define XB_PAGES 2
struct xfs_buf_map {
@@ -150,7 +155,6 @@
xfs_buftarg_t *b_target; /* buffer target (device) */
void *b_addr; /* virtual address of buffer */
struct work_struct b_ioend_work;
- xfs_buf_iodone_t b_iodone; /* I/O completion function */
struct completion b_iowait; /* queue for I/O waiters */
struct xfs_buf_log_item *b_log_item;
struct list_head b_li_list; /* Log items list head */
@@ -257,9 +261,23 @@
#define xfs_buf_islocked(bp) \
((bp)->b_sema.count <= 0)
+static inline void xfs_buf_relse(xfs_buf_t *bp)
+{
+ xfs_buf_unlock(bp);
+ xfs_buf_rele(bp);
+}
+
/* Buffer Read and Write Routines */
extern int xfs_bwrite(struct xfs_buf *bp);
extern void xfs_buf_ioend(struct xfs_buf *bp);
+static inline void xfs_buf_ioend_finish(struct xfs_buf *bp)
+{
+ if (bp->b_flags & XBF_ASYNC)
+ xfs_buf_relse(bp);
+ else
+ complete(&bp->b_iowait);
+}
+
extern void __xfs_buf_ioerror(struct xfs_buf *bp, int error,
xfs_failaddr_t failaddr);
#define xfs_buf_ioerror(bp, err) __xfs_buf_ioerror((bp), (err), __this_address)
@@ -324,12 +342,6 @@
return atomic_read(&bp->b_pin_count);
}
-static inline void xfs_buf_relse(xfs_buf_t *bp)
-{
- xfs_buf_unlock(bp);
- xfs_buf_rele(bp);
-}
-
static inline int
xfs_buf_verify_cksum(struct xfs_buf *bp, unsigned long cksum_offset)
{
diff --git a/fs/xfs/xfs_buf_item.c b/fs/xfs/xfs_buf_item.c
index 9e75e8d..5bb6f22 100644
--- a/fs/xfs/xfs_buf_item.c
+++ b/fs/xfs/xfs_buf_item.c
@@ -12,8 +12,13 @@
#include "xfs_bit.h"
#include "xfs_mount.h"
#include "xfs_trans.h"
-#include "xfs_buf_item.h"
#include "xfs_trans_priv.h"
+#include "xfs_buf_item.h"
+#include "xfs_inode.h"
+#include "xfs_inode_item.h"
+#include "xfs_quota.h"
+#include "xfs_dquot_item.h"
+#include "xfs_dquot.h"
#include "xfs_trace.h"
#include "xfs_log.h"
@@ -25,7 +30,7 @@
return container_of(lip, struct xfs_buf_log_item, bli_item);
}
-STATIC void xfs_buf_do_callbacks(struct xfs_buf *bp);
+static void xfs_buf_item_done(struct xfs_buf *bp);
/* Is this log iovec plausibly large enough to contain the buffer log format? */
bool
@@ -457,10 +462,9 @@
* the AIL lock.
*/
if (bip->bli_flags & XFS_BLI_STALE_INODE) {
- xfs_buf_do_callbacks(bp);
- bp->b_log_item = NULL;
- list_del_init(&bp->b_li_list);
- bp->b_iodone = NULL;
+ xfs_buf_item_done(bp);
+ xfs_iflush_done(bp);
+ ASSERT(list_empty(&bp->b_li_list));
} else {
xfs_trans_ail_delete(lip, SHUTDOWN_LOG_IO_ERROR);
xfs_buf_item_relse(bp);
@@ -734,7 +738,7 @@
return 0;
}
- bip = kmem_zone_zalloc(xfs_buf_item_zone, 0);
+ bip = kmem_cache_zalloc(xfs_buf_item_zone, GFP_KERNEL | __GFP_NOFAIL);
xfs_log_item_init(mp, &bip->bli_item, XFS_LI_BUF, &xfs_buf_item_ops);
bip->bli_buf = bp;
@@ -936,11 +940,7 @@
}
/*
- * This is called when the buf log item is no longer needed. It should
- * free the buf log item associated with the given buffer and clear
- * the buffer's pointer to the buf log item. If there are no more
- * items in the list, clear the b_iodone field of the buffer (see
- * xfs_buf_attach_iodone() below).
+ * xfs_buf_item_relse() is called when the buf log item is no longer needed.
*/
void
xfs_buf_item_relse(
@@ -952,137 +952,28 @@
ASSERT(!test_bit(XFS_LI_IN_AIL, &bip->bli_item.li_flags));
bp->b_log_item = NULL;
- if (list_empty(&bp->b_li_list))
- bp->b_iodone = NULL;
-
xfs_buf_rele(bp);
xfs_buf_item_free(bip);
}
-
/*
- * Add the given log item with its callback to the list of callbacks
- * to be called when the buffer's I/O completes. If it is not set
- * already, set the buffer's b_iodone() routine to be
- * xfs_buf_iodone_callbacks() and link the log item into the list of
- * items rooted at b_li_list.
+ * Decide if we're going to retry the write after a failure, and prepare
+ * the buffer for retrying the write.
*/
-void
-xfs_buf_attach_iodone(
- struct xfs_buf *bp,
- void (*cb)(struct xfs_buf *, struct xfs_log_item *),
- struct xfs_log_item *lip)
-{
- ASSERT(xfs_buf_islocked(bp));
-
- lip->li_cb = cb;
- list_add_tail(&lip->li_bio_list, &bp->b_li_list);
-
- ASSERT(bp->b_iodone == NULL ||
- bp->b_iodone == xfs_buf_iodone_callbacks);
- bp->b_iodone = xfs_buf_iodone_callbacks;
-}
-
-/*
- * We can have many callbacks on a buffer. Running the callbacks individually
- * can cause a lot of contention on the AIL lock, so we allow for a single
- * callback to be able to scan the remaining items in bp->b_li_list for other
- * items of the same type and callback to be processed in the first call.
- *
- * As a result, the loop walking the callback list below will also modify the
- * list. it removes the first item from the list and then runs the callback.
- * The loop then restarts from the new first item int the list. This allows the
- * callback to scan and modify the list attached to the buffer and we don't
- * have to care about maintaining a next item pointer.
- */
-STATIC void
-xfs_buf_do_callbacks(
- struct xfs_buf *bp)
-{
- struct xfs_buf_log_item *blip = bp->b_log_item;
- struct xfs_log_item *lip;
-
- /* If there is a buf_log_item attached, run its callback */
- if (blip) {
- lip = &blip->bli_item;
- lip->li_cb(bp, lip);
- }
-
- while (!list_empty(&bp->b_li_list)) {
- lip = list_first_entry(&bp->b_li_list, struct xfs_log_item,
- li_bio_list);
-
- /*
- * Remove the item from the list, so we don't have any
- * confusion if the item is added to another buf.
- * Don't touch the log item after calling its
- * callback, because it could have freed itself.
- */
- list_del_init(&lip->li_bio_list);
- lip->li_cb(bp, lip);
- }
-}
-
-/*
- * Invoke the error state callback for each log item affected by the failed I/O.
- *
- * If a metadata buffer write fails with a non-permanent error, the buffer is
- * eventually resubmitted and so the completion callbacks are not run. The error
- * state may need to be propagated to the log items attached to the buffer,
- * however, so the next AIL push of the item knows hot to handle it correctly.
- */
-STATIC void
-xfs_buf_do_callbacks_fail(
- struct xfs_buf *bp)
-{
- struct xfs_log_item *lip;
- struct xfs_ail *ailp;
-
- /*
- * Buffer log item errors are handled directly by xfs_buf_item_push()
- * and xfs_buf_iodone_callback_error, and they have no IO error
- * callbacks. Check only for items in b_li_list.
- */
- if (list_empty(&bp->b_li_list))
- return;
-
- lip = list_first_entry(&bp->b_li_list, struct xfs_log_item,
- li_bio_list);
- ailp = lip->li_ailp;
- spin_lock(&ailp->ail_lock);
- list_for_each_entry(lip, &bp->b_li_list, li_bio_list) {
- if (lip->li_ops->iop_error)
- lip->li_ops->iop_error(lip, bp);
- }
- spin_unlock(&ailp->ail_lock);
-}
-
static bool
-xfs_buf_iodone_callback_error(
+xfs_buf_ioerror_fail_without_retry(
struct xfs_buf *bp)
{
- struct xfs_buf_log_item *bip = bp->b_log_item;
- struct xfs_log_item *lip;
- struct xfs_mount *mp;
+ struct xfs_mount *mp = bp->b_mount;
static ulong lasttime;
static xfs_buftarg_t *lasttarg;
- struct xfs_error_cfg *cfg;
-
- /*
- * The failed buffer might not have a buf_log_item attached or the
- * log_item list might be empty. Get the mp from the available
- * xfs_log_item
- */
- lip = list_first_entry_or_null(&bp->b_li_list, struct xfs_log_item,
- li_bio_list);
- mp = lip ? lip->li_mountp : bip->bli_item.li_mountp;
/*
* If we've already decided to shutdown the filesystem because of
* I/O errors, there's no point in giving this a retry.
*/
if (XFS_FORCED_SHUTDOWN(mp))
- goto out_stale;
+ return true;
if (bp->b_target != lasttarg ||
time_after(jiffies, (lasttime + 5*HZ))) {
@@ -1093,121 +984,126 @@
/* synchronous writes will have callers process the error */
if (!(bp->b_flags & XBF_ASYNC))
- goto out_stale;
-
- trace_xfs_buf_item_iodone_async(bp, _RET_IP_);
- ASSERT(bp->b_iodone != NULL);
-
- cfg = xfs_error_get_cfg(mp, XFS_ERR_METADATA, bp->b_error);
-
- /*
- * If the write was asynchronous then no one will be looking for the
- * error. If this is the first failure of this type, clear the error
- * state and write the buffer out again. This means we always retry an
- * async write failure at least once, but we also need to set the buffer
- * up to behave correctly now for repeated failures.
- */
- if (!(bp->b_flags & (XBF_STALE | XBF_WRITE_FAIL)) ||
- bp->b_last_error != bp->b_error) {
- bp->b_flags |= (XBF_WRITE | XBF_DONE | XBF_WRITE_FAIL);
- bp->b_last_error = bp->b_error;
- if (cfg->retry_timeout != XFS_ERR_RETRY_FOREVER &&
- !bp->b_first_retry_time)
- bp->b_first_retry_time = jiffies;
-
- xfs_buf_ioerror(bp, 0);
- xfs_buf_submit(bp);
return true;
- }
+ return false;
+}
- /*
- * Repeated failure on an async write. Take action according to the
- * error configuration we have been set up to use.
- */
+static bool
+xfs_buf_ioerror_retry(
+ struct xfs_buf *bp,
+ struct xfs_error_cfg *cfg)
+{
+ if ((bp->b_flags & (XBF_STALE | XBF_WRITE_FAIL)) &&
+ bp->b_last_error == bp->b_error)
+ return false;
+
+ bp->b_flags |= (XBF_WRITE | XBF_DONE | XBF_WRITE_FAIL);
+ bp->b_last_error = bp->b_error;
+ if (cfg->retry_timeout != XFS_ERR_RETRY_FOREVER &&
+ !bp->b_first_retry_time)
+ bp->b_first_retry_time = jiffies;
+ return true;
+}
+
+/*
+ * Account for this latest trip around the retry handler, and decide if
+ * we've failed enough times to constitute a permanent failure.
+ */
+static bool
+xfs_buf_ioerror_permanent(
+ struct xfs_buf *bp,
+ struct xfs_error_cfg *cfg)
+{
+ struct xfs_mount *mp = bp->b_mount;
if (cfg->max_retries != XFS_ERR_RETRY_FOREVER &&
++bp->b_retries > cfg->max_retries)
- goto permanent_error;
+ return true;
if (cfg->retry_timeout != XFS_ERR_RETRY_FOREVER &&
time_after(jiffies, cfg->retry_timeout + bp->b_first_retry_time))
- goto permanent_error;
+ return true;
/* At unmount we may treat errors differently */
if ((mp->m_flags & XFS_MOUNT_UNMOUNTING) && mp->m_fail_unmount)
- goto permanent_error;
+ return true;
- /*
- * Still a transient error, run IO completion failure callbacks and let
- * the higher layers retry the buffer.
- */
- xfs_buf_do_callbacks_fail(bp);
- xfs_buf_ioerror(bp, 0);
- xfs_buf_relse(bp);
- return true;
+ return false;
+}
+
+/*
+ * On a sync write or shutdown we just want to stale the buffer and let the
+ * caller handle the error in bp->b_error appropriately.
+ *
+ * If the write was asynchronous then no one will be looking for the error. If
+ * this is the first failure of this type, clear the error state and write the
+ * buffer out again. This means we always retry an async write failure at least
+ * once, but we also need to set the buffer up to behave correctly now for
+ * repeated failures.
+ *
+ * If we get repeated async write failures, then we take action according to the
+ * error configuration we have been set up to use.
+ *
+ * Multi-state return value:
+ *
+ * XBF_IOERROR_FINISH: clear IO error retry state and run callback completions
+ * XBF_IOERROR_DONE: resubmitted immediately, do not run any completions
+ * XBF_IOERROR_FAIL: transient error, run failure callback completions and then
+ * release the buffer
+ */
+enum {
+ XBF_IOERROR_FINISH,
+ XBF_IOERROR_DONE,
+ XBF_IOERROR_FAIL,
+};
+
+static int
+xfs_buf_iodone_error(
+ struct xfs_buf *bp)
+{
+ struct xfs_mount *mp = bp->b_mount;
+ struct xfs_error_cfg *cfg;
+
+ if (xfs_buf_ioerror_fail_without_retry(bp))
+ goto out_stale;
+
+ trace_xfs_buf_item_iodone_async(bp, _RET_IP_);
+
+ cfg = xfs_error_get_cfg(mp, XFS_ERR_METADATA, bp->b_error);
+ if (xfs_buf_ioerror_retry(bp, cfg)) {
+ xfs_buf_ioerror(bp, 0);
+ xfs_buf_submit(bp);
+ return XBF_IOERROR_DONE;
+ }
/*
* Permanent error - we need to trigger a shutdown if we haven't already
* to indicate that inconsistency will result from this action.
*/
-permanent_error:
- xfs_force_shutdown(mp, SHUTDOWN_META_IO_ERROR);
+ if (xfs_buf_ioerror_permanent(bp, cfg)) {
+ xfs_force_shutdown(mp, SHUTDOWN_META_IO_ERROR);
+ goto out_stale;
+ }
+
+ /* Still considered a transient error. Caller will schedule retries. */
+ return XBF_IOERROR_FAIL;
+
out_stale:
xfs_buf_stale(bp);
bp->b_flags |= XBF_DONE;
trace_xfs_buf_error_relse(bp, _RET_IP_);
- return false;
+ return XBF_IOERROR_FINISH;
}
-/*
- * This is the iodone() function for buffers which have had callbacks attached
- * to them by xfs_buf_attach_iodone(). We need to iterate the items on the
- * callback list, mark the buffer as having no more callbacks and then push the
- * buffer through IO completion processing.
- */
-void
-xfs_buf_iodone_callbacks(
+static void
+xfs_buf_item_done(
struct xfs_buf *bp)
{
- /*
- * If there is an error, process it. Some errors require us
- * to run callbacks after failure processing is done so we
- * detect that and take appropriate action.
- */
- if (bp->b_error && xfs_buf_iodone_callback_error(bp))
+ struct xfs_buf_log_item *bip = bp->b_log_item;
+
+ if (!bip)
return;
/*
- * Successful IO or permanent error. Either way, we can clear the
- * retry state here in preparation for the next error that may occur.
- */
- bp->b_last_error = 0;
- bp->b_retries = 0;
- bp->b_first_retry_time = 0;
-
- xfs_buf_do_callbacks(bp);
- bp->b_log_item = NULL;
- list_del_init(&bp->b_li_list);
- bp->b_iodone = NULL;
- xfs_buf_ioend(bp);
-}
-
-/*
- * This is the iodone() function for buffers which have been
- * logged. It is called when they are eventually flushed out.
- * It should remove the buf item from the AIL, and free the buf item.
- * It is called by xfs_buf_iodone_callbacks() above which will take
- * care of cleaning up the buffer itself.
- */
-void
-xfs_buf_iodone(
- struct xfs_buf *bp,
- struct xfs_log_item *lip)
-{
- ASSERT(BUF_ITEM(lip)->bli_buf == bp);
-
- xfs_buf_rele(bp);
-
- /*
* If we are forcibly shutting down, this may well be off the AIL
* already. That's because we simulate the log-committed callbacks to
* unpin these buffers. Or we may never have put this item on AIL
@@ -1216,6 +1112,112 @@
*
* Either way, AIL is useless if we're forcing a shutdown.
*/
- xfs_trans_ail_delete(lip, SHUTDOWN_CORRUPT_INCORE);
- xfs_buf_item_free(BUF_ITEM(lip));
+ xfs_trans_ail_delete(&bip->bli_item, SHUTDOWN_CORRUPT_INCORE);
+ bp->b_log_item = NULL;
+ xfs_buf_item_free(bip);
+ xfs_buf_rele(bp);
+}
+
+static inline void
+xfs_buf_clear_ioerror_retry_state(
+ struct xfs_buf *bp)
+{
+ bp->b_last_error = 0;
+ bp->b_retries = 0;
+ bp->b_first_retry_time = 0;
+}
+
+/*
+ * Inode buffer iodone callback function.
+ */
+void
+xfs_buf_inode_iodone(
+ struct xfs_buf *bp)
+{
+ if (bp->b_error) {
+ struct xfs_log_item *lip;
+ int ret = xfs_buf_iodone_error(bp);
+
+ if (ret == XBF_IOERROR_FINISH)
+ goto finish_iodone;
+ if (ret == XBF_IOERROR_DONE)
+ return;
+ ASSERT(ret == XBF_IOERROR_FAIL);
+ list_for_each_entry(lip, &bp->b_li_list, li_bio_list) {
+ set_bit(XFS_LI_FAILED, &lip->li_flags);
+ }
+ xfs_buf_ioerror(bp, 0);
+ xfs_buf_relse(bp);
+ return;
+ }
+
+finish_iodone:
+ xfs_buf_clear_ioerror_retry_state(bp);
+ xfs_buf_item_done(bp);
+ xfs_iflush_done(bp);
+ xfs_buf_ioend_finish(bp);
+}
+
+/*
+ * Dquot buffer iodone callback function.
+ */
+void
+xfs_buf_dquot_iodone(
+ struct xfs_buf *bp)
+{
+ if (bp->b_error) {
+ struct xfs_log_item *lip;
+ int ret = xfs_buf_iodone_error(bp);
+
+ if (ret == XBF_IOERROR_FINISH)
+ goto finish_iodone;
+ if (ret == XBF_IOERROR_DONE)
+ return;
+ ASSERT(ret == XBF_IOERROR_FAIL);
+ spin_lock(&bp->b_mount->m_ail->ail_lock);
+ list_for_each_entry(lip, &bp->b_li_list, li_bio_list) {
+ xfs_set_li_failed(lip, bp);
+ }
+ spin_unlock(&bp->b_mount->m_ail->ail_lock);
+ xfs_buf_ioerror(bp, 0);
+ xfs_buf_relse(bp);
+ return;
+ }
+
+finish_iodone:
+ xfs_buf_clear_ioerror_retry_state(bp);
+ /* a newly allocated dquot buffer might have a log item attached */
+ xfs_buf_item_done(bp);
+ xfs_dquot_done(bp);
+ xfs_buf_ioend_finish(bp);
+}
+
+/*
+ * Dirty buffer iodone callback function.
+ *
+ * Note that for things like remote attribute buffers, there may not be a buffer
+ * log item here, so processing the buffer log item must remain be optional.
+ */
+void
+xfs_buf_iodone(
+ struct xfs_buf *bp)
+{
+ if (bp->b_error) {
+ int ret = xfs_buf_iodone_error(bp);
+
+ if (ret == XBF_IOERROR_FINISH)
+ goto finish_iodone;
+ if (ret == XBF_IOERROR_DONE)
+ return;
+ ASSERT(ret == XBF_IOERROR_FAIL);
+ ASSERT(list_empty(&bp->b_li_list));
+ xfs_buf_ioerror(bp, 0);
+ xfs_buf_relse(bp);
+ return;
+ }
+
+finish_iodone:
+ xfs_buf_clear_ioerror_retry_state(bp);
+ xfs_buf_item_done(bp);
+ xfs_buf_ioend_finish(bp);
}
diff --git a/fs/xfs/xfs_buf_item.h b/fs/xfs/xfs_buf_item.h
index c9c57e2..23507cb 100644
--- a/fs/xfs/xfs_buf_item.h
+++ b/fs/xfs/xfs_buf_item.h
@@ -54,11 +54,9 @@
bool xfs_buf_item_put(struct xfs_buf_log_item *);
void xfs_buf_item_log(struct xfs_buf_log_item *, uint, uint);
bool xfs_buf_item_dirty_format(struct xfs_buf_log_item *);
-void xfs_buf_attach_iodone(struct xfs_buf *,
- void(*)(struct xfs_buf *, struct xfs_log_item *),
- struct xfs_log_item *);
-void xfs_buf_iodone_callbacks(struct xfs_buf *);
-void xfs_buf_iodone(struct xfs_buf *, struct xfs_log_item *);
+void xfs_buf_inode_iodone(struct xfs_buf *);
+void xfs_buf_dquot_iodone(struct xfs_buf *);
+void xfs_buf_iodone(struct xfs_buf *);
bool xfs_buf_log_check_iovec(struct xfs_log_iovec *iovec);
extern kmem_zone_t *xfs_buf_item_zone;
diff --git a/fs/xfs/xfs_buf_item_recover.c b/fs/xfs/xfs_buf_item_recover.c
index 04faa73..d480f11 100644
--- a/fs/xfs/xfs_buf_item_recover.c
+++ b/fs/xfs/xfs_buf_item_recover.c
@@ -419,8 +419,7 @@
if (bp->b_ops) {
struct xfs_buf_log_item *bip;
- ASSERT(!bp->b_iodone || bp->b_iodone == xlog_recover_iodone);
- bp->b_iodone = xlog_recover_iodone;
+ bp->b_flags |= _XBF_LOGRECOVERY;
xfs_buf_item_init(bp, mp);
bip = bp->b_log_item;
bip->bli_item.li_lsn = current_lsn;
@@ -494,8 +493,7 @@
item->ri_buf[i].i_len, __func__);
goto next;
}
- fa = xfs_dquot_verify(mp, item->ri_buf[i].i_addr,
- -1, 0);
+ fa = xfs_dquot_verify(mp, item->ri_buf[i].i_addr, -1);
if (fa) {
xfs_alert(mp,
"dquot corrupt at %pS trying to replay into block 0x%llx",
@@ -548,11 +546,11 @@
type = 0;
if (buf_f->blf_flags & XFS_BLF_UDQUOT_BUF)
- type |= XFS_DQ_USER;
+ type |= XFS_DQTYPE_USER;
if (buf_f->blf_flags & XFS_BLF_PDQUOT_BUF)
- type |= XFS_DQ_PROJ;
+ type |= XFS_DQTYPE_PROJ;
if (buf_f->blf_flags & XFS_BLF_GDQUOT_BUF)
- type |= XFS_DQ_GROUP;
+ type |= XFS_DQTYPE_GROUP;
/*
* This type of quotas was turned off, so ignore this buffer
*/
@@ -963,7 +961,7 @@
error = xfs_bwrite(bp);
} else {
ASSERT(bp->b_mount == mp);
- bp->b_iodone = xlog_recover_iodone;
+ bp->b_flags |= _XBF_LOGRECOVERY;
xfs_buf_delwri_queue(bp, buffer_list);
}
diff --git a/fs/xfs/xfs_dquot.c b/fs/xfs/xfs_dquot.c
index d5b7f03..04dc2be 100644
--- a/fs/xfs/xfs_dquot.c
+++ b/fs/xfs/xfs_dquot.c
@@ -23,6 +23,7 @@
#include "xfs_trace.h"
#include "xfs_log.h"
#include "xfs_bmap_btree.h"
+#include "xfs_error.h"
/*
* Lock order:
@@ -66,39 +67,61 @@
*/
void
xfs_qm_adjust_dqlimits(
- struct xfs_mount *mp,
struct xfs_dquot *dq)
{
+ struct xfs_mount *mp = dq->q_mount;
struct xfs_quotainfo *q = mp->m_quotainfo;
- struct xfs_disk_dquot *d = &dq->q_core;
struct xfs_def_quota *defq;
int prealloc = 0;
- ASSERT(d->d_id);
+ ASSERT(dq->q_id);
defq = xfs_get_defquota(q, xfs_dquot_type(dq));
- if (defq->bsoftlimit && !d->d_blk_softlimit) {
- d->d_blk_softlimit = cpu_to_be64(defq->bsoftlimit);
+ if (!dq->q_blk.softlimit) {
+ dq->q_blk.softlimit = defq->blk.soft;
prealloc = 1;
}
- if (defq->bhardlimit && !d->d_blk_hardlimit) {
- d->d_blk_hardlimit = cpu_to_be64(defq->bhardlimit);
+ if (!dq->q_blk.hardlimit) {
+ dq->q_blk.hardlimit = defq->blk.hard;
prealloc = 1;
}
- if (defq->isoftlimit && !d->d_ino_softlimit)
- d->d_ino_softlimit = cpu_to_be64(defq->isoftlimit);
- if (defq->ihardlimit && !d->d_ino_hardlimit)
- d->d_ino_hardlimit = cpu_to_be64(defq->ihardlimit);
- if (defq->rtbsoftlimit && !d->d_rtb_softlimit)
- d->d_rtb_softlimit = cpu_to_be64(defq->rtbsoftlimit);
- if (defq->rtbhardlimit && !d->d_rtb_hardlimit)
- d->d_rtb_hardlimit = cpu_to_be64(defq->rtbhardlimit);
+ if (!dq->q_ino.softlimit)
+ dq->q_ino.softlimit = defq->ino.soft;
+ if (!dq->q_ino.hardlimit)
+ dq->q_ino.hardlimit = defq->ino.hard;
+ if (!dq->q_rtb.softlimit)
+ dq->q_rtb.softlimit = defq->rtb.soft;
+ if (!dq->q_rtb.hardlimit)
+ dq->q_rtb.hardlimit = defq->rtb.hard;
if (prealloc)
xfs_dquot_set_prealloc_limits(dq);
}
/*
+ * Determine if this quota counter is over either limit and set the quota
+ * timers as appropriate.
+ */
+static inline void
+xfs_qm_adjust_res_timer(
+ struct xfs_dquot_res *res,
+ struct xfs_quota_limits *qlim)
+{
+ ASSERT(res->hardlimit == 0 || res->softlimit <= res->hardlimit);
+
+ if ((res->softlimit && res->count > res->softlimit) ||
+ (res->hardlimit && res->count > res->hardlimit)) {
+ if (res->timer == 0)
+ res->timer = ktime_get_real_seconds() + qlim->time;
+ } else {
+ if (res->timer == 0)
+ res->warnings = 0;
+ else
+ res->timer = 0;
+ }
+}
+
+/*
* Check the limits and timers of a dquot and start or reset timers
* if necessary.
* This gets called even when quota enforcement is OFF, which makes our
@@ -113,96 +136,18 @@
*/
void
xfs_qm_adjust_dqtimers(
- struct xfs_mount *mp,
struct xfs_dquot *dq)
{
+ struct xfs_mount *mp = dq->q_mount;
struct xfs_quotainfo *qi = mp->m_quotainfo;
- struct xfs_disk_dquot *d = &dq->q_core;
struct xfs_def_quota *defq;
- ASSERT(d->d_id);
+ ASSERT(dq->q_id);
defq = xfs_get_defquota(qi, xfs_dquot_type(dq));
-#ifdef DEBUG
- if (d->d_blk_hardlimit)
- ASSERT(be64_to_cpu(d->d_blk_softlimit) <=
- be64_to_cpu(d->d_blk_hardlimit));
- if (d->d_ino_hardlimit)
- ASSERT(be64_to_cpu(d->d_ino_softlimit) <=
- be64_to_cpu(d->d_ino_hardlimit));
- if (d->d_rtb_hardlimit)
- ASSERT(be64_to_cpu(d->d_rtb_softlimit) <=
- be64_to_cpu(d->d_rtb_hardlimit));
-#endif
-
- if (!d->d_btimer) {
- if ((d->d_blk_softlimit &&
- (be64_to_cpu(d->d_bcount) >
- be64_to_cpu(d->d_blk_softlimit))) ||
- (d->d_blk_hardlimit &&
- (be64_to_cpu(d->d_bcount) >
- be64_to_cpu(d->d_blk_hardlimit)))) {
- d->d_btimer = cpu_to_be32(ktime_get_real_seconds() +
- defq->btimelimit);
- } else {
- d->d_bwarns = 0;
- }
- } else {
- if ((!d->d_blk_softlimit ||
- (be64_to_cpu(d->d_bcount) <=
- be64_to_cpu(d->d_blk_softlimit))) &&
- (!d->d_blk_hardlimit ||
- (be64_to_cpu(d->d_bcount) <=
- be64_to_cpu(d->d_blk_hardlimit)))) {
- d->d_btimer = 0;
- }
- }
-
- if (!d->d_itimer) {
- if ((d->d_ino_softlimit &&
- (be64_to_cpu(d->d_icount) >
- be64_to_cpu(d->d_ino_softlimit))) ||
- (d->d_ino_hardlimit &&
- (be64_to_cpu(d->d_icount) >
- be64_to_cpu(d->d_ino_hardlimit)))) {
- d->d_itimer = cpu_to_be32(ktime_get_real_seconds() +
- defq->itimelimit);
- } else {
- d->d_iwarns = 0;
- }
- } else {
- if ((!d->d_ino_softlimit ||
- (be64_to_cpu(d->d_icount) <=
- be64_to_cpu(d->d_ino_softlimit))) &&
- (!d->d_ino_hardlimit ||
- (be64_to_cpu(d->d_icount) <=
- be64_to_cpu(d->d_ino_hardlimit)))) {
- d->d_itimer = 0;
- }
- }
-
- if (!d->d_rtbtimer) {
- if ((d->d_rtb_softlimit &&
- (be64_to_cpu(d->d_rtbcount) >
- be64_to_cpu(d->d_rtb_softlimit))) ||
- (d->d_rtb_hardlimit &&
- (be64_to_cpu(d->d_rtbcount) >
- be64_to_cpu(d->d_rtb_hardlimit)))) {
- d->d_rtbtimer = cpu_to_be32(ktime_get_real_seconds() +
- defq->rtbtimelimit);
- } else {
- d->d_rtbwarns = 0;
- }
- } else {
- if ((!d->d_rtb_softlimit ||
- (be64_to_cpu(d->d_rtbcount) <=
- be64_to_cpu(d->d_rtb_softlimit))) &&
- (!d->d_rtb_hardlimit ||
- (be64_to_cpu(d->d_rtbcount) <=
- be64_to_cpu(d->d_rtb_hardlimit)))) {
- d->d_rtbtimer = 0;
- }
- }
+ xfs_qm_adjust_res_timer(&dq->q_blk, &defq->blk);
+ xfs_qm_adjust_res_timer(&dq->q_ino, &defq->ino);
+ xfs_qm_adjust_res_timer(&dq->q_rtb, &defq->rtb);
}
/*
@@ -213,7 +158,7 @@
struct xfs_trans *tp,
struct xfs_mount *mp,
xfs_dqid_t id,
- uint type,
+ xfs_dqtype_t type,
struct xfs_buf *bp)
{
struct xfs_quotainfo *q = mp->m_quotainfo;
@@ -226,6 +171,24 @@
ASSERT(tp);
ASSERT(xfs_buf_islocked(bp));
+ switch (type) {
+ case XFS_DQTYPE_USER:
+ qflag = XFS_UQUOTA_CHKD;
+ blftype = XFS_BLF_UDQUOT_BUF;
+ break;
+ case XFS_DQTYPE_PROJ:
+ qflag = XFS_PQUOTA_CHKD;
+ blftype = XFS_BLF_PDQUOT_BUF;
+ break;
+ case XFS_DQTYPE_GROUP:
+ qflag = XFS_GQUOTA_CHKD;
+ blftype = XFS_BLF_GDQUOT_BUF;
+ break;
+ default:
+ ASSERT(0);
+ return;
+ }
+
d = bp->b_addr;
/*
@@ -237,7 +200,7 @@
d->dd_diskdq.d_magic = cpu_to_be16(XFS_DQUOT_MAGIC);
d->dd_diskdq.d_version = XFS_DQUOT_VERSION;
d->dd_diskdq.d_id = cpu_to_be32(curid);
- d->dd_diskdq.d_flags = type;
+ d->dd_diskdq.d_type = type;
if (xfs_sb_version_hascrc(&mp->m_sb)) {
uuid_copy(&d->dd_uuid, &mp->m_sb.sb_meta_uuid);
xfs_update_cksum((char *)d, sizeof(struct xfs_dqblk),
@@ -245,17 +208,6 @@
}
}
- if (type & XFS_DQ_USER) {
- qflag = XFS_UQUOTA_CHKD;
- blftype = XFS_BLF_UDQUOT_BUF;
- } else if (type & XFS_DQ_PROJ) {
- qflag = XFS_PQUOTA_CHKD;
- blftype = XFS_BLF_PDQUOT_BUF;
- } else {
- qflag = XFS_GQUOTA_CHKD;
- blftype = XFS_BLF_GDQUOT_BUF;
- }
-
xfs_trans_dquot_buf(tp, bp, blftype);
/*
@@ -290,8 +242,8 @@
{
uint64_t space;
- dqp->q_prealloc_hi_wmark = be64_to_cpu(dqp->q_core.d_blk_hardlimit);
- dqp->q_prealloc_lo_wmark = be64_to_cpu(dqp->q_core.d_blk_softlimit);
+ dqp->q_prealloc_hi_wmark = dqp->q_blk.hardlimit;
+ dqp->q_prealloc_lo_wmark = dqp->q_blk.softlimit;
if (!dqp->q_prealloc_lo_wmark) {
dqp->q_prealloc_lo_wmark = dqp->q_prealloc_hi_wmark;
do_div(dqp->q_prealloc_lo_wmark, 100);
@@ -321,14 +273,15 @@
struct xfs_trans *tp = *tpp;
struct xfs_mount *mp = tp->t_mountp;
struct xfs_buf *bp;
- struct xfs_inode *quotip = xfs_quota_inode(mp, dqp->dq_flags);
+ xfs_dqtype_t qtype = xfs_dquot_type(dqp);
+ struct xfs_inode *quotip = xfs_quota_inode(mp, qtype);
int nmaps = 1;
int error;
trace_xfs_dqalloc(dqp);
xfs_ilock(quotip, XFS_ILOCK_EXCL);
- if (!xfs_this_quota_on(dqp->q_mount, dqp->dq_flags)) {
+ if (!xfs_this_quota_on(dqp->q_mount, qtype)) {
/*
* Return if this type of quotas is turned off while we didn't
* have an inode lock
@@ -365,8 +318,7 @@
* Make a chunk of dquots out of this buffer and log
* the entire thing.
*/
- xfs_qm_init_dquot_blk(tp, mp, be32_to_cpu(dqp->q_core.d_id),
- dqp->dq_flags & XFS_DQ_ALLTYPES, bp);
+ xfs_qm_init_dquot_blk(tp, mp, dqp->q_id, qtype, bp);
xfs_buf_set_ref(bp, XFS_DQUOT_REF);
/*
@@ -413,13 +365,14 @@
{
struct xfs_bmbt_irec map;
struct xfs_buf *bp;
- struct xfs_inode *quotip = xfs_quota_inode(mp, dqp->dq_flags);
+ xfs_dqtype_t qtype = xfs_dquot_type(dqp);
+ struct xfs_inode *quotip = xfs_quota_inode(mp, qtype);
uint lock_mode;
int nmaps = 1;
int error;
lock_mode = xfs_ilock_data_map_shared(quotip);
- if (!xfs_this_quota_on(mp, dqp->dq_flags)) {
+ if (!xfs_this_quota_on(mp, qtype)) {
/*
* Return if this type of quotas is turned off while we
* didn't have the quota inode lock.
@@ -471,14 +424,14 @@
xfs_dquot_alloc(
struct xfs_mount *mp,
xfs_dqid_t id,
- uint type)
+ xfs_dqtype_t type)
{
struct xfs_dquot *dqp;
- dqp = kmem_zone_zalloc(xfs_qm_dqzone, 0);
+ dqp = kmem_cache_zalloc(xfs_qm_dqzone, GFP_KERNEL | __GFP_NOFAIL);
- dqp->dq_flags = type;
- dqp->q_core.d_id = cpu_to_be32(id);
+ dqp->q_type = type;
+ dqp->q_id = id;
dqp->q_mount = mp;
INIT_LIST_HEAD(&dqp->q_lru);
mutex_init(&dqp->q_qlock);
@@ -503,13 +456,13 @@
* quotas.
*/
switch (type) {
- case XFS_DQ_USER:
+ case XFS_DQTYPE_USER:
/* uses the default lock class */
break;
- case XFS_DQ_GROUP:
+ case XFS_DQTYPE_GROUP:
lockdep_set_class(&dqp->q_qlock, &xfs_dquot_group_class);
break;
- case XFS_DQ_PROJ:
+ case XFS_DQTYPE_PROJ:
lockdep_set_class(&dqp->q_qlock, &xfs_dquot_project_class);
break;
default:
@@ -524,26 +477,91 @@
}
/* Copy the in-core quota fields in from the on-disk buffer. */
-STATIC void
+STATIC int
xfs_dquot_from_disk(
struct xfs_dquot *dqp,
struct xfs_buf *bp)
{
struct xfs_disk_dquot *ddqp = bp->b_addr + dqp->q_bufoffset;
+ /*
+ * Ensure that we got the type and ID we were looking for.
+ * Everything else was checked by the dquot buffer verifier.
+ */
+ if ((ddqp->d_type & XFS_DQTYPE_REC_MASK) != xfs_dquot_type(dqp) ||
+ be32_to_cpu(ddqp->d_id) != dqp->q_id) {
+ xfs_alert_tag(bp->b_mount, XFS_PTAG_VERIFIER_ERROR,
+ "Metadata corruption detected at %pS, quota %u",
+ __this_address, dqp->q_id);
+ xfs_alert(bp->b_mount, "Unmount and run xfs_repair");
+ return -EFSCORRUPTED;
+ }
+
/* copy everything from disk dquot to the incore dquot */
- memcpy(&dqp->q_core, ddqp, sizeof(struct xfs_disk_dquot));
+ dqp->q_type = ddqp->d_type;
+ dqp->q_blk.hardlimit = be64_to_cpu(ddqp->d_blk_hardlimit);
+ dqp->q_blk.softlimit = be64_to_cpu(ddqp->d_blk_softlimit);
+ dqp->q_ino.hardlimit = be64_to_cpu(ddqp->d_ino_hardlimit);
+ dqp->q_ino.softlimit = be64_to_cpu(ddqp->d_ino_softlimit);
+ dqp->q_rtb.hardlimit = be64_to_cpu(ddqp->d_rtb_hardlimit);
+ dqp->q_rtb.softlimit = be64_to_cpu(ddqp->d_rtb_softlimit);
+
+ dqp->q_blk.count = be64_to_cpu(ddqp->d_bcount);
+ dqp->q_ino.count = be64_to_cpu(ddqp->d_icount);
+ dqp->q_rtb.count = be64_to_cpu(ddqp->d_rtbcount);
+
+ dqp->q_blk.warnings = be16_to_cpu(ddqp->d_bwarns);
+ dqp->q_ino.warnings = be16_to_cpu(ddqp->d_iwarns);
+ dqp->q_rtb.warnings = be16_to_cpu(ddqp->d_rtbwarns);
+
+ dqp->q_blk.timer = be32_to_cpu(ddqp->d_btimer);
+ dqp->q_ino.timer = be32_to_cpu(ddqp->d_itimer);
+ dqp->q_rtb.timer = be32_to_cpu(ddqp->d_rtbtimer);
/*
* Reservation counters are defined as reservation plus current usage
* to avoid having to add every time.
*/
- dqp->q_res_bcount = be64_to_cpu(ddqp->d_bcount);
- dqp->q_res_icount = be64_to_cpu(ddqp->d_icount);
- dqp->q_res_rtbcount = be64_to_cpu(ddqp->d_rtbcount);
+ dqp->q_blk.reserved = dqp->q_blk.count;
+ dqp->q_ino.reserved = dqp->q_ino.count;
+ dqp->q_rtb.reserved = dqp->q_rtb.count;
/* initialize the dquot speculative prealloc thresholds */
xfs_dquot_set_prealloc_limits(dqp);
+ return 0;
+}
+
+/* Copy the in-core quota fields into the on-disk buffer. */
+void
+xfs_dquot_to_disk(
+ struct xfs_disk_dquot *ddqp,
+ struct xfs_dquot *dqp)
+{
+ ddqp->d_magic = cpu_to_be16(XFS_DQUOT_MAGIC);
+ ddqp->d_version = XFS_DQUOT_VERSION;
+ ddqp->d_type = dqp->q_type;
+ ddqp->d_id = cpu_to_be32(dqp->q_id);
+ ddqp->d_pad0 = 0;
+ ddqp->d_pad = 0;
+
+ ddqp->d_blk_hardlimit = cpu_to_be64(dqp->q_blk.hardlimit);
+ ddqp->d_blk_softlimit = cpu_to_be64(dqp->q_blk.softlimit);
+ ddqp->d_ino_hardlimit = cpu_to_be64(dqp->q_ino.hardlimit);
+ ddqp->d_ino_softlimit = cpu_to_be64(dqp->q_ino.softlimit);
+ ddqp->d_rtb_hardlimit = cpu_to_be64(dqp->q_rtb.hardlimit);
+ ddqp->d_rtb_softlimit = cpu_to_be64(dqp->q_rtb.softlimit);
+
+ ddqp->d_bcount = cpu_to_be64(dqp->q_blk.count);
+ ddqp->d_icount = cpu_to_be64(dqp->q_ino.count);
+ ddqp->d_rtbcount = cpu_to_be64(dqp->q_rtb.count);
+
+ ddqp->d_bwarns = cpu_to_be16(dqp->q_blk.warnings);
+ ddqp->d_iwarns = cpu_to_be16(dqp->q_ino.warnings);
+ ddqp->d_rtbwarns = cpu_to_be16(dqp->q_rtb.warnings);
+
+ ddqp->d_btimer = cpu_to_be32(dqp->q_blk.timer);
+ ddqp->d_itimer = cpu_to_be32(dqp->q_ino.timer);
+ ddqp->d_rtbtimer = cpu_to_be32(dqp->q_rtb.timer);
}
/* Allocate and initialize the dquot buffer for this in-core dquot. */
@@ -592,7 +610,7 @@
xfs_qm_dqread(
struct xfs_mount *mp,
xfs_dqid_t id,
- uint type,
+ xfs_dqtype_t type,
bool can_alloc,
struct xfs_dquot **dqpp)
{
@@ -617,9 +635,11 @@
* further.
*/
ASSERT(xfs_buf_islocked(bp));
- xfs_dquot_from_disk(dqp, bp);
-
+ error = xfs_dquot_from_disk(dqp, bp);
xfs_buf_relse(bp);
+ if (error)
+ goto err;
+
*dqpp = dqp;
return error;
@@ -638,7 +658,7 @@
static int
xfs_dq_get_next_id(
struct xfs_mount *mp,
- uint type,
+ xfs_dqtype_t type,
xfs_dqid_t *id)
{
struct xfs_inode *quotip = xfs_quota_inode(mp, type);
@@ -706,7 +726,7 @@
}
xfs_dqlock(dqp);
- if (dqp->dq_flags & XFS_DQ_FREEING) {
+ if (dqp->q_flags & XFS_DQFLAG_FREEING) {
xfs_dqunlock(dqp);
mutex_unlock(&qi->qi_tree_lock);
trace_xfs_dqget_freeing(dqp);
@@ -762,21 +782,21 @@
static int
xfs_qm_dqget_checks(
struct xfs_mount *mp,
- uint type)
+ xfs_dqtype_t type)
{
if (WARN_ON_ONCE(!XFS_IS_QUOTA_RUNNING(mp)))
return -ESRCH;
switch (type) {
- case XFS_DQ_USER:
+ case XFS_DQTYPE_USER:
if (!XFS_IS_UQUOTA_ON(mp))
return -ESRCH;
return 0;
- case XFS_DQ_GROUP:
+ case XFS_DQTYPE_GROUP:
if (!XFS_IS_GQUOTA_ON(mp))
return -ESRCH;
return 0;
- case XFS_DQ_PROJ:
+ case XFS_DQTYPE_PROJ:
if (!XFS_IS_PQUOTA_ON(mp))
return -ESRCH;
return 0;
@@ -794,7 +814,7 @@
xfs_qm_dqget(
struct xfs_mount *mp,
xfs_dqid_t id,
- uint type,
+ xfs_dqtype_t type,
bool can_alloc,
struct xfs_dquot **O_dqpp)
{
@@ -844,7 +864,7 @@
xfs_qm_dqget_uncached(
struct xfs_mount *mp,
xfs_dqid_t id,
- uint type,
+ xfs_dqtype_t type,
struct xfs_dquot **dqpp)
{
int error;
@@ -860,14 +880,14 @@
xfs_dqid_t
xfs_qm_id_for_quotatype(
struct xfs_inode *ip,
- uint type)
+ xfs_dqtype_t type)
{
switch (type) {
- case XFS_DQ_USER:
+ case XFS_DQTYPE_USER:
return i_uid_read(VFS_I(ip));
- case XFS_DQ_GROUP:
+ case XFS_DQTYPE_GROUP:
return i_gid_read(VFS_I(ip));
- case XFS_DQ_PROJ:
+ case XFS_DQTYPE_PROJ:
return ip->i_d.di_projid;
}
ASSERT(0);
@@ -882,7 +902,7 @@
int
xfs_qm_dqget_inode(
struct xfs_inode *ip,
- uint type,
+ xfs_dqtype_t type,
bool can_alloc,
struct xfs_dquot **O_dqpp)
{
@@ -968,7 +988,7 @@
xfs_qm_dqget_next(
struct xfs_mount *mp,
xfs_dqid_t id,
- uint type,
+ xfs_dqtype_t type,
struct xfs_dquot **dqpp)
{
struct xfs_dquot *dqp;
@@ -1048,9 +1068,8 @@
* from the AIL if it has not been re-logged, and unlocking the dquot's
* flush lock. This behavior is very similar to that of inodes..
*/
-STATIC void
+static void
xfs_qm_dqflush_done(
- struct xfs_buf *bp,
struct xfs_log_item *lip)
{
struct xfs_dq_logitem *qip = (struct xfs_dq_logitem *)lip;
@@ -1071,16 +1090,12 @@
test_bit(XFS_LI_FAILED, &lip->li_flags))) {
spin_lock(&ailp->ail_lock);
+ xfs_clear_li_failed(lip);
if (lip->li_lsn == qip->qli_flush_lsn) {
/* xfs_ail_update_finish() drops the AIL lock */
tail_lsn = xfs_ail_delete_one(ailp, lip);
xfs_ail_update_finish(ailp, tail_lsn);
} else {
- /*
- * Clear the failed state since we are about to drop the
- * flush lock
- */
- xfs_clear_li_failed(lip);
spin_unlock(&ailp->ail_lock);
}
}
@@ -1091,6 +1106,48 @@
xfs_dqfunlock(dqp);
}
+void
+xfs_dquot_done(
+ struct xfs_buf *bp)
+{
+ struct xfs_log_item *lip, *n;
+
+ list_for_each_entry_safe(lip, n, &bp->b_li_list, li_bio_list) {
+ list_del_init(&lip->li_bio_list);
+ xfs_qm_dqflush_done(lip);
+ }
+}
+
+/* Check incore dquot for errors before we flush. */
+static xfs_failaddr_t
+xfs_qm_dqflush_check(
+ struct xfs_dquot *dqp)
+{
+ xfs_dqtype_t type = xfs_dquot_type(dqp);
+
+ if (type != XFS_DQTYPE_USER &&
+ type != XFS_DQTYPE_GROUP &&
+ type != XFS_DQTYPE_PROJ)
+ return __this_address;
+
+ if (dqp->q_id == 0)
+ return NULL;
+
+ if (dqp->q_blk.softlimit && dqp->q_blk.count > dqp->q_blk.softlimit &&
+ !dqp->q_blk.timer)
+ return __this_address;
+
+ if (dqp->q_ino.softlimit && dqp->q_ino.count > dqp->q_ino.softlimit &&
+ !dqp->q_ino.timer)
+ return __this_address;
+
+ if (dqp->q_rtb.softlimit && dqp->q_rtb.count > dqp->q_rtb.softlimit &&
+ !dqp->q_rtb.timer)
+ return __this_address;
+
+ return NULL;
+}
+
/*
* Write a modified dquot to disk.
* The dquot must be locked and the flush lock too taken by caller.
@@ -1107,8 +1164,7 @@
struct xfs_mount *mp = dqp->q_mount;
struct xfs_log_item *lip = &dqp->q_logitem.qli_item;
struct xfs_buf *bp;
- struct xfs_dqblk *dqb;
- struct xfs_disk_dquot *ddqp;
+ struct xfs_dqblk *dqblk;
xfs_failaddr_t fa;
int error;
@@ -1132,30 +1188,23 @@
if (error)
goto out_abort;
- /*
- * Calculate the location of the dquot inside the buffer.
- */
- dqb = bp->b_addr + dqp->q_bufoffset;
- ddqp = &dqb->dd_diskdq;
-
- /* sanity check the in-core structure before we flush */
- fa = xfs_dquot_verify(mp, &dqp->q_core, be32_to_cpu(dqp->q_core.d_id),
- 0);
+ fa = xfs_qm_dqflush_check(dqp);
if (fa) {
xfs_alert(mp, "corrupt dquot ID 0x%x in memory at %pS",
- be32_to_cpu(dqp->q_core.d_id), fa);
+ dqp->q_id, fa);
xfs_buf_relse(bp);
error = -EFSCORRUPTED;
goto out_abort;
}
- /* This is the only portion of data that needs to persist */
- memcpy(ddqp, &dqp->q_core, sizeof(struct xfs_disk_dquot));
+ /* Flush the incore dquot to the ondisk buffer. */
+ dqblk = bp->b_addr + dqp->q_bufoffset;
+ xfs_dquot_to_disk(&dqblk->dd_diskdq, dqp);
/*
* Clear the dirty field and remember the flush lsn for later use.
*/
- dqp->dq_flags &= ~XFS_DQ_DIRTY;
+ dqp->q_flags &= ~XFS_DQFLAG_DIRTY;
xfs_trans_ail_copy_lsn(mp->m_ail, &dqp->q_logitem.qli_flush_lsn,
&dqp->q_logitem.qli_item.li_lsn);
@@ -1170,17 +1219,17 @@
* of a dquot without an up-to-date CRC getting to disk.
*/
if (xfs_sb_version_hascrc(&mp->m_sb)) {
- dqb->dd_lsn = cpu_to_be64(dqp->q_logitem.qli_item.li_lsn);
- xfs_update_cksum((char *)dqb, sizeof(struct xfs_dqblk),
+ dqblk->dd_lsn = cpu_to_be64(dqp->q_logitem.qli_item.li_lsn);
+ xfs_update_cksum((char *)dqblk, sizeof(struct xfs_dqblk),
XFS_DQUOT_CRC_OFF);
}
/*
- * Attach an iodone routine so that we can remove this dquot from the
- * AIL and release the flush lock once the dquot is synced to disk.
+ * Attach the dquot to the buffer so that we can remove this dquot from
+ * the AIL and release the flush lock once the dquot is synced to disk.
*/
- xfs_buf_attach_iodone(bp, xfs_qm_dqflush_done,
- &dqp->q_logitem.qli_item);
+ bp->b_flags |= _XBF_DQUOTS;
+ list_add_tail(&dqp->q_logitem.qli_item.li_bio_list, &bp->b_li_list);
/*
* If the buffer is pinned then push on the log so we won't
@@ -1196,7 +1245,7 @@
return 0;
out_abort:
- dqp->dq_flags &= ~XFS_DQ_DIRTY;
+ dqp->q_flags &= ~XFS_DQFLAG_DIRTY;
xfs_trans_ail_delete(lip, 0);
xfs_force_shutdown(mp, SHUTDOWN_CORRUPT_INCORE);
out_unlock:
@@ -1217,8 +1266,7 @@
{
if (d1 && d2) {
ASSERT(d1 != d2);
- if (be32_to_cpu(d1->q_core.d_id) >
- be32_to_cpu(d2->q_core.d_id)) {
+ if (d1->q_id > d2->q_id) {
mutex_lock(&d2->q_qlock);
mutex_lock_nested(&d1->q_qlock, XFS_QLOCK_NESTED);
} else {
@@ -1270,7 +1318,7 @@
int
xfs_qm_dqiterate(
struct xfs_mount *mp,
- uint dqtype,
+ xfs_dqtype_t type,
xfs_qm_dqiterate_fn iter_fn,
void *priv)
{
@@ -1279,16 +1327,15 @@
int error;
do {
- error = xfs_qm_dqget_next(mp, id, dqtype, &dq);
+ error = xfs_qm_dqget_next(mp, id, type, &dq);
if (error == -ENOENT)
return 0;
if (error)
return error;
- error = iter_fn(dq, dqtype, priv);
- id = be32_to_cpu(dq->q_core.d_id);
+ error = iter_fn(dq, type, priv);
+ id = dq->q_id;
xfs_qm_dqput(dq);
- id++;
} while (error == 0 && id != 0);
return error;
diff --git a/fs/xfs/xfs_dquot.h b/fs/xfs/xfs_dquot.h
index 71e36c8..282a65d 100644
--- a/fs/xfs/xfs_dquot.h
+++ b/fs/xfs/xfs_dquot.h
@@ -27,26 +27,53 @@
XFS_QLOWSP_MAX
};
+struct xfs_dquot_res {
+ /* Total resources allocated and reserved. */
+ xfs_qcnt_t reserved;
+
+ /* Total resources allocated. */
+ xfs_qcnt_t count;
+
+ /* Absolute and preferred limits. */
+ xfs_qcnt_t hardlimit;
+ xfs_qcnt_t softlimit;
+
+ /*
+ * For root dquots, this is the default grace period, in seconds.
+ * Otherwise, this is when the quota grace period expires,
+ * in seconds since the Unix epoch.
+ */
+ time64_t timer;
+
+ /*
+ * For root dquots, this is the maximum number of warnings that will
+ * be issued for this quota type. Otherwise, this is the number of
+ * warnings issued against this quota. Note that none of this is
+ * implemented.
+ */
+ xfs_qwarncnt_t warnings;
+};
+
/*
* The incore dquot structure
*/
struct xfs_dquot {
- uint dq_flags;
struct list_head q_lru;
struct xfs_mount *q_mount;
+ xfs_dqtype_t q_type;
+ uint16_t q_flags;
+ xfs_dqid_t q_id;
uint q_nrefs;
- xfs_daddr_t q_blkno;
int q_bufoffset;
+ xfs_daddr_t q_blkno;
xfs_fileoff_t q_fileoffset;
- struct xfs_disk_dquot q_core;
+ struct xfs_dquot_res q_blk; /* regular blocks */
+ struct xfs_dquot_res q_ino; /* inodes */
+ struct xfs_dquot_res q_rtb; /* realtime blocks */
+
struct xfs_dq_logitem q_logitem;
- /* total regular nblks used+reserved */
- xfs_qcnt_t q_res_bcount;
- /* total inos allocd+reserved */
- xfs_qcnt_t q_res_icount;
- /* total realtime blks used+reserved */
- xfs_qcnt_t q_res_rtbcount;
+
xfs_qcnt_t q_prealloc_lo_wmark;
xfs_qcnt_t q_prealloc_hi_wmark;
int64_t q_low_space[XFS_QLOWSP_MAX];
@@ -101,34 +128,59 @@
mutex_unlock(&dqp->q_qlock);
}
-static inline int xfs_this_quota_on(struct xfs_mount *mp, int type)
+static inline int
+xfs_dquot_type(const struct xfs_dquot *dqp)
{
- switch (type & XFS_DQ_ALLTYPES) {
- case XFS_DQ_USER:
+ return dqp->q_type & XFS_DQTYPE_REC_MASK;
+}
+
+static inline int xfs_this_quota_on(struct xfs_mount *mp, xfs_dqtype_t type)
+{
+ switch (type) {
+ case XFS_DQTYPE_USER:
return XFS_IS_UQUOTA_ON(mp);
- case XFS_DQ_GROUP:
+ case XFS_DQTYPE_GROUP:
return XFS_IS_GQUOTA_ON(mp);
- case XFS_DQ_PROJ:
+ case XFS_DQTYPE_PROJ:
return XFS_IS_PQUOTA_ON(mp);
default:
return 0;
}
}
-static inline struct xfs_dquot *xfs_inode_dquot(struct xfs_inode *ip, int type)
+static inline struct xfs_dquot *xfs_inode_dquot(
+ struct xfs_inode *ip,
+ xfs_dqtype_t type)
{
- switch (type & XFS_DQ_ALLTYPES) {
- case XFS_DQ_USER:
+ switch (type) {
+ case XFS_DQTYPE_USER:
return ip->i_udquot;
- case XFS_DQ_GROUP:
+ case XFS_DQTYPE_GROUP:
return ip->i_gdquot;
- case XFS_DQ_PROJ:
+ case XFS_DQTYPE_PROJ:
return ip->i_pdquot;
default:
return NULL;
}
}
+/* Decide if the dquot's limits are actually being enforced. */
+static inline bool
+xfs_dquot_is_enforced(
+ const struct xfs_dquot *dqp)
+{
+ switch (xfs_dquot_type(dqp)) {
+ case XFS_DQTYPE_USER:
+ return XFS_IS_UQUOTA_ENFORCED(dqp->q_mount);
+ case XFS_DQTYPE_GROUP:
+ return XFS_IS_GQUOTA_ENFORCED(dqp->q_mount);
+ case XFS_DQTYPE_PROJ:
+ return XFS_IS_PQUOTA_ENFORCED(dqp->q_mount);
+ }
+ ASSERT(0);
+ return false;
+}
+
/*
* Check whether a dquot is under low free space conditions. We assume the quota
* is enabled and enforced.
@@ -137,38 +189,35 @@
{
int64_t freesp;
- freesp = be64_to_cpu(dqp->q_core.d_blk_hardlimit) - dqp->q_res_bcount;
+ freesp = dqp->q_blk.hardlimit - dqp->q_blk.reserved;
if (freesp < dqp->q_low_space[XFS_QLOWSP_1_PCNT])
return true;
return false;
}
+void xfs_dquot_to_disk(struct xfs_disk_dquot *ddqp, struct xfs_dquot *dqp);
+
#define XFS_DQ_IS_LOCKED(dqp) (mutex_is_locked(&((dqp)->q_qlock)))
-#define XFS_DQ_IS_DIRTY(dqp) ((dqp)->dq_flags & XFS_DQ_DIRTY)
-#define XFS_QM_ISUDQ(dqp) ((dqp)->dq_flags & XFS_DQ_USER)
-#define XFS_QM_ISPDQ(dqp) ((dqp)->dq_flags & XFS_DQ_PROJ)
-#define XFS_QM_ISGDQ(dqp) ((dqp)->dq_flags & XFS_DQ_GROUP)
+#define XFS_DQ_IS_DIRTY(dqp) ((dqp)->q_flags & XFS_DQFLAG_DIRTY)
void xfs_qm_dqdestroy(struct xfs_dquot *dqp);
int xfs_qm_dqflush(struct xfs_dquot *dqp, struct xfs_buf **bpp);
void xfs_qm_dqunpin_wait(struct xfs_dquot *dqp);
-void xfs_qm_adjust_dqtimers(struct xfs_mount *mp,
- struct xfs_dquot *d);
-void xfs_qm_adjust_dqlimits(struct xfs_mount *mp,
- struct xfs_dquot *d);
-xfs_dqid_t xfs_qm_id_for_quotatype(struct xfs_inode *ip, uint type);
+void xfs_qm_adjust_dqtimers(struct xfs_dquot *d);
+void xfs_qm_adjust_dqlimits(struct xfs_dquot *d);
+xfs_dqid_t xfs_qm_id_for_quotatype(struct xfs_inode *ip,
+ xfs_dqtype_t type);
int xfs_qm_dqget(struct xfs_mount *mp, xfs_dqid_t id,
- uint type, bool can_alloc,
- struct xfs_dquot **dqpp);
-int xfs_qm_dqget_inode(struct xfs_inode *ip, uint type,
- bool can_alloc,
- struct xfs_dquot **dqpp);
+ xfs_dqtype_t type, bool can_alloc,
+ struct xfs_dquot **dqpp);
+int xfs_qm_dqget_inode(struct xfs_inode *ip, xfs_dqtype_t type,
+ bool can_alloc, struct xfs_dquot **dqpp);
int xfs_qm_dqget_next(struct xfs_mount *mp, xfs_dqid_t id,
- uint type, struct xfs_dquot **dqpp);
+ xfs_dqtype_t type, struct xfs_dquot **dqpp);
int xfs_qm_dqget_uncached(struct xfs_mount *mp,
- xfs_dqid_t id, uint type,
- struct xfs_dquot **dqpp);
+ xfs_dqid_t id, xfs_dqtype_t type,
+ struct xfs_dquot **dqpp);
void xfs_qm_dqput(struct xfs_dquot *dqp);
void xfs_dqlock2(struct xfs_dquot *, struct xfs_dquot *);
@@ -183,9 +232,9 @@
return dqp;
}
-typedef int (*xfs_qm_dqiterate_fn)(struct xfs_dquot *dq, uint dqtype,
- void *priv);
-int xfs_qm_dqiterate(struct xfs_mount *mp, uint dqtype,
+typedef int (*xfs_qm_dqiterate_fn)(struct xfs_dquot *dq,
+ xfs_dqtype_t type, void *priv);
+int xfs_qm_dqiterate(struct xfs_mount *mp, xfs_dqtype_t type,
xfs_qm_dqiterate_fn iter_fn, void *priv);
#endif /* __XFS_DQUOT_H__ */
diff --git a/fs/xfs/xfs_dquot_item.c b/fs/xfs/xfs_dquot_item.c
index 349c92d2..8c1fdf3 100644
--- a/fs/xfs/xfs_dquot_item.c
+++ b/fs/xfs/xfs_dquot_item.c
@@ -45,6 +45,7 @@
struct xfs_log_item *lip,
struct xfs_log_vec *lv)
{
+ struct xfs_disk_dquot ddq;
struct xfs_dq_logitem *qlip = DQUOT_ITEM(lip);
struct xfs_log_iovec *vecp = NULL;
struct xfs_dq_logformat *qlf;
@@ -52,14 +53,15 @@
qlf = xlog_prepare_iovec(lv, &vecp, XLOG_REG_TYPE_QFORMAT);
qlf->qlf_type = XFS_LI_DQUOT;
qlf->qlf_size = 2;
- qlf->qlf_id = be32_to_cpu(qlip->qli_dquot->q_core.d_id);
+ qlf->qlf_id = qlip->qli_dquot->q_id;
qlf->qlf_blkno = qlip->qli_dquot->q_blkno;
qlf->qlf_len = 1;
qlf->qlf_boffset = qlip->qli_dquot->q_bufoffset;
xlog_finish_iovec(lv, vecp, sizeof(struct xfs_dq_logformat));
- xlog_copy_iovec(lv, &vecp, XLOG_REG_TYPE_DQUOT,
- &qlip->qli_dquot->q_core,
+ xfs_dquot_to_disk(&ddq, qlip->qli_dquot);
+
+ xlog_copy_iovec(lv, &vecp, XLOG_REG_TYPE_DQUOT, &ddq,
sizeof(struct xfs_disk_dquot));
}
@@ -113,23 +115,6 @@
wait_event(dqp->q_pinwait, (atomic_read(&dqp->q_pincount) == 0));
}
-/*
- * Callback used to mark a buffer with XFS_LI_FAILED when items in the buffer
- * have been failed during writeback
- *
- * this informs the AIL that the dquot is already flush locked on the next push,
- * and acquires a hold on the buffer to ensure that it isn't reclaimed before
- * dirty data makes it to disk.
- */
-STATIC void
-xfs_dquot_item_error(
- struct xfs_log_item *lip,
- struct xfs_buf *bp)
-{
- ASSERT(!completion_done(&DQUOT_ITEM(lip)->qli_dquot->q_flush));
- xfs_set_li_failed(lip, bp);
-}
-
STATIC uint
xfs_qm_dquot_logitem_push(
struct xfs_log_item *lip,
@@ -216,7 +201,6 @@
.iop_release = xfs_qm_dquot_logitem_release,
.iop_committing = xfs_qm_dquot_logitem_committing,
.iop_push = xfs_qm_dquot_logitem_push,
- .iop_error = xfs_dquot_item_error
};
/*
diff --git a/fs/xfs/xfs_dquot_item_recover.c b/fs/xfs/xfs_dquot_item_recover.c
index 3400be4..5875c7e 100644
--- a/fs/xfs/xfs_dquot_item_recover.c
+++ b/fs/xfs/xfs_dquot_item_recover.c
@@ -39,7 +39,7 @@
if (item->ri_buf[1].i_len < sizeof(struct xfs_disk_dquot))
return;
- type = recddq->d_flags & (XFS_DQ_USER | XFS_DQ_PROJ | XFS_DQ_GROUP);
+ type = recddq->d_type & XFS_DQTYPE_REC_MASK;
ASSERT(type);
if (log->l_quotaoffs_flag & type)
return;
@@ -91,7 +91,7 @@
/*
* This type of quotas was turned off, so ignore this record.
*/
- type = recddq->d_flags & (XFS_DQ_USER | XFS_DQ_PROJ | XFS_DQ_GROUP);
+ type = recddq->d_type & XFS_DQTYPE_REC_MASK;
ASSERT(type);
if (log->l_quotaoffs_flag & type)
return 0;
@@ -108,7 +108,7 @@
*/
dq_f = item->ri_buf[0].i_addr;
ASSERT(dq_f);
- fa = xfs_dquot_verify(mp, recddq, dq_f->qlf_id, 0);
+ fa = xfs_dquot_verify(mp, recddq, dq_f->qlf_id);
if (fa) {
xfs_alert(mp, "corrupt dquot ID 0x%x in log at %pS",
dq_f->qlf_id, fa);
@@ -153,7 +153,7 @@
ASSERT(dq_f->qlf_size == 2);
ASSERT(bp->b_mount == mp);
- bp->b_iodone = xlog_recover_iodone;
+ bp->b_flags |= _XBF_LOGRECOVERY;
xfs_buf_delwri_queue(bp, buffer_list);
out_release:
@@ -185,11 +185,11 @@
* group/project quotaoff or both.
*/
if (qoff_f->qf_flags & XFS_UQUOTA_ACCT)
- log->l_quotaoffs_flag |= XFS_DQ_USER;
+ log->l_quotaoffs_flag |= XFS_DQTYPE_USER;
if (qoff_f->qf_flags & XFS_PQUOTA_ACCT)
- log->l_quotaoffs_flag |= XFS_DQ_PROJ;
+ log->l_quotaoffs_flag |= XFS_DQTYPE_PROJ;
if (qoff_f->qf_flags & XFS_GQUOTA_ACCT)
- log->l_quotaoffs_flag |= XFS_DQ_GROUP;
+ log->l_quotaoffs_flag |= XFS_DQTYPE_GROUP;
return 0;
}
diff --git a/fs/xfs/xfs_extfree_item.c b/fs/xfs/xfs_extfree_item.c
index b9c333b..6cb8cd1 100644
--- a/fs/xfs/xfs_extfree_item.c
+++ b/fs/xfs/xfs_extfree_item.c
@@ -161,7 +161,8 @@
((nextents - 1) * sizeof(xfs_extent_t)));
efip = kmem_zalloc(size, 0);
} else {
- efip = kmem_zone_zalloc(xfs_efi_zone, 0);
+ efip = kmem_cache_zalloc(xfs_efi_zone,
+ GFP_KERNEL | __GFP_NOFAIL);
}
xfs_log_item_init(mp, &efip->efi_item, XFS_LI_EFI, &xfs_efi_item_ops);
@@ -332,7 +333,8 @@
(nextents - 1) * sizeof(struct xfs_extent),
0);
} else {
- efdp = kmem_zone_zalloc(xfs_efd_zone, 0);
+ efdp = kmem_cache_zalloc(xfs_efd_zone,
+ GFP_KERNEL | __GFP_NOFAIL);
}
xfs_log_item_init(tp->t_mountp, &efdp->efd_item, XFS_LI_EFD,
diff --git a/fs/xfs/xfs_file.c b/fs/xfs/xfs_file.c
index fdbff48..c31cd3b 100644
--- a/fs/xfs/xfs_file.c
+++ b/fs/xfs/xfs_file.c
@@ -94,6 +94,7 @@
{
struct inode *inode = file->f_mapping->host;
struct xfs_inode *ip = XFS_I(inode);
+ struct xfs_inode_log_item *iip = ip->i_itemp;
struct xfs_mount *mp = ip->i_mount;
int error = 0;
int log_flushed = 0;
@@ -137,13 +138,15 @@
xfs_ilock(ip, XFS_ILOCK_SHARED);
if (xfs_ipincount(ip)) {
if (!datasync ||
- (ip->i_itemp->ili_fsync_fields & ~XFS_ILOG_TIMESTAMP))
- lsn = ip->i_itemp->ili_last_lsn;
+ (iip->ili_fsync_fields & ~XFS_ILOG_TIMESTAMP))
+ lsn = iip->ili_last_lsn;
}
if (lsn) {
error = xfs_log_force_lsn(mp, lsn, XFS_LOG_SYNC, &log_flushed);
- ip->i_itemp->ili_fsync_fields = 0;
+ spin_lock(&iip->ili_lock);
+ iip->ili_fsync_fields = 0;
+ spin_unlock(&iip->ili_lock);
}
xfs_iunlock(ip, XFS_ILOCK_SHARED);
@@ -505,7 +508,7 @@
*/
if (xfs_is_cow_inode(ip)) {
trace_xfs_reflink_bounce_dio_write(ip, iocb->ki_pos, count);
- return -EREMCHG;
+ return -ENOTBLK;
}
iolock = XFS_IOLOCK_EXCL;
} else {
@@ -553,8 +556,8 @@
xfs_iunlock(ip, iolock);
/*
- * No fallback to buffered IO on errors for XFS, direct IO will either
- * complete fully or fail.
+ * No fallback to buffered IO after short writes for XFS, direct I/O
+ * will either complete fully or return an error.
*/
ASSERT(ret < 0 || ret == count);
return ret;
@@ -714,7 +717,7 @@
* allow an operation to fall back to buffered mode.
*/
ret = xfs_file_dio_aio_write(iocb, from);
- if (ret != -EREMCHG)
+ if (ret != -ENOTBLK)
return ret;
}
@@ -1035,7 +1038,7 @@
/* Prepare and then clone file data. */
ret = xfs_reflink_remap_prep(file_in, pos_in, file_out, pos_out,
&len, remap_flags);
- if (ret < 0 || len == 0)
+ if (ret || len == 0)
return ret;
trace_xfs_reflink_remap_range(src, pos_in, len, dest, pos_out);
@@ -1065,7 +1068,7 @@
if (mp->m_flags & XFS_MOUNT_WSYNC)
xfs_log_force_inode(dest);
out_unlock:
- xfs_reflink_remap_unlock(file_in, file_out);
+ xfs_iunlock2_io_mmap(src, dest);
if (ret)
trace_xfs_reflink_remap_range_error(dest, ret, _RET_IP_);
return remapped > 0 ? remapped : ret;
@@ -1263,10 +1266,23 @@
return __xfs_filemap_fault(vmf, PE_SIZE_PTE, true);
}
+static void
+xfs_filemap_map_pages(
+ struct vm_fault *vmf,
+ pgoff_t start_pgoff,
+ pgoff_t end_pgoff)
+{
+ struct inode *inode = file_inode(vmf->vma->vm_file);
+
+ xfs_ilock(XFS_I(inode), XFS_MMAPLOCK_SHARED);
+ filemap_map_pages(vmf, start_pgoff, end_pgoff);
+ xfs_iunlock(XFS_I(inode), XFS_MMAPLOCK_SHARED);
+}
+
static const struct vm_operations_struct xfs_file_vm_ops = {
.fault = xfs_filemap_fault,
.huge_fault = xfs_filemap_huge_fault,
- .map_pages = filemap_map_pages,
+ .map_pages = xfs_filemap_map_pages,
.page_mkwrite = xfs_filemap_page_mkwrite,
.pfn_mkwrite = xfs_filemap_pfn_mkwrite,
};
diff --git a/fs/xfs/xfs_icache.c b/fs/xfs/xfs_icache.c
index 5daef65..101028e 100644
--- a/fs/xfs/xfs_icache.c
+++ b/fs/xfs/xfs_icache.c
@@ -37,13 +37,11 @@
struct xfs_inode *ip;
/*
- * if this didn't occur in transactions, we could use
- * KM_MAYFAIL and return NULL here on ENOMEM. Set the
- * code up to do this anyway.
+ * XXX: If this didn't occur in transactions, we could drop GFP_NOFAIL
+ * and return NULL here on ENOMEM.
*/
- ip = kmem_zone_alloc(xfs_inode_zone, 0);
- if (!ip)
- return NULL;
+ ip = kmem_cache_alloc(xfs_inode_zone, GFP_KERNEL | __GFP_NOFAIL);
+
if (inode_init_always(mp->m_super, VFS_I(ip))) {
kmem_cache_free(xfs_inode_zone, ip);
return NULL;
@@ -115,6 +113,7 @@
{
/* asserts to verify all state is correct here */
ASSERT(atomic_read(&ip->i_pincount) == 0);
+ ASSERT(!ip->i_itemp || list_empty(&ip->i_itemp->ili_item.li_bio_list));
XFS_STATS_DEC(ip->i_mount, vn_active);
call_rcu(&VFS_I(ip)->i_rcu, xfs_inode_free_callback);
@@ -141,11 +140,8 @@
}
/*
- * Queue a new inode reclaim pass if there are reclaimable inodes and there
- * isn't a reclaim pass already in progress. By default it runs every 5s based
- * on the xfs periodic sync default of 30s. Perhaps this should have it's own
- * tunable, but that can be done if this method proves to be ineffective or too
- * aggressive.
+ * Queue background inode reclaim work if there are reclaimable inodes and there
+ * isn't reclaim work already scheduled or in progress.
*/
static void
xfs_reclaim_work_queue(
@@ -160,24 +156,6 @@
rcu_read_unlock();
}
-/*
- * This is a fast pass over the inode cache to try to get reclaim moving on as
- * many inodes as possible in a short period of time. It kicks itself every few
- * seconds, as well as being kicked by the inode cache shrinker when memory
- * goes low. It scans as quickly as possible avoiding locked inodes or those
- * already being flushed, and once done schedules a future pass.
- */
-void
-xfs_reclaim_worker(
- struct work_struct *work)
-{
- struct xfs_mount *mp = container_of(to_delayed_work(work),
- struct xfs_mount, m_reclaim_work);
-
- xfs_reclaim_inodes(mp, SYNC_TRYLOCK);
- xfs_reclaim_work_queue(mp);
-}
-
static void
xfs_perag_set_reclaim_tag(
struct xfs_perag *pag)
@@ -618,48 +596,31 @@
}
/*
- * Look up an inode by number in the given file system.
- * The inode is looked up in the cache held in each AG.
- * If the inode is found in the cache, initialise the vfs inode
- * if necessary.
+ * Look up an inode by number in the given file system. The inode is looked up
+ * in the cache held in each AG. If the inode is found in the cache, initialise
+ * the vfs inode if necessary.
*
- * If it is not in core, read it in from the file system's device,
- * add it to the cache and initialise the vfs inode.
+ * If it is not in core, read it in from the file system's device, add it to the
+ * cache and initialise the vfs inode.
*
* The inode is locked according to the value of the lock_flags parameter.
- * This flag parameter indicates how and if the inode's IO lock and inode lock
- * should be taken.
- *
- * mp -- the mount point structure for the current file system. It points
- * to the inode hash table.
- * tp -- a pointer to the current transaction if there is one. This is
- * simply passed through to the xfs_iread() call.
- * ino -- the number of the inode desired. This is the unique identifier
- * within the file system for the inode being requested.
- * lock_flags -- flags indicating how to lock the inode. See the comment
- * for xfs_ilock() for a list of valid values.
+ * Inode lookup is only done during metadata operations and not as part of the
+ * data IO path. Hence we only allow locking of the XFS_ILOCK during lookup.
*/
int
xfs_iget(
- xfs_mount_t *mp,
- xfs_trans_t *tp,
- xfs_ino_t ino,
- uint flags,
- uint lock_flags,
- xfs_inode_t **ipp)
+ struct xfs_mount *mp,
+ struct xfs_trans *tp,
+ xfs_ino_t ino,
+ uint flags,
+ uint lock_flags,
+ struct xfs_inode **ipp)
{
- xfs_inode_t *ip;
- int error;
- xfs_perag_t *pag;
- xfs_agino_t agino;
+ struct xfs_inode *ip;
+ struct xfs_perag *pag;
+ xfs_agino_t agino;
+ int error;
- /*
- * xfs_reclaim_inode() uses the ILOCK to ensure an inode
- * doesn't get freed while it's being referenced during a
- * radix tree traversal here. It assumes this function
- * aqcuires only the ILOCK (and therefore it has no need to
- * involve the IOLOCK in this synchronization).
- */
ASSERT((lock_flags & (XFS_IOLOCK_EXCL | XFS_IOLOCK_SHARED)) == 0);
/* reject inode numbers outside existing AGs */
@@ -776,15 +737,7 @@
ASSERT(rcu_read_lock_held());
- /*
- * check for stale RCU freed inode
- *
- * If the inode has been reallocated, it doesn't matter if it's not in
- * the AG we are walking - we are walking for writeback, so if it
- * passes all the "valid inode" checks and is dirty, then we'll write
- * it back anyway. If it has been reallocated and still being
- * initialised, the XFS_INEW check below will catch it.
- */
+ /* Check for stale RCU freed inode */
spin_lock(&ip->i_flags_lock);
if (!ip->i_ino)
goto out_unlock_noent;
@@ -1028,107 +981,62 @@
/*
* Grab the inode for reclaim exclusively.
- * Return 0 if we grabbed it, non-zero otherwise.
+ *
+ * We have found this inode via a lookup under RCU, so the inode may have
+ * already been freed, or it may be in the process of being recycled by
+ * xfs_iget(). In both cases, the inode will have XFS_IRECLAIM set. If the inode
+ * has been fully recycled by the time we get the i_flags_lock, XFS_IRECLAIMABLE
+ * will not be set. Hence we need to check for both these flag conditions to
+ * avoid inodes that are no longer reclaim candidates.
+ *
+ * Note: checking for other state flags here, under the i_flags_lock or not, is
+ * racy and should be avoided. Those races should be resolved only after we have
+ * ensured that we are able to reclaim this inode and the world can see that we
+ * are going to reclaim it.
+ *
+ * Return true if we grabbed it, false otherwise.
*/
-STATIC int
+static bool
xfs_reclaim_inode_grab(
- struct xfs_inode *ip,
- int flags)
+ struct xfs_inode *ip)
{
ASSERT(rcu_read_lock_held());
- /* quick check for stale RCU freed inode */
- if (!ip->i_ino)
- return 1;
-
- /*
- * If we are asked for non-blocking operation, do unlocked checks to
- * see if the inode already is being flushed or in reclaim to avoid
- * lock traffic.
- */
- if ((flags & SYNC_TRYLOCK) &&
- __xfs_iflags_test(ip, XFS_IFLOCK | XFS_IRECLAIM))
- return 1;
-
- /*
- * The radix tree lock here protects a thread in xfs_iget from racing
- * with us starting reclaim on the inode. Once we have the
- * XFS_IRECLAIM flag set it will not touch us.
- *
- * Due to RCU lookup, we may find inodes that have been freed and only
- * have XFS_IRECLAIM set. Indeed, we may see reallocated inodes that
- * aren't candidates for reclaim at all, so we must check the
- * XFS_IRECLAIMABLE is set first before proceeding to reclaim.
- */
spin_lock(&ip->i_flags_lock);
if (!__xfs_iflags_test(ip, XFS_IRECLAIMABLE) ||
__xfs_iflags_test(ip, XFS_IRECLAIM)) {
/* not a reclaim candidate. */
spin_unlock(&ip->i_flags_lock);
- return 1;
+ return false;
}
__xfs_iflags_set(ip, XFS_IRECLAIM);
spin_unlock(&ip->i_flags_lock);
- return 0;
+ return true;
}
/*
- * Inodes in different states need to be treated differently. The following
- * table lists the inode states and the reclaim actions necessary:
+ * Inode reclaim is non-blocking, so the default action if progress cannot be
+ * made is to "requeue" the inode for reclaim by unlocking it and clearing the
+ * XFS_IRECLAIM flag. If we are in a shutdown state, we don't care about
+ * blocking anymore and hence we can wait for the inode to be able to reclaim
+ * it.
*
- * inode state iflush ret required action
- * --------------- ---------- ---------------
- * bad - reclaim
- * shutdown EIO unpin and reclaim
- * clean, unpinned 0 reclaim
- * stale, unpinned 0 reclaim
- * clean, pinned(*) 0 requeue
- * stale, pinned EAGAIN requeue
- * dirty, async - requeue
- * dirty, sync 0 reclaim
- *
- * (*) dgc: I don't think the clean, pinned state is possible but it gets
- * handled anyway given the order of checks implemented.
- *
- * Also, because we get the flush lock first, we know that any inode that has
- * been flushed delwri has had the flush completed by the time we check that
- * the inode is clean.
- *
- * Note that because the inode is flushed delayed write by AIL pushing, the
- * flush lock may already be held here and waiting on it can result in very
- * long latencies. Hence for sync reclaims, where we wait on the flush lock,
- * the caller should push the AIL first before trying to reclaim inodes to
- * minimise the amount of time spent waiting. For background relaim, we only
- * bother to reclaim clean inodes anyway.
- *
- * Hence the order of actions after gaining the locks should be:
- * bad => reclaim
- * shutdown => unpin and reclaim
- * pinned, async => requeue
- * pinned, sync => unpin
- * stale => reclaim
- * clean => reclaim
- * dirty, async => requeue
- * dirty, sync => flush, wait and reclaim
+ * We do no IO here - if callers require inodes to be cleaned they must push the
+ * AIL first to trigger writeback of dirty inodes. This enables writeback to be
+ * done in the background in a non-blocking manner, and enables memory reclaim
+ * to make progress without blocking.
*/
-STATIC int
+static void
xfs_reclaim_inode(
struct xfs_inode *ip,
- struct xfs_perag *pag,
- int sync_mode)
+ struct xfs_perag *pag)
{
- struct xfs_buf *bp = NULL;
xfs_ino_t ino = ip->i_ino; /* for radix_tree_delete */
- int error;
-restart:
- error = 0;
- xfs_ilock(ip, XFS_ILOCK_EXCL);
- if (!xfs_iflock_nowait(ip)) {
- if (!(sync_mode & SYNC_WAIT))
- goto out;
- xfs_iflock(ip);
- }
+ if (!xfs_ilock_nowait(ip, XFS_ILOCK_EXCL))
+ goto out;
+ if (!xfs_iflock_nowait(ip))
+ goto out_iunlock;
if (XFS_FORCED_SHUTDOWN(ip->i_mount)) {
xfs_iunpin_wait(ip);
@@ -1136,52 +1044,12 @@
xfs_iflush_abort(ip);
goto reclaim;
}
- if (xfs_ipincount(ip)) {
- if (!(sync_mode & SYNC_WAIT))
- goto out_ifunlock;
- xfs_iunpin_wait(ip);
- }
- if (xfs_iflags_test(ip, XFS_ISTALE) || xfs_inode_clean(ip)) {
- xfs_ifunlock(ip);
- goto reclaim;
- }
-
- /*
- * Never flush out dirty data during non-blocking reclaim, as it would
- * just contend with AIL pushing trying to do the same job.
- */
- if (!(sync_mode & SYNC_WAIT))
+ if (xfs_ipincount(ip))
+ goto out_ifunlock;
+ if (!xfs_inode_clean(ip))
goto out_ifunlock;
- /*
- * Now we have an inode that needs flushing.
- *
- * Note that xfs_iflush will never block on the inode buffer lock, as
- * xfs_ifree_cluster() can lock the inode buffer before it locks the
- * ip->i_lock, and we are doing the exact opposite here. As a result,
- * doing a blocking xfs_imap_to_bp() to get the cluster buffer would
- * result in an ABBA deadlock with xfs_ifree_cluster().
- *
- * As xfs_ifree_cluser() must gather all inodes that are active in the
- * cache to mark them stale, if we hit this case we don't actually want
- * to do IO here - we want the inode marked stale so we can simply
- * reclaim it. Hence if we get an EAGAIN error here, just unlock the
- * inode, back off and try again. Hopefully the next pass through will
- * see the stale flag set on the inode.
- */
- error = xfs_iflush(ip, &bp);
- if (error == -EAGAIN) {
- xfs_iunlock(ip, XFS_ILOCK_EXCL);
- /* backoff longer than in xfs_ifree_cluster */
- delay(2);
- goto restart;
- }
-
- if (!error) {
- error = xfs_bwrite(bp);
- xfs_buf_relse(bp);
- }
-
+ xfs_ifunlock(ip);
reclaim:
ASSERT(!xfs_isiflocked(ip));
@@ -1228,23 +1096,17 @@
xfs_ilock(ip, XFS_ILOCK_EXCL);
xfs_qm_dqdetach(ip);
xfs_iunlock(ip, XFS_ILOCK_EXCL);
+ ASSERT(xfs_inode_clean(ip));
__xfs_inode_free(ip);
- return error;
+ return;
out_ifunlock:
xfs_ifunlock(ip);
+out_iunlock:
+ xfs_iunlock(ip, XFS_ILOCK_EXCL);
out:
xfs_iflags_clear(ip, XFS_IRECLAIM);
- xfs_iunlock(ip, XFS_ILOCK_EXCL);
- /*
- * We could return -EAGAIN here to make reclaim rescan the inode tree in
- * a short while. However, this just burns CPU time scanning the tree
- * waiting for IO to complete and the reclaim work never goes back to
- * the idle state. Instead, return 0 to let the next scheduled
- * background reclaim attempt to reclaim the inode again.
- */
- return 0;
}
/*
@@ -1252,23 +1114,19 @@
* corrupted, we still want to try to reclaim all the inodes. If we don't,
* then a shut down during filesystem unmount reclaim walk leak all the
* unreclaimed inodes.
+ *
+ * Returns non-zero if any AGs or inodes were skipped in the reclaim pass
+ * so that callers that want to block until all dirty inodes are written back
+ * and reclaimed can sanely loop.
*/
-STATIC int
+static void
xfs_reclaim_inodes_ag(
struct xfs_mount *mp,
- int flags,
int *nr_to_scan)
{
struct xfs_perag *pag;
- int error = 0;
- int last_error = 0;
- xfs_agnumber_t ag;
- int trylock = flags & SYNC_TRYLOCK;
- int skipped;
+ xfs_agnumber_t ag = 0;
-restart:
- ag = 0;
- skipped = 0;
while ((pag = xfs_perag_get_tag(mp, ag, XFS_ICI_RECLAIM_TAG))) {
unsigned long first_index = 0;
int done = 0;
@@ -1276,16 +1134,7 @@
ag = pag->pag_agno + 1;
- if (trylock) {
- if (!mutex_trylock(&pag->pag_ici_reclaim_lock)) {
- skipped++;
- xfs_perag_put(pag);
- continue;
- }
- first_index = pag->pag_ici_reclaim_cursor;
- } else
- mutex_lock(&pag->pag_ici_reclaim_lock);
-
+ first_index = READ_ONCE(pag->pag_ici_reclaim_cursor);
do {
struct xfs_inode *batch[XFS_LOOKUP_BATCH];
int i;
@@ -1309,7 +1158,7 @@
for (i = 0; i < nr_found; i++) {
struct xfs_inode *ip = batch[i];
- if (done || xfs_reclaim_inode_grab(ip, flags))
+ if (done || !xfs_reclaim_inode_grab(ip))
batch[i] = NULL;
/*
@@ -1338,59 +1187,39 @@
rcu_read_unlock();
for (i = 0; i < nr_found; i++) {
- if (!batch[i])
- continue;
- error = xfs_reclaim_inode(batch[i], pag, flags);
- if (error && last_error != -EFSCORRUPTED)
- last_error = error;
+ if (batch[i])
+ xfs_reclaim_inode(batch[i], pag);
}
*nr_to_scan -= XFS_LOOKUP_BATCH;
-
cond_resched();
-
} while (nr_found && !done && *nr_to_scan > 0);
- if (trylock && !done)
- pag->pag_ici_reclaim_cursor = first_index;
- else
- pag->pag_ici_reclaim_cursor = 0;
- mutex_unlock(&pag->pag_ici_reclaim_lock);
+ if (done)
+ first_index = 0;
+ WRITE_ONCE(pag->pag_ici_reclaim_cursor, first_index);
xfs_perag_put(pag);
}
-
- /*
- * if we skipped any AG, and we still have scan count remaining, do
- * another pass this time using blocking reclaim semantics (i.e
- * waiting on the reclaim locks and ignoring the reclaim cursors). This
- * ensure that when we get more reclaimers than AGs we block rather
- * than spin trying to execute reclaim.
- */
- if (skipped && (flags & SYNC_WAIT) && *nr_to_scan > 0) {
- trylock = 0;
- goto restart;
- }
- return last_error;
}
-int
+void
xfs_reclaim_inodes(
- xfs_mount_t *mp,
- int mode)
+ struct xfs_mount *mp)
{
int nr_to_scan = INT_MAX;
- return xfs_reclaim_inodes_ag(mp, mode, &nr_to_scan);
+ while (radix_tree_tagged(&mp->m_perag_tree, XFS_ICI_RECLAIM_TAG)) {
+ xfs_ail_push_all_sync(mp->m_ail);
+ xfs_reclaim_inodes_ag(mp, &nr_to_scan);
+ };
}
/*
- * Scan a certain number of inodes for reclaim.
- *
- * When called we make sure that there is a background (fast) inode reclaim in
- * progress, while we will throttle the speed of reclaim via doing synchronous
- * reclaim of inodes. That means if we come across dirty inodes, we wait for
- * them to be cleaned, which we hope will not be very long due to the
- * background walker having already kicked the IO off on those dirty inodes.
+ * The shrinker infrastructure determines how many inodes we should scan for
+ * reclaim. We want as many clean inodes ready to reclaim as possible, so we
+ * push the AIL here. We also want to proactively free up memory if we can to
+ * minimise the amount of work memory reclaim has to do so we kick the
+ * background reclaim if it isn't already scheduled.
*/
long
xfs_reclaim_inodes_nr(
@@ -1401,7 +1230,8 @@
xfs_reclaim_work_queue(mp);
xfs_ail_push_all(mp->m_ail);
- return xfs_reclaim_inodes_ag(mp, SYNC_TRYLOCK | SYNC_WAIT, &nr_to_scan);
+ xfs_reclaim_inodes_ag(mp, &nr_to_scan);
+ return 0;
}
/*
@@ -1498,6 +1328,24 @@
return true;
}
+/*
+ * This is a fast pass over the inode cache to try to get reclaim moving on as
+ * many inodes as possible in a short period of time. It kicks itself every few
+ * seconds, as well as being kicked by the inode cache shrinker when memory
+ * goes low.
+ */
+void
+xfs_reclaim_worker(
+ struct work_struct *work)
+{
+ struct xfs_mount *mp = container_of(to_delayed_work(work),
+ struct xfs_mount, m_reclaim_work);
+ int nr_to_scan = INT_MAX;
+
+ xfs_reclaim_inodes_ag(mp, &nr_to_scan);
+ xfs_reclaim_work_queue(mp);
+}
+
STATIC int
xfs_inode_free_eofblocks(
struct xfs_inode *ip,
@@ -1574,7 +1422,7 @@
eofb.eof_flags = XFS_EOF_FLAGS_UNION|XFS_EOF_FLAGS_SYNC;
if (XFS_IS_UQUOTA_ENFORCED(ip->i_mount)) {
- dq = xfs_inode_dquot(ip, XFS_DQ_USER);
+ dq = xfs_inode_dquot(ip, XFS_DQTYPE_USER);
if (dq && xfs_dquot_lowsp(dq)) {
eofb.eof_uid = VFS_I(ip)->i_uid;
eofb.eof_flags |= XFS_EOF_FLAGS_UID;
@@ -1583,7 +1431,7 @@
}
if (XFS_IS_GQUOTA_ENFORCED(ip->i_mount)) {
- dq = xfs_inode_dquot(ip, XFS_DQ_GROUP);
+ dq = xfs_inode_dquot(ip, XFS_DQTYPE_GROUP);
if (dq && xfs_dquot_lowsp(dq)) {
eofb.eof_gid = VFS_I(ip)->i_gid;
eofb.eof_flags |= XFS_EOF_FLAGS_GID;
diff --git a/fs/xfs/xfs_icache.h b/fs/xfs/xfs_icache.h
index 93b54e7..3a4c8b3 100644
--- a/fs/xfs/xfs_icache.h
+++ b/fs/xfs/xfs_icache.h
@@ -17,9 +17,6 @@
__u64 eof_min_file_size;
};
-#define SYNC_WAIT 0x0001 /* wait for i/o to complete */
-#define SYNC_TRYLOCK 0x0002 /* only try to lock inodes */
-
/*
* tags for inode radix tree
*/
@@ -51,7 +48,7 @@
void xfs_reclaim_worker(struct work_struct *work);
-int xfs_reclaim_inodes(struct xfs_mount *mp, int mode);
+void xfs_reclaim_inodes(struct xfs_mount *mp);
int xfs_reclaim_inodes_count(struct xfs_mount *mp);
long xfs_reclaim_inodes_nr(struct xfs_mount *mp, int nr_to_scan);
diff --git a/fs/xfs/xfs_icreate_item.c b/fs/xfs/xfs_icreate_item.c
index 287a9e5..9b3994b 100644
--- a/fs/xfs/xfs_icreate_item.c
+++ b/fs/xfs/xfs_icreate_item.c
@@ -97,7 +97,7 @@
{
struct xfs_icreate_item *icp;
- icp = kmem_zone_zalloc(xfs_icreate_zone, 0);
+ icp = kmem_cache_zalloc(xfs_icreate_zone, GFP_KERNEL | __GFP_NOFAIL);
xfs_log_item_init(tp->t_mountp, &icp->ic_item, XFS_LI_ICREATE,
&xfs_icreate_item_ops);
diff --git a/fs/xfs/xfs_inode.c b/fs/xfs/xfs_inode.c
index 9aea7d6..407d629 100644
--- a/fs/xfs/xfs_inode.c
+++ b/fs/xfs/xfs_inode.c
@@ -44,7 +44,6 @@
*/
#define XFS_ITRUNC_MAX_EXTENTS 2
-STATIC int xfs_iflush_int(struct xfs_inode *, struct xfs_buf *);
STATIC int xfs_iunlink(struct xfs_trans *, struct xfs_inode *);
STATIC int xfs_iunlink_remove(struct xfs_trans *, struct xfs_inode *);
@@ -1740,10 +1739,31 @@
return error;
}
+ /*
+ * We do not hold the inode locked across the entire rolling transaction
+ * here. We only need to hold it for the first transaction that
+ * xfs_ifree() builds, which may mark the inode XFS_ISTALE if the
+ * underlying cluster buffer is freed. Relogging an XFS_ISTALE inode
+ * here breaks the relationship between cluster buffer invalidation and
+ * stale inode invalidation on cluster buffer item journal commit
+ * completion, and can result in leaving dirty stale inodes hanging
+ * around in memory.
+ *
+ * We have no need for serialising this inode operation against other
+ * operations - we freed the inode and hence reallocation is required
+ * and that will serialise on reallocating the space the deferops need
+ * to free. Hence we can unlock the inode on the first commit of
+ * the transaction rather than roll it right through the deferops. This
+ * avoids relogging the XFS_ISTALE inode.
+ *
+ * We check that xfs_ifree() hasn't grown an internal transaction roll
+ * by asserting that the inode is still locked when it returns.
+ */
xfs_ilock(ip, XFS_ILOCK_EXCL);
- xfs_trans_ijoin(tp, ip, 0);
+ xfs_trans_ijoin(tp, ip, XFS_ILOCK_EXCL);
error = xfs_ifree(tp, ip);
+ ASSERT(xfs_isilocked(ip, XFS_ILOCK_EXCL));
if (error) {
/*
* If we fail to free the inode, shut down. The cancel
@@ -1756,7 +1776,6 @@
xfs_force_shutdown(mp, SHUTDOWN_META_IO_ERROR);
}
xfs_trans_cancel(tp);
- xfs_iunlock(ip, XFS_ILOCK_EXCL);
return error;
}
@@ -1774,7 +1793,6 @@
xfs_notice(mp, "%s: xfs_trans_commit returned error %d",
__func__, error);
- xfs_iunlock(ip, XFS_ILOCK_EXCL);
return 0;
}
@@ -2147,7 +2165,6 @@
xfs_dinode_calc_crc(mp, dip);
xfs_trans_inode_buf(tp, ibp);
xfs_trans_log_buf(tp, ibp, offset, offset + sizeof(xfs_agino_t) - 1);
- xfs_inobp_check(mp, ibp);
}
/* Set an in-core inode's unlinked pointer and return the old value. */
@@ -2248,7 +2265,6 @@
}
if (next_agino != NULLAGINO) {
- struct xfs_perag *pag;
xfs_agino_t old_agino;
/*
@@ -2265,9 +2281,7 @@
* agino has been unlinked, add a backref from the next inode
* back to agino.
*/
- pag = xfs_perag_get(mp, agno);
- error = xfs_iunlink_add_backref(pag, agino, next_agino);
- xfs_perag_put(pag);
+ error = xfs_iunlink_add_backref(agibp->b_pag, agino, next_agino);
if (error)
return error;
}
@@ -2403,7 +2417,6 @@
struct xfs_buf *agibp;
struct xfs_buf *last_ibp;
struct xfs_dinode *last_dip = NULL;
- struct xfs_perag *pag = NULL;
xfs_agnumber_t agno = XFS_INO_TO_AGNO(mp, ip->i_ino);
xfs_agino_t agino = XFS_INO_TO_AGINO(mp, ip->i_ino);
xfs_agino_t next_agino;
@@ -2447,32 +2460,22 @@
* this inode's backref to point from the next inode.
*/
if (next_agino != NULLAGINO) {
- pag = xfs_perag_get(mp, agno);
- error = xfs_iunlink_change_backref(pag, next_agino,
+ error = xfs_iunlink_change_backref(agibp->b_pag, next_agino,
NULLAGINO);
if (error)
- goto out;
+ return error;
}
- if (head_agino == agino) {
- /* Point the head of the list to the next unlinked inode. */
- error = xfs_iunlink_update_bucket(tp, agno, agibp, bucket_index,
- next_agino);
- if (error)
- goto out;
- } else {
+ if (head_agino != agino) {
struct xfs_imap imap;
xfs_agino_t prev_agino;
- if (!pag)
- pag = xfs_perag_get(mp, agno);
-
/* We need to search the list for the inode being freed. */
error = xfs_iunlink_map_prev(tp, agno, head_agino, agino,
&prev_agino, &imap, &last_dip, &last_ibp,
- pag);
+ agibp->b_pag);
if (error)
- goto out;
+ return error;
/* Point the previous inode on the list to the next inode. */
xfs_iunlink_update_dinode(tp, agno, prev_agino, last_ibp,
@@ -2486,29 +2489,29 @@
* change_backref takes care of deleting the backref if
* next_agino is NULLAGINO.
*/
- error = xfs_iunlink_change_backref(pag, agino, next_agino);
- if (error)
- goto out;
+ return xfs_iunlink_change_backref(agibp->b_pag, agino,
+ next_agino);
}
-out:
- if (pag)
- xfs_perag_put(pag);
- return error;
+ /* Point the head of the list to the next unlinked inode. */
+ return xfs_iunlink_update_bucket(tp, agno, agibp, bucket_index,
+ next_agino);
}
/*
- * Look up the inode number specified and mark it stale if it is found. If it is
- * dirty, return the inode so it can be attached to the cluster buffer so it can
- * be processed appropriately when the cluster free transaction completes.
+ * Look up the inode number specified and if it is not already marked XFS_ISTALE
+ * mark it stale. We should only find clean inodes in this lookup that aren't
+ * already stale.
*/
-static struct xfs_inode *
-xfs_ifree_get_one_inode(
- struct xfs_perag *pag,
+static void
+xfs_ifree_mark_inode_stale(
+ struct xfs_buf *bp,
struct xfs_inode *free_ip,
xfs_ino_t inum)
{
- struct xfs_mount *mp = pag->pag_mount;
+ struct xfs_mount *mp = bp->b_mount;
+ struct xfs_perag *pag = bp->b_pag;
+ struct xfs_inode_log_item *iip;
struct xfs_inode *ip;
retry:
@@ -2516,8 +2519,10 @@
ip = radix_tree_lookup(&pag->pag_ici_root, XFS_INO_TO_AGINO(mp, inum));
/* Inode not in memory, nothing to do */
- if (!ip)
- goto out_rcu_unlock;
+ if (!ip) {
+ rcu_read_unlock();
+ return;
+ }
/*
* because this is an RCU protected lookup, we could find a recently
@@ -2528,9 +2533,9 @@
spin_lock(&ip->i_flags_lock);
if (ip->i_ino != inum || __xfs_iflags_test(ip, XFS_ISTALE)) {
spin_unlock(&ip->i_flags_lock);
- goto out_rcu_unlock;
+ rcu_read_unlock();
+ return;
}
- spin_unlock(&ip->i_flags_lock);
/*
* Don't try to lock/unlock the current inode, but we _cannot_ skip the
@@ -2540,43 +2545,50 @@
*/
if (ip != free_ip) {
if (!xfs_ilock_nowait(ip, XFS_ILOCK_EXCL)) {
+ spin_unlock(&ip->i_flags_lock);
rcu_read_unlock();
delay(1);
goto retry;
}
-
- /*
- * Check the inode number again in case we're racing with
- * freeing in xfs_reclaim_inode(). See the comments in that
- * function for more information as to why the initial check is
- * not sufficient.
- */
- if (ip->i_ino != inum) {
- xfs_iunlock(ip, XFS_ILOCK_EXCL);
- goto out_rcu_unlock;
- }
}
+ ip->i_flags |= XFS_ISTALE;
+ spin_unlock(&ip->i_flags_lock);
rcu_read_unlock();
- xfs_iflock(ip);
- xfs_iflags_set(ip, XFS_ISTALE);
-
/*
- * We don't need to attach clean inodes or those only with unlogged
- * changes (which we throw away, anyway).
+ * If we can't get the flush lock, the inode is already attached. All
+ * we needed to do here is mark the inode stale so buffer IO completion
+ * will remove it from the AIL.
*/
- if (!ip->i_itemp || xfs_inode_clean(ip)) {
- ASSERT(ip != free_ip);
- xfs_ifunlock(ip);
- xfs_iunlock(ip, XFS_ILOCK_EXCL);
- goto out_no_inode;
+ iip = ip->i_itemp;
+ if (!xfs_iflock_nowait(ip)) {
+ ASSERT(!list_empty(&iip->ili_item.li_bio_list));
+ ASSERT(iip->ili_last_fields);
+ goto out_iunlock;
}
- return ip;
-out_rcu_unlock:
- rcu_read_unlock();
-out_no_inode:
- return NULL;
+ /*
+ * Inodes not attached to the buffer can be released immediately.
+ * Everything else has to go through xfs_iflush_abort() on journal
+ * commit as the flock synchronises removal of the inode from the
+ * cluster buffer against inode reclaim.
+ */
+ if (!iip || list_empty(&iip->ili_item.li_bio_list)) {
+ xfs_ifunlock(ip);
+ goto out_iunlock;
+ }
+
+ /* we have a dirty inode in memory that has not yet been flushed. */
+ spin_lock(&iip->ili_lock);
+ iip->ili_last_fields = iip->ili_fields;
+ iip->ili_fields = 0;
+ iip->ili_fsync_fields = 0;
+ spin_unlock(&iip->ili_lock);
+ ASSERT(iip->ili_last_fields);
+
+out_iunlock:
+ if (ip != free_ip)
+ xfs_iunlock(ip, XFS_ILOCK_EXCL);
}
/*
@@ -2586,26 +2598,20 @@
*/
STATIC int
xfs_ifree_cluster(
- xfs_inode_t *free_ip,
- xfs_trans_t *tp,
+ struct xfs_inode *free_ip,
+ struct xfs_trans *tp,
struct xfs_icluster *xic)
{
- xfs_mount_t *mp = free_ip->i_mount;
+ struct xfs_mount *mp = free_ip->i_mount;
+ struct xfs_ino_geometry *igeo = M_IGEO(mp);
+ struct xfs_buf *bp;
+ xfs_daddr_t blkno;
+ xfs_ino_t inum = xic->first_ino;
int nbufs;
int i, j;
int ioffset;
- xfs_daddr_t blkno;
- xfs_buf_t *bp;
- xfs_inode_t *ip;
- struct xfs_inode_log_item *iip;
- struct xfs_log_item *lip;
- struct xfs_perag *pag;
- struct xfs_ino_geometry *igeo = M_IGEO(mp);
- xfs_ino_t inum;
int error;
- inum = xic->first_ino;
- pag = xfs_perag_get(mp, XFS_INO_TO_AGNO(mp, inum));
nbufs = igeo->ialloc_blks / igeo->blocks_per_cluster;
for (j = 0; j < nbufs; j++, inum += igeo->inodes_per_cluster) {
@@ -2634,10 +2640,8 @@
error = xfs_trans_get_buf(tp, mp->m_ddev_targp, blkno,
mp->m_bsize * igeo->blocks_per_cluster,
XBF_UNMAPPED, &bp);
- if (error) {
- xfs_perag_put(pag);
+ if (error)
return error;
- }
/*
* This buffer may not have been correctly initialised as we
@@ -2651,60 +2655,16 @@
bp->b_ops = &xfs_inode_buf_ops;
/*
- * Walk the inodes already attached to the buffer and mark them
- * stale. These will all have the flush locks held, so an
- * in-memory inode walk can't lock them. By marking them all
- * stale first, we will not attempt to lock them in the loop
- * below as the XFS_ISTALE flag will be set.
+ * Now we need to set all the cached clean inodes as XFS_ISTALE,
+ * too. This requires lookups, and will skip inodes that we've
+ * already marked XFS_ISTALE.
*/
- list_for_each_entry(lip, &bp->b_li_list, li_bio_list) {
- if (lip->li_type == XFS_LI_INODE) {
- iip = (struct xfs_inode_log_item *)lip;
- ASSERT(iip->ili_logged == 1);
- lip->li_cb = xfs_istale_done;
- xfs_trans_ail_copy_lsn(mp->m_ail,
- &iip->ili_flush_lsn,
- &iip->ili_item.li_lsn);
- xfs_iflags_set(iip->ili_inode, XFS_ISTALE);
- }
- }
-
-
- /*
- * For each inode in memory attempt to add it to the inode
- * buffer and set it up for being staled on buffer IO
- * completion. This is safe as we've locked out tail pushing
- * and flushing by locking the buffer.
- *
- * We have already marked every inode that was part of a
- * transaction stale above, which means there is no point in
- * even trying to lock them.
- */
- for (i = 0; i < igeo->inodes_per_cluster; i++) {
- ip = xfs_ifree_get_one_inode(pag, free_ip, inum + i);
- if (!ip)
- continue;
-
- iip = ip->i_itemp;
- iip->ili_last_fields = iip->ili_fields;
- iip->ili_fields = 0;
- iip->ili_fsync_fields = 0;
- iip->ili_logged = 1;
- xfs_trans_ail_copy_lsn(mp->m_ail, &iip->ili_flush_lsn,
- &iip->ili_item.li_lsn);
-
- xfs_buf_attach_iodone(bp, xfs_istale_done,
- &iip->ili_item);
-
- if (ip != free_ip)
- xfs_iunlock(ip, XFS_ILOCK_EXCL);
- }
+ for (i = 0; i < igeo->inodes_per_cluster; i++)
+ xfs_ifree_mark_inode_stale(bp, free_ip, inum + i);
xfs_trans_stale_inode_buf(tp, bp);
xfs_trans_binval(tp, bp);
}
-
- xfs_perag_put(pag);
return 0;
}
@@ -2725,6 +2685,7 @@
{
int error;
struct xfs_icluster xic = { 0 };
+ struct xfs_inode_log_item *iip = ip->i_itemp;
ASSERT(xfs_isilocked(ip, XFS_ILOCK_EXCL));
ASSERT(VFS_I(ip)->i_nlink == 0);
@@ -2762,7 +2723,9 @@
ip->i_df.if_format = XFS_DINODE_FMT_EXTENTS;
/* Don't attempt to replay owner changes for a deleted inode */
- ip->i_itemp->ili_fields &= ~(XFS_ILOG_AOWNER|XFS_ILOG_DOWNER);
+ spin_lock(&iip->ili_lock);
+ iip->ili_fields &= ~(XFS_ILOG_AOWNER | XFS_ILOG_DOWNER);
+ spin_unlock(&iip->ili_lock);
/*
* Bump the generation count so no one will be confused
@@ -3469,232 +3432,9 @@
return error;
}
-STATIC int
-xfs_iflush_cluster(
- struct xfs_inode *ip,
- struct xfs_buf *bp)
-{
- struct xfs_mount *mp = ip->i_mount;
- struct xfs_perag *pag;
- unsigned long first_index, mask;
- int cilist_size;
- struct xfs_inode **cilist;
- struct xfs_inode *cip;
- struct xfs_ino_geometry *igeo = M_IGEO(mp);
- int error = 0;
- int nr_found;
- int clcount = 0;
- int i;
-
- pag = xfs_perag_get(mp, XFS_INO_TO_AGNO(mp, ip->i_ino));
-
- cilist_size = igeo->inodes_per_cluster * sizeof(struct xfs_inode *);
- cilist = kmem_alloc(cilist_size, KM_MAYFAIL|KM_NOFS);
- if (!cilist)
- goto out_put;
-
- mask = ~(igeo->inodes_per_cluster - 1);
- first_index = XFS_INO_TO_AGINO(mp, ip->i_ino) & mask;
- rcu_read_lock();
- /* really need a gang lookup range call here */
- nr_found = radix_tree_gang_lookup(&pag->pag_ici_root, (void**)cilist,
- first_index, igeo->inodes_per_cluster);
- if (nr_found == 0)
- goto out_free;
-
- for (i = 0; i < nr_found; i++) {
- cip = cilist[i];
- if (cip == ip)
- continue;
-
- /*
- * because this is an RCU protected lookup, we could find a
- * recently freed or even reallocated inode during the lookup.
- * We need to check under the i_flags_lock for a valid inode
- * here. Skip it if it is not valid or the wrong inode.
- */
- spin_lock(&cip->i_flags_lock);
- if (!cip->i_ino ||
- __xfs_iflags_test(cip, XFS_ISTALE)) {
- spin_unlock(&cip->i_flags_lock);
- continue;
- }
-
- /*
- * Once we fall off the end of the cluster, no point checking
- * any more inodes in the list because they will also all be
- * outside the cluster.
- */
- if ((XFS_INO_TO_AGINO(mp, cip->i_ino) & mask) != first_index) {
- spin_unlock(&cip->i_flags_lock);
- break;
- }
- spin_unlock(&cip->i_flags_lock);
-
- /*
- * Do an un-protected check to see if the inode is dirty and
- * is a candidate for flushing. These checks will be repeated
- * later after the appropriate locks are acquired.
- */
- if (xfs_inode_clean(cip) && xfs_ipincount(cip) == 0)
- continue;
-
- /*
- * Try to get locks. If any are unavailable or it is pinned,
- * then this inode cannot be flushed and is skipped.
- */
-
- if (!xfs_ilock_nowait(cip, XFS_ILOCK_SHARED))
- continue;
- if (!xfs_iflock_nowait(cip)) {
- xfs_iunlock(cip, XFS_ILOCK_SHARED);
- continue;
- }
- if (xfs_ipincount(cip)) {
- xfs_ifunlock(cip);
- xfs_iunlock(cip, XFS_ILOCK_SHARED);
- continue;
- }
-
-
- /*
- * Check the inode number again, just to be certain we are not
- * racing with freeing in xfs_reclaim_inode(). See the comments
- * in that function for more information as to why the initial
- * check is not sufficient.
- */
- if (!cip->i_ino) {
- xfs_ifunlock(cip);
- xfs_iunlock(cip, XFS_ILOCK_SHARED);
- continue;
- }
-
- /*
- * arriving here means that this inode can be flushed. First
- * re-check that it's dirty before flushing.
- */
- if (!xfs_inode_clean(cip)) {
- error = xfs_iflush_int(cip, bp);
- if (error) {
- xfs_iunlock(cip, XFS_ILOCK_SHARED);
- goto out_free;
- }
- clcount++;
- } else {
- xfs_ifunlock(cip);
- }
- xfs_iunlock(cip, XFS_ILOCK_SHARED);
- }
-
- if (clcount) {
- XFS_STATS_INC(mp, xs_icluster_flushcnt);
- XFS_STATS_ADD(mp, xs_icluster_flushinode, clcount);
- }
-
-out_free:
- rcu_read_unlock();
- kmem_free(cilist);
-out_put:
- xfs_perag_put(pag);
- return error;
-}
-
-/*
- * Flush dirty inode metadata into the backing buffer.
- *
- * The caller must have the inode lock and the inode flush lock held. The
- * inode lock will still be held upon return to the caller, and the inode
- * flush lock will be released after the inode has reached the disk.
- *
- * The caller must write out the buffer returned in *bpp and release it.
- */
-int
+static int
xfs_iflush(
struct xfs_inode *ip,
- struct xfs_buf **bpp)
-{
- struct xfs_mount *mp = ip->i_mount;
- struct xfs_buf *bp = NULL;
- struct xfs_dinode *dip;
- int error;
-
- XFS_STATS_INC(mp, xs_iflush_count);
-
- ASSERT(xfs_isilocked(ip, XFS_ILOCK_EXCL|XFS_ILOCK_SHARED));
- ASSERT(xfs_isiflocked(ip));
- ASSERT(ip->i_df.if_format != XFS_DINODE_FMT_BTREE ||
- ip->i_df.if_nextents > XFS_IFORK_MAXEXT(ip, XFS_DATA_FORK));
-
- *bpp = NULL;
-
- xfs_iunpin_wait(ip);
-
- /*
- * For stale inodes we cannot rely on the backing buffer remaining
- * stale in cache for the remaining life of the stale inode and so
- * xfs_imap_to_bp() below may give us a buffer that no longer contains
- * inodes below. We have to check this after ensuring the inode is
- * unpinned so that it is safe to reclaim the stale inode after the
- * flush call.
- */
- if (xfs_iflags_test(ip, XFS_ISTALE)) {
- xfs_ifunlock(ip);
- return 0;
- }
-
- /*
- * Get the buffer containing the on-disk inode. We are doing a try-lock
- * operation here, so we may get an EAGAIN error. In that case, return
- * leaving the inode dirty.
- *
- * If we get any other error, we effectively have a corruption situation
- * and we cannot flush the inode. Abort the flush and shut down.
- */
- error = xfs_imap_to_bp(mp, NULL, &ip->i_imap, &dip, &bp, XBF_TRYLOCK);
- if (error == -EAGAIN) {
- xfs_ifunlock(ip);
- return error;
- }
- if (error)
- goto abort;
-
- /*
- * If the buffer is pinned then push on the log now so we won't
- * get stuck waiting in the write for too long.
- */
- if (xfs_buf_ispinned(bp))
- xfs_log_force(mp, 0);
-
- /*
- * Flush the provided inode then attempt to gather others from the
- * cluster into the write.
- *
- * Note: Once we attempt to flush an inode, we must run buffer
- * completion callbacks on any failure. If this fails, simulate an I/O
- * failure on the buffer and shut down.
- */
- error = xfs_iflush_int(ip, bp);
- if (!error)
- error = xfs_iflush_cluster(ip, bp);
- if (error) {
- bp->b_flags |= XBF_ASYNC;
- xfs_buf_ioend_fail(bp);
- goto shutdown;
- }
-
- *bpp = bp;
- return 0;
-
-abort:
- xfs_iflush_abort(ip);
-shutdown:
- xfs_force_shutdown(mp, SHUTDOWN_CORRUPT_INCORE);
- return error;
-}
-
-STATIC int
-xfs_iflush_int(
- struct xfs_inode *ip,
struct xfs_buf *bp)
{
struct xfs_inode_log_item *iip = ip->i_itemp;
@@ -3706,7 +3446,7 @@
ASSERT(xfs_isiflocked(ip));
ASSERT(ip->i_df.if_format != XFS_DINODE_FMT_BTREE ||
ip->i_df.if_nextents > XFS_IFORK_MAXEXT(ip, XFS_DATA_FORK));
- ASSERT(iip != NULL && iip->ili_fields != 0);
+ ASSERT(iip->ili_item.li_buf == bp);
dip = xfs_buf_offset(bp, ip->i_imap.im_boffset);
@@ -3801,7 +3541,6 @@
xfs_iflush_fork(ip, dip, iip, XFS_DATA_FORK);
if (XFS_IFORK_Q(ip))
xfs_iflush_fork(ip, dip, iip, XFS_ATTR_FORK);
- xfs_inobp_check(mp, bp);
/*
* We've recorded everything logged in the inode, so we'd like to clear
@@ -3818,42 +3557,150 @@
* know that the information those bits represent is permanently on
* disk. As long as the flush completes before the inode is logged
* again, then both ili_fields and ili_last_fields will be cleared.
- *
- * We can play with the ili_fields bits here, because the inode lock
- * must be held exclusively in order to set bits there and the flush
- * lock protects the ili_last_fields bits. Set ili_logged so the flush
- * done routine can tell whether or not to look in the AIL. Also, store
- * the current LSN of the inode so that we can tell whether the item has
- * moved in the AIL from xfs_iflush_done(). In order to read the lsn we
- * need the AIL lock, because it is a 64 bit value that cannot be read
- * atomically.
*/
error = 0;
flush_out:
+ spin_lock(&iip->ili_lock);
iip->ili_last_fields = iip->ili_fields;
iip->ili_fields = 0;
iip->ili_fsync_fields = 0;
- iip->ili_logged = 1;
+ spin_unlock(&iip->ili_lock);
+ /*
+ * Store the current LSN of the inode so that we can tell whether the
+ * item has moved in the AIL from xfs_iflush_done().
+ */
xfs_trans_ail_copy_lsn(mp->m_ail, &iip->ili_flush_lsn,
&iip->ili_item.li_lsn);
- /*
- * Attach the inode item callback to the buffer whether the flush
- * succeeded or not. If not, the caller will shut down and fail I/O
- * completion on the buffer to remove the inode from the AIL and release
- * the flush lock.
- */
- xfs_buf_attach_iodone(bp, xfs_iflush_done, &iip->ili_item);
-
/* generate the checksum. */
xfs_dinode_calc_crc(mp, dip);
-
- ASSERT(!list_empty(&bp->b_li_list));
- ASSERT(bp->b_iodone != NULL);
return error;
}
+/*
+ * Non-blocking flush of dirty inode metadata into the backing buffer.
+ *
+ * The caller must have a reference to the inode and hold the cluster buffer
+ * locked. The function will walk across all the inodes on the cluster buffer it
+ * can find and lock without blocking, and flush them to the cluster buffer.
+ *
+ * On successful flushing of at least one inode, the caller must write out the
+ * buffer and release it. If no inodes are flushed, -EAGAIN will be returned and
+ * the caller needs to release the buffer. On failure, the filesystem will be
+ * shut down, the buffer will have been unlocked and released, and EFSCORRUPTED
+ * will be returned.
+ */
+int
+xfs_iflush_cluster(
+ struct xfs_buf *bp)
+{
+ struct xfs_mount *mp = bp->b_mount;
+ struct xfs_log_item *lip, *n;
+ struct xfs_inode *ip;
+ struct xfs_inode_log_item *iip;
+ int clcount = 0;
+ int error = 0;
+
+ /*
+ * We must use the safe variant here as on shutdown xfs_iflush_abort()
+ * can remove itself from the list.
+ */
+ list_for_each_entry_safe(lip, n, &bp->b_li_list, li_bio_list) {
+ iip = (struct xfs_inode_log_item *)lip;
+ ip = iip->ili_inode;
+
+ /*
+ * Quick and dirty check to avoid locks if possible.
+ */
+ if (__xfs_iflags_test(ip, XFS_IRECLAIM | XFS_IFLOCK))
+ continue;
+ if (xfs_ipincount(ip))
+ continue;
+
+ /*
+ * The inode is still attached to the buffer, which means it is
+ * dirty but reclaim might try to grab it. Check carefully for
+ * that, and grab the ilock while still holding the i_flags_lock
+ * to guarantee reclaim will not be able to reclaim this inode
+ * once we drop the i_flags_lock.
+ */
+ spin_lock(&ip->i_flags_lock);
+ ASSERT(!__xfs_iflags_test(ip, XFS_ISTALE));
+ if (__xfs_iflags_test(ip, XFS_IRECLAIM | XFS_IFLOCK)) {
+ spin_unlock(&ip->i_flags_lock);
+ continue;
+ }
+
+ /*
+ * ILOCK will pin the inode against reclaim and prevent
+ * concurrent transactions modifying the inode while we are
+ * flushing the inode.
+ */
+ if (!xfs_ilock_nowait(ip, XFS_ILOCK_SHARED)) {
+ spin_unlock(&ip->i_flags_lock);
+ continue;
+ }
+ spin_unlock(&ip->i_flags_lock);
+
+ /*
+ * Skip inodes that are already flush locked as they have
+ * already been written to the buffer.
+ */
+ if (!xfs_iflock_nowait(ip)) {
+ xfs_iunlock(ip, XFS_ILOCK_SHARED);
+ continue;
+ }
+
+ /*
+ * Abort flushing this inode if we are shut down because the
+ * inode may not currently be in the AIL. This can occur when
+ * log I/O failure unpins the inode without inserting into the
+ * AIL, leaving a dirty/unpinned inode attached to the buffer
+ * that otherwise looks like it should be flushed.
+ */
+ if (XFS_FORCED_SHUTDOWN(mp)) {
+ xfs_iunpin_wait(ip);
+ /* xfs_iflush_abort() drops the flush lock */
+ xfs_iflush_abort(ip);
+ xfs_iunlock(ip, XFS_ILOCK_SHARED);
+ error = -EIO;
+ continue;
+ }
+
+ /* don't block waiting on a log force to unpin dirty inodes */
+ if (xfs_ipincount(ip)) {
+ xfs_ifunlock(ip);
+ xfs_iunlock(ip, XFS_ILOCK_SHARED);
+ continue;
+ }
+
+ if (!xfs_inode_clean(ip))
+ error = xfs_iflush(ip, bp);
+ else
+ xfs_ifunlock(ip);
+ xfs_iunlock(ip, XFS_ILOCK_SHARED);
+ if (error)
+ break;
+ clcount++;
+ }
+
+ if (error) {
+ bp->b_flags |= XBF_ASYNC;
+ xfs_buf_ioend_fail(bp);
+ xfs_force_shutdown(mp, SHUTDOWN_CORRUPT_INCORE);
+ return error;
+ }
+
+ if (!clcount)
+ return -EAGAIN;
+
+ XFS_STATS_INC(mp, xs_icluster_flushcnt);
+ XFS_STATS_ADD(mp, xs_icluster_flushinode, clcount);
+ return 0;
+
+}
+
/* Release an inode. */
void
xfs_irele(
@@ -3881,3 +3728,96 @@
return 0;
return xfs_log_force_lsn(ip->i_mount, lsn, XFS_LOG_SYNC, NULL);
}
+
+/*
+ * Grab the exclusive iolock for a data copy from src to dest, making sure to
+ * abide vfs locking order (lowest pointer value goes first) and breaking the
+ * layout leases before proceeding. The loop is needed because we cannot call
+ * the blocking break_layout() with the iolocks held, and therefore have to
+ * back out both locks.
+ */
+static int
+xfs_iolock_two_inodes_and_break_layout(
+ struct inode *src,
+ struct inode *dest)
+{
+ int error;
+
+ if (src > dest)
+ swap(src, dest);
+
+retry:
+ /* Wait to break both inodes' layouts before we start locking. */
+ error = break_layout(src, true);
+ if (error)
+ return error;
+ if (src != dest) {
+ error = break_layout(dest, true);
+ if (error)
+ return error;
+ }
+
+ /* Lock one inode and make sure nobody got in and leased it. */
+ inode_lock(src);
+ error = break_layout(src, false);
+ if (error) {
+ inode_unlock(src);
+ if (error == -EWOULDBLOCK)
+ goto retry;
+ return error;
+ }
+
+ if (src == dest)
+ return 0;
+
+ /* Lock the other inode and make sure nobody got in and leased it. */
+ inode_lock_nested(dest, I_MUTEX_NONDIR2);
+ error = break_layout(dest, false);
+ if (error) {
+ inode_unlock(src);
+ inode_unlock(dest);
+ if (error == -EWOULDBLOCK)
+ goto retry;
+ return error;
+ }
+
+ return 0;
+}
+
+/*
+ * Lock two inodes so that userspace cannot initiate I/O via file syscalls or
+ * mmap activity.
+ */
+int
+xfs_ilock2_io_mmap(
+ struct xfs_inode *ip1,
+ struct xfs_inode *ip2)
+{
+ int ret;
+
+ ret = xfs_iolock_two_inodes_and_break_layout(VFS_I(ip1), VFS_I(ip2));
+ if (ret)
+ return ret;
+ if (ip1 == ip2)
+ xfs_ilock(ip1, XFS_MMAPLOCK_EXCL);
+ else
+ xfs_lock_two_inodes(ip1, XFS_MMAPLOCK_EXCL,
+ ip2, XFS_MMAPLOCK_EXCL);
+ return 0;
+}
+
+/* Unlock both inodes to allow IO and mmap activity. */
+void
+xfs_iunlock2_io_mmap(
+ struct xfs_inode *ip1,
+ struct xfs_inode *ip2)
+{
+ bool same_inode = (ip1 == ip2);
+
+ xfs_iunlock(ip2, XFS_MMAPLOCK_EXCL);
+ if (!same_inode)
+ xfs_iunlock(ip1, XFS_MMAPLOCK_EXCL);
+ inode_unlock(VFS_I(ip2));
+ if (!same_inode)
+ inode_unlock(VFS_I(ip1));
+}
diff --git a/fs/xfs/xfs_inode.h b/fs/xfs/xfs_inode.h
index 47d3b39..e9a8bb1 100644
--- a/fs/xfs/xfs_inode.h
+++ b/fs/xfs/xfs_inode.h
@@ -426,7 +426,7 @@
void xfs_iunpin_wait(xfs_inode_t *);
#define xfs_ipincount(ip) ((unsigned int) atomic_read(&ip->i_pincount))
-int xfs_iflush(struct xfs_inode *, struct xfs_buf **);
+int xfs_iflush_cluster(struct xfs_buf *);
void xfs_lock_two_inodes(struct xfs_inode *ip0, uint ip0_mode,
struct xfs_inode *ip1, uint ip1_mode);
@@ -499,4 +499,7 @@
void xfs_end_io(struct work_struct *work);
+int xfs_ilock2_io_mmap(struct xfs_inode *ip1, struct xfs_inode *ip2);
+void xfs_iunlock2_io_mmap(struct xfs_inode *ip1, struct xfs_inode *ip2);
+
#endif /* __XFS_INODE_H__ */
diff --git a/fs/xfs/xfs_inode_item.c b/fs/xfs/xfs_inode_item.c
index ba47bf6..895f61b 100644
--- a/fs/xfs/xfs_inode_item.c
+++ b/fs/xfs/xfs_inode_item.c
@@ -439,6 +439,7 @@
struct xfs_inode *ip = INODE_ITEM(lip)->ili_inode;
ASSERT(xfs_isilocked(ip, XFS_ILOCK_EXCL));
+ ASSERT(lip->li_buf);
trace_xfs_inode_pin(ip, _RET_IP_);
atomic_inc(&ip->i_pincount);
@@ -450,6 +451,12 @@
* item which was previously pinned with a call to xfs_inode_item_pin().
*
* Also wake up anyone in xfs_iunpin_wait() if the count goes to 0.
+ *
+ * Note that unpin can race with inode cluster buffer freeing marking the buffer
+ * stale. In that case, flush completions are run from the buffer unpin call,
+ * which may happen before the inode is unpinned. If we lose the race, there
+ * will be no buffer attached to the log item, but the inode will be marked
+ * XFS_ISTALE.
*/
STATIC void
xfs_inode_item_unpin(
@@ -459,28 +466,12 @@
struct xfs_inode *ip = INODE_ITEM(lip)->ili_inode;
trace_xfs_inode_unpin(ip, _RET_IP_);
+ ASSERT(lip->li_buf || xfs_iflags_test(ip, XFS_ISTALE));
ASSERT(atomic_read(&ip->i_pincount) > 0);
if (atomic_dec_and_test(&ip->i_pincount))
wake_up_bit(&ip->i_flags, __XFS_IPINNED_BIT);
}
-/*
- * Callback used to mark a buffer with XFS_LI_FAILED when items in the buffer
- * have been failed during writeback
- *
- * This informs the AIL that the inode is already flush locked on the next push,
- * and acquires a hold on the buffer to ensure that it isn't reclaimed before
- * dirty data makes it to disk.
- */
-STATIC void
-xfs_inode_item_error(
- struct xfs_log_item *lip,
- struct xfs_buf *bp)
-{
- ASSERT(xfs_isiflocked(INODE_ITEM(lip)->ili_inode));
- xfs_set_li_failed(lip, bp);
-}
-
STATIC uint
xfs_inode_item_push(
struct xfs_log_item *lip,
@@ -494,55 +485,44 @@
uint rval = XFS_ITEM_SUCCESS;
int error;
- if (xfs_ipincount(ip) > 0)
+ ASSERT(iip->ili_item.li_buf);
+
+ if (xfs_ipincount(ip) > 0 || xfs_buf_ispinned(bp) ||
+ (ip->i_flags & XFS_ISTALE))
return XFS_ITEM_PINNED;
- if (!xfs_ilock_nowait(ip, XFS_ILOCK_SHARED))
+ /* If the inode is already flush locked, we're already flushing. */
+ if (xfs_isiflocked(ip))
+ return XFS_ITEM_FLUSHING;
+
+ if (!xfs_buf_trylock(bp))
return XFS_ITEM_LOCKED;
- /*
- * Re-check the pincount now that we stabilized the value by
- * taking the ilock.
- */
- if (xfs_ipincount(ip) > 0) {
- rval = XFS_ITEM_PINNED;
- goto out_unlock;
- }
-
- /*
- * Stale inode items should force out the iclog.
- */
- if (ip->i_flags & XFS_ISTALE) {
- rval = XFS_ITEM_PINNED;
- goto out_unlock;
- }
-
- /*
- * Someone else is already flushing the inode. Nothing we can do
- * here but wait for the flush to finish and remove the item from
- * the AIL.
- */
- if (!xfs_iflock_nowait(ip)) {
- rval = XFS_ITEM_FLUSHING;
- goto out_unlock;
- }
-
- ASSERT(iip->ili_fields != 0 || XFS_FORCED_SHUTDOWN(ip->i_mount));
- ASSERT(iip->ili_logged == 0 || XFS_FORCED_SHUTDOWN(ip->i_mount));
-
spin_unlock(&lip->li_ailp->ail_lock);
- error = xfs_iflush(ip, &bp);
+ /*
+ * We need to hold a reference for flushing the cluster buffer as it may
+ * fail the buffer without IO submission. In which case, we better get a
+ * reference for that completion because otherwise we don't get a
+ * reference for IO until we queue the buffer for delwri submission.
+ */
+ xfs_buf_hold(bp);
+ error = xfs_iflush_cluster(bp);
if (!error) {
if (!xfs_buf_delwri_queue(bp, buffer_list))
rval = XFS_ITEM_FLUSHING;
xfs_buf_relse(bp);
- } else if (error == -EAGAIN)
+ } else {
+ /*
+ * Release the buffer if we were unable to flush anything. On
+ * any other error, the buffer has already been released.
+ */
+ if (error == -EAGAIN)
+ xfs_buf_relse(bp);
rval = XFS_ITEM_LOCKED;
+ }
spin_lock(&lip->li_ailp->ail_lock);
-out_unlock:
- xfs_iunlock(ip, XFS_ILOCK_SHARED);
return rval;
}
@@ -621,7 +601,6 @@
.iop_committed = xfs_inode_item_committed,
.iop_push = xfs_inode_item_push,
.iop_committing = xfs_inode_item_committing,
- .iop_error = xfs_inode_item_error
};
@@ -636,9 +615,11 @@
struct xfs_inode_log_item *iip;
ASSERT(ip->i_itemp == NULL);
- iip = ip->i_itemp = kmem_zone_zalloc(xfs_ili_zone, 0);
+ iip = ip->i_itemp = kmem_cache_zalloc(xfs_ili_zone,
+ GFP_KERNEL | __GFP_NOFAIL);
iip->ili_inode = ip;
+ spin_lock_init(&iip->ili_lock);
xfs_log_item_init(mp, &iip->ili_item, XFS_LI_INODE,
&xfs_inode_item_ops);
}
@@ -648,110 +629,129 @@
*/
void
xfs_inode_item_destroy(
- xfs_inode_t *ip)
+ struct xfs_inode *ip)
{
- kmem_free(ip->i_itemp->ili_item.li_lv_shadow);
- kmem_cache_free(xfs_ili_zone, ip->i_itemp);
+ struct xfs_inode_log_item *iip = ip->i_itemp;
+
+ ASSERT(iip->ili_item.li_buf == NULL);
+
+ ip->i_itemp = NULL;
+ kmem_free(iip->ili_item.li_lv_shadow);
+ kmem_cache_free(xfs_ili_zone, iip);
}
/*
- * This is the inode flushing I/O completion routine. It is called
- * from interrupt level when the buffer containing the inode is
- * flushed to disk. It is responsible for removing the inode item
- * from the AIL if it has not been re-logged, and unlocking the inode's
- * flush lock.
- *
- * To reduce AIL lock traffic as much as possible, we scan the buffer log item
- * list for other inodes that will run this function. We remove them from the
- * buffer list so we can process all the inode IO completions in one AIL lock
- * traversal.
+ * We only want to pull the item from the AIL if it is actually there
+ * and its location in the log has not changed since we started the
+ * flush. Thus, we only bother if the inode's lsn has not changed.
+ */
+static void
+xfs_iflush_ail_updates(
+ struct xfs_ail *ailp,
+ struct list_head *list)
+{
+ struct xfs_log_item *lip;
+ xfs_lsn_t tail_lsn = 0;
+
+ /* this is an opencoded batch version of xfs_trans_ail_delete */
+ spin_lock(&ailp->ail_lock);
+ list_for_each_entry(lip, list, li_bio_list) {
+ xfs_lsn_t lsn;
+
+ clear_bit(XFS_LI_FAILED, &lip->li_flags);
+ if (INODE_ITEM(lip)->ili_flush_lsn != lip->li_lsn)
+ continue;
+
+ lsn = xfs_ail_delete_one(ailp, lip);
+ if (!tail_lsn && lsn)
+ tail_lsn = lsn;
+ }
+ xfs_ail_update_finish(ailp, tail_lsn);
+}
+
+/*
+ * Walk the list of inodes that have completed their IOs. If they are clean
+ * remove them from the list and dissociate them from the buffer. Buffers that
+ * are still dirty remain linked to the buffer and on the list. Caller must
+ * handle them appropriately.
+ */
+static void
+xfs_iflush_finish(
+ struct xfs_buf *bp,
+ struct list_head *list)
+{
+ struct xfs_log_item *lip, *n;
+
+ list_for_each_entry_safe(lip, n, list, li_bio_list) {
+ struct xfs_inode_log_item *iip = INODE_ITEM(lip);
+ bool drop_buffer = false;
+
+ spin_lock(&iip->ili_lock);
+
+ /*
+ * Remove the reference to the cluster buffer if the inode is
+ * clean in memory and drop the buffer reference once we've
+ * dropped the locks we hold.
+ */
+ ASSERT(iip->ili_item.li_buf == bp);
+ if (!iip->ili_fields) {
+ iip->ili_item.li_buf = NULL;
+ list_del_init(&lip->li_bio_list);
+ drop_buffer = true;
+ }
+ iip->ili_last_fields = 0;
+ iip->ili_flush_lsn = 0;
+ spin_unlock(&iip->ili_lock);
+ xfs_ifunlock(iip->ili_inode);
+ if (drop_buffer)
+ xfs_buf_rele(bp);
+ }
+}
+
+/*
+ * Inode buffer IO completion routine. It is responsible for removing inodes
+ * attached to the buffer from the AIL if they have not been re-logged, as well
+ * as completing the flush and unlocking the inode.
*/
void
xfs_iflush_done(
- struct xfs_buf *bp,
- struct xfs_log_item *lip)
+ struct xfs_buf *bp)
{
- struct xfs_inode_log_item *iip;
- struct xfs_log_item *blip, *n;
- struct xfs_ail *ailp = lip->li_ailp;
- int need_ail = 0;
- LIST_HEAD(tmp);
+ struct xfs_log_item *lip, *n;
+ LIST_HEAD(flushed_inodes);
+ LIST_HEAD(ail_updates);
/*
- * Scan the buffer IO completions for other inodes being completed and
- * attach them to the current inode log item.
+ * Pull the attached inodes from the buffer one at a time and take the
+ * appropriate action on them.
*/
+ list_for_each_entry_safe(lip, n, &bp->b_li_list, li_bio_list) {
+ struct xfs_inode_log_item *iip = INODE_ITEM(lip);
- list_add_tail(&lip->li_bio_list, &tmp);
-
- list_for_each_entry_safe(blip, n, &bp->b_li_list, li_bio_list) {
- if (lip->li_cb != xfs_iflush_done)
+ if (xfs_iflags_test(iip->ili_inode, XFS_ISTALE)) {
+ xfs_iflush_abort(iip->ili_inode);
+ continue;
+ }
+ if (!iip->ili_last_fields)
continue;
- list_move_tail(&blip->li_bio_list, &tmp);
- /*
- * while we have the item, do the unlocked check for needing
- * the AIL lock.
- */
- iip = INODE_ITEM(blip);
- if ((iip->ili_logged && blip->li_lsn == iip->ili_flush_lsn) ||
- test_bit(XFS_LI_FAILED, &blip->li_flags))
- need_ail++;
+ /* Do an unlocked check for needing the AIL lock. */
+ if (iip->ili_flush_lsn == lip->li_lsn ||
+ test_bit(XFS_LI_FAILED, &lip->li_flags))
+ list_move_tail(&lip->li_bio_list, &ail_updates);
+ else
+ list_move_tail(&lip->li_bio_list, &flushed_inodes);
}
- /* make sure we capture the state of the initial inode. */
- iip = INODE_ITEM(lip);
- if ((iip->ili_logged && lip->li_lsn == iip->ili_flush_lsn) ||
- test_bit(XFS_LI_FAILED, &lip->li_flags))
- need_ail++;
-
- /*
- * We only want to pull the item from the AIL if it is
- * actually there and its location in the log has not
- * changed since we started the flush. Thus, we only bother
- * if the ili_logged flag is set and the inode's lsn has not
- * changed. First we check the lsn outside
- * the lock since it's cheaper, and then we recheck while
- * holding the lock before removing the inode from the AIL.
- */
- if (need_ail) {
- xfs_lsn_t tail_lsn = 0;
-
- /* this is an opencoded batch version of xfs_trans_ail_delete */
- spin_lock(&ailp->ail_lock);
- list_for_each_entry(blip, &tmp, li_bio_list) {
- if (INODE_ITEM(blip)->ili_logged &&
- blip->li_lsn == INODE_ITEM(blip)->ili_flush_lsn) {
- /*
- * xfs_ail_update_finish() only cares about the
- * lsn of the first tail item removed, any
- * others will be at the same or higher lsn so
- * we just ignore them.
- */
- xfs_lsn_t lsn = xfs_ail_delete_one(ailp, blip);
- if (!tail_lsn && lsn)
- tail_lsn = lsn;
- } else {
- xfs_clear_li_failed(blip);
- }
- }
- xfs_ail_update_finish(ailp, tail_lsn);
+ if (!list_empty(&ail_updates)) {
+ xfs_iflush_ail_updates(bp->b_mount->m_ail, &ail_updates);
+ list_splice_tail(&ail_updates, &flushed_inodes);
}
- /*
- * clean up and unlock the flush lock now we are done. We can clear the
- * ili_last_fields bits now that we know that the data corresponding to
- * them is safely on disk.
- */
- list_for_each_entry_safe(blip, n, &tmp, li_bio_list) {
- list_del_init(&blip->li_bio_list);
- iip = INODE_ITEM(blip);
- iip->ili_logged = 0;
- iip->ili_last_fields = 0;
- xfs_ifunlock(iip->ili_inode);
- }
- list_del(&tmp);
+ xfs_iflush_finish(bp, &flushed_inodes);
+ if (!list_empty(&flushed_inodes))
+ list_splice_tail(&flushed_inodes, &bp->b_li_list);
}
/*
@@ -762,37 +762,37 @@
*/
void
xfs_iflush_abort(
- struct xfs_inode *ip)
+ struct xfs_inode *ip)
{
- struct xfs_inode_log_item *iip = ip->i_itemp;
+ struct xfs_inode_log_item *iip = ip->i_itemp;
+ struct xfs_buf *bp = NULL;
if (iip) {
- xfs_trans_ail_delete(&iip->ili_item, 0);
- iip->ili_logged = 0;
/*
- * Clear the ili_last_fields bits now that we know that the
- * data corresponding to them is safely on disk.
+ * Clear the failed bit before removing the item from the AIL so
+ * xfs_trans_ail_delete() doesn't try to clear and release the
+ * buffer attached to the log item before we are done with it.
*/
- iip->ili_last_fields = 0;
+ clear_bit(XFS_LI_FAILED, &iip->ili_item.li_flags);
+ xfs_trans_ail_delete(&iip->ili_item, 0);
+
/*
* Clear the inode logging fields so no more flushes are
* attempted.
*/
+ spin_lock(&iip->ili_lock);
+ iip->ili_last_fields = 0;
iip->ili_fields = 0;
iip->ili_fsync_fields = 0;
+ iip->ili_flush_lsn = 0;
+ bp = iip->ili_item.li_buf;
+ iip->ili_item.li_buf = NULL;
+ list_del_init(&iip->ili_item.li_bio_list);
+ spin_unlock(&iip->ili_lock);
}
- /*
- * Release the inode's flush lock since we're done with it.
- */
xfs_ifunlock(ip);
-}
-
-void
-xfs_istale_done(
- struct xfs_buf *bp,
- struct xfs_log_item *lip)
-{
- xfs_iflush_abort(INODE_ITEM(lip)->ili_inode);
+ if (bp)
+ xfs_buf_rele(bp);
}
/*
diff --git a/fs/xfs/xfs_inode_item.h b/fs/xfs/xfs_inode_item.h
index 60b34bb..048b5e7 100644
--- a/fs/xfs/xfs_inode_item.h
+++ b/fs/xfs/xfs_inode_item.h
@@ -16,24 +16,34 @@
struct xfs_inode_log_item {
struct xfs_log_item ili_item; /* common portion */
struct xfs_inode *ili_inode; /* inode ptr */
- xfs_lsn_t ili_flush_lsn; /* lsn at last flush */
- xfs_lsn_t ili_last_lsn; /* lsn at last transaction */
- unsigned short ili_lock_flags; /* lock flags */
- unsigned short ili_logged; /* flushed logged data */
+ unsigned short ili_lock_flags; /* inode lock flags */
+ /*
+ * The ili_lock protects the interactions between the dirty state and
+ * the flush state of the inode log item. This allows us to do atomic
+ * modifications of multiple state fields without having to hold a
+ * specific inode lock to serialise them.
+ *
+ * We need atomic changes between inode dirtying, inode flushing and
+ * inode completion, but these all hold different combinations of
+ * ILOCK and iflock and hence we need some other method of serialising
+ * updates to the flush state.
+ */
+ spinlock_t ili_lock; /* flush state lock */
unsigned int ili_last_fields; /* fields when flushed */
unsigned int ili_fields; /* fields to be logged */
unsigned int ili_fsync_fields; /* logged since last fsync */
+ xfs_lsn_t ili_flush_lsn; /* lsn at last flush */
+ xfs_lsn_t ili_last_lsn; /* lsn at last transaction */
};
-static inline int xfs_inode_clean(xfs_inode_t *ip)
+static inline int xfs_inode_clean(struct xfs_inode *ip)
{
return !ip->i_itemp || !(ip->i_itemp->ili_fields & XFS_ILOG_ALL);
}
extern void xfs_inode_item_init(struct xfs_inode *, struct xfs_mount *);
extern void xfs_inode_item_destroy(struct xfs_inode *);
-extern void xfs_iflush_done(struct xfs_buf *, struct xfs_log_item *);
-extern void xfs_istale_done(struct xfs_buf *, struct xfs_log_item *);
+extern void xfs_iflush_done(struct xfs_buf *);
extern void xfs_iflush_abort(struct xfs_inode *);
extern int xfs_inode_item_format_convert(xfs_log_iovec_t *,
struct xfs_inode_log_format *);
diff --git a/fs/xfs/xfs_inode_item_recover.c b/fs/xfs/xfs_inode_item_recover.c
index dc3e26f..5e0d291 100644
--- a/fs/xfs/xfs_inode_item_recover.c
+++ b/fs/xfs/xfs_inode_item_recover.c
@@ -376,7 +376,7 @@
xfs_dinode_calc_crc(log->l_mp, dip);
ASSERT(bp->b_mount == mp);
- bp->b_iodone = xlog_recover_iodone;
+ bp->b_flags |= _XBF_LOGRECOVERY;
xfs_buf_delwri_queue(bp, buffer_list);
out_release:
diff --git a/fs/xfs/xfs_ioctl.c b/fs/xfs/xfs_ioctl.c
index a190212..6f22a66 100644
--- a/fs/xfs/xfs_ioctl.c
+++ b/fs/xfs/xfs_ioctl.c
@@ -1075,13 +1075,18 @@
xflags |= FS_XFLAG_NODUMP;
else
xflags &= ~FS_XFLAG_NODUMP;
+ if (flags & FS_DAX_FL)
+ xflags |= FS_XFLAG_DAX;
+ else
+ xflags &= ~FS_XFLAG_DAX;
return xflags;
}
STATIC unsigned int
xfs_di2lxflags(
- uint16_t di_flags)
+ uint16_t di_flags,
+ uint64_t di_flags2)
{
unsigned int flags = 0;
@@ -1095,6 +1100,9 @@
flags |= FS_NOATIME_FL;
if (di_flags & XFS_DIFLAG_NODUMP)
flags |= FS_NODUMP_FL;
+ if (di_flags2 & XFS_DIFLAG2_DAX) {
+ flags |= FS_DAX_FL;
+ }
return flags;
}
@@ -1565,7 +1573,7 @@
{
unsigned int flags;
- flags = xfs_di2lxflags(ip->i_d.di_flags);
+ flags = xfs_di2lxflags(ip->i_d.di_flags, ip->i_d.di_flags2);
if (copy_to_user(arg, &flags, sizeof(flags)))
return -EFAULT;
return 0;
@@ -1588,7 +1596,7 @@
if (flags & ~(FS_IMMUTABLE_FL | FS_APPEND_FL | \
FS_NOATIME_FL | FS_NODUMP_FL | \
- FS_SYNC_FL))
+ FS_SYNC_FL | FS_DAX_FL))
return -EOPNOTSUPP;
fa.fsx_xflags = xfs_merge_ioc_xflags(flags, xfs_ip2xflags(ip));
diff --git a/fs/xfs/xfs_iomap.c b/fs/xfs/xfs_iomap.c
index b9a8c37..0e3f62c 100644
--- a/fs/xfs/xfs_iomap.c
+++ b/fs/xfs/xfs_iomap.c
@@ -293,11 +293,11 @@
STATIC bool
xfs_quota_need_throttle(
- struct xfs_inode *ip,
- int type,
- xfs_fsblock_t alloc_blocks)
+ struct xfs_inode *ip,
+ xfs_dqtype_t type,
+ xfs_fsblock_t alloc_blocks)
{
- struct xfs_dquot *dq = xfs_inode_dquot(ip, type);
+ struct xfs_dquot *dq = xfs_inode_dquot(ip, type);
if (!dq || !xfs_this_quota_on(ip->i_mount, type))
return false;
@@ -307,7 +307,7 @@
return false;
/* under the lo watermark, no throttle */
- if (dq->q_res_bcount + alloc_blocks < dq->q_prealloc_lo_wmark)
+ if (dq->q_blk.reserved + alloc_blocks < dq->q_prealloc_lo_wmark)
return false;
return true;
@@ -315,24 +315,24 @@
STATIC void
xfs_quota_calc_throttle(
- struct xfs_inode *ip,
- int type,
- xfs_fsblock_t *qblocks,
- int *qshift,
- int64_t *qfreesp)
+ struct xfs_inode *ip,
+ xfs_dqtype_t type,
+ xfs_fsblock_t *qblocks,
+ int *qshift,
+ int64_t *qfreesp)
{
- int64_t freesp;
- int shift = 0;
- struct xfs_dquot *dq = xfs_inode_dquot(ip, type);
+ struct xfs_dquot *dq = xfs_inode_dquot(ip, type);
+ int64_t freesp;
+ int shift = 0;
/* no dq, or over hi wmark, squash the prealloc completely */
- if (!dq || dq->q_res_bcount >= dq->q_prealloc_hi_wmark) {
+ if (!dq || dq->q_blk.reserved >= dq->q_prealloc_hi_wmark) {
*qblocks = 0;
*qfreesp = 0;
return;
}
- freesp = dq->q_prealloc_hi_wmark - dq->q_res_bcount;
+ freesp = dq->q_prealloc_hi_wmark - dq->q_blk.reserved;
if (freesp < dq->q_low_space[XFS_QLOWSP_5_PCNT]) {
shift = 2;
if (freesp < dq->q_low_space[XFS_QLOWSP_3_PCNT])
@@ -450,14 +450,14 @@
* Check each quota to cap the prealloc size, provide a shift value to
* throttle with and adjust amount of available space.
*/
- if (xfs_quota_need_throttle(ip, XFS_DQ_USER, alloc_blocks))
- xfs_quota_calc_throttle(ip, XFS_DQ_USER, &qblocks, &qshift,
+ if (xfs_quota_need_throttle(ip, XFS_DQTYPE_USER, alloc_blocks))
+ xfs_quota_calc_throttle(ip, XFS_DQTYPE_USER, &qblocks, &qshift,
&freesp);
- if (xfs_quota_need_throttle(ip, XFS_DQ_GROUP, alloc_blocks))
- xfs_quota_calc_throttle(ip, XFS_DQ_GROUP, &qblocks, &qshift,
+ if (xfs_quota_need_throttle(ip, XFS_DQTYPE_GROUP, alloc_blocks))
+ xfs_quota_calc_throttle(ip, XFS_DQTYPE_GROUP, &qblocks, &qshift,
&freesp);
- if (xfs_quota_need_throttle(ip, XFS_DQ_PROJ, alloc_blocks))
- xfs_quota_calc_throttle(ip, XFS_DQ_PROJ, &qblocks, &qshift,
+ if (xfs_quota_need_throttle(ip, XFS_DQTYPE_PROJ, alloc_blocks))
+ xfs_quota_calc_throttle(ip, XFS_DQTYPE_PROJ, &qblocks, &qshift,
&freesp);
/*
diff --git a/fs/xfs/xfs_linux.h b/fs/xfs/xfs_linux.h
index 9f70d2f..ab737fe 100644
--- a/fs/xfs/xfs_linux.h
+++ b/fs/xfs/xfs_linux.h
@@ -102,12 +102,8 @@
#define xfs_cowb_secs xfs_params.cowb_timer.val
#define current_cpu() (raw_smp_processor_id())
-#define current_pid() (current->pid)
-#define current_test_flags(f) (current->flags & (f))
#define current_set_flags_nested(sp, f) \
(*(sp) = current->flags, current->flags |= (f))
-#define current_clear_flags_nested(sp, f) \
- (*(sp) = current->flags, current->flags &= ~(f))
#define current_restore_flags_nested(sp, f) \
(current->flags = ((current->flags & ~(f)) | (*(sp) & (f))))
diff --git a/fs/xfs/xfs_log.c b/fs/xfs/xfs_log.c
index 00fda2e..ad0c69ee 100644
--- a/fs/xfs/xfs_log.c
+++ b/fs/xfs/xfs_log.c
@@ -433,7 +433,7 @@
XFS_STATS_INC(mp, xs_try_logspace);
ASSERT(*ticp == NULL);
- tic = xlog_ticket_alloc(log, unit_bytes, cnt, client, permanent, 0);
+ tic = xlog_ticket_alloc(log, unit_bytes, cnt, client, permanent);
*ticp = tic;
xlog_grant_push_ail(log, tic->t_cnt ? tic->t_unit_res * tic->t_cnt
@@ -3408,15 +3408,12 @@
int unit_bytes,
int cnt,
char client,
- bool permanent,
- xfs_km_flags_t alloc_flags)
+ bool permanent)
{
struct xlog_ticket *tic;
int unit_res;
- tic = kmem_zone_zalloc(xfs_log_ticket_zone, alloc_flags);
- if (!tic)
- return NULL;
+ tic = kmem_cache_zalloc(xfs_log_ticket_zone, GFP_NOFS | __GFP_NOFAIL);
unit_res = xfs_log_calc_unit_res(log->l_mp, unit_bytes);
diff --git a/fs/xfs/xfs_log_cil.c b/fs/xfs/xfs_log_cil.c
index 9ed9036..56c32ee 100644
--- a/fs/xfs/xfs_log_cil.c
+++ b/fs/xfs/xfs_log_cil.c
@@ -37,8 +37,7 @@
{
struct xlog_ticket *tic;
- tic = xlog_ticket_alloc(log, 0, 1, XFS_TRANSACTION, 0,
- KM_NOFS);
+ tic = xlog_ticket_alloc(log, 0, 1, XFS_TRANSACTION, 0);
/*
* set the current reservation to zero so we know to steal the basic
diff --git a/fs/xfs/xfs_log_priv.h b/fs/xfs/xfs_log_priv.h
index 75a6287..1c6fdbf 100644
--- a/fs/xfs/xfs_log_priv.h
+++ b/fs/xfs/xfs_log_priv.h
@@ -464,9 +464,7 @@
int unit_bytes,
int count,
char client,
- bool permanent,
- xfs_km_flags_t alloc_flags);
-
+ bool permanent);
static inline void
xlog_write_adv_cnt(void **ptr, int *len, int *off, size_t bytes)
diff --git a/fs/xfs/xfs_log_recover.c b/fs/xfs/xfs_log_recover.c
index ec015df5..52a65a7 100644
--- a/fs/xfs/xfs_log_recover.c
+++ b/fs/xfs/xfs_log_recover.c
@@ -287,9 +287,8 @@
if (bp->b_log_item)
xfs_buf_item_relse(bp);
ASSERT(bp->b_log_item == NULL);
-
- bp->b_iodone = NULL;
- xfs_buf_ioend(bp);
+ bp->b_flags &= ~_XBF_LOGRECOVERY;
+ xfs_buf_ioend_finish(bp);
}
/*
diff --git a/fs/xfs/xfs_mount.c b/fs/xfs/xfs_mount.c
index d5dcf98..c8ae49a 100644
--- a/fs/xfs/xfs_mount.c
+++ b/fs/xfs/xfs_mount.c
@@ -148,7 +148,6 @@
ASSERT(atomic_read(&pag->pag_ref) == 0);
xfs_iunlink_destroy(pag);
xfs_buf_hash_destroy(pag);
- mutex_destroy(&pag->pag_ici_reclaim_lock);
call_rcu(&pag->rcu_head, __xfs_free_perag);
}
}
@@ -200,7 +199,6 @@
pag->pag_agno = index;
pag->pag_mount = mp;
spin_lock_init(&pag->pag_ici_lock);
- mutex_init(&pag->pag_ici_reclaim_lock);
INIT_RADIX_TREE(&pag->pag_ici_root, GFP_ATOMIC);
if (xfs_buf_hash_init(pag))
goto out_free_pag;
@@ -242,7 +240,6 @@
out_hash_destroy:
xfs_buf_hash_destroy(pag);
out_free_pag:
- mutex_destroy(&pag->pag_ici_reclaim_lock);
kmem_free(pag);
out_unwind_new_pags:
/* unwind any prior newly initialized pags */
@@ -252,7 +249,6 @@
break;
xfs_buf_hash_destroy(pag);
xfs_iunlink_destroy(pag);
- mutex_destroy(&pag->pag_ici_reclaim_lock);
kmem_free(pag);
}
return error;
@@ -1015,7 +1011,7 @@
* quota inodes.
*/
cancel_delayed_work_sync(&mp->m_reclaim_work);
- xfs_reclaim_inodes(mp, SYNC_WAIT);
+ xfs_reclaim_inodes(mp);
xfs_health_unmount(mp);
out_log_dealloc:
mp->m_flags |= XFS_MOUNT_UNMOUNTING;
@@ -1092,13 +1088,12 @@
xfs_ail_push_all_sync(mp->m_ail);
/*
- * And reclaim all inodes. At this point there should be no dirty
- * inodes and none should be pinned or locked, but use synchronous
- * reclaim just to be sure. We can stop background inode reclaim
- * here as well if it is still running.
+ * Reclaim all inodes. At this point there should be no dirty inodes and
+ * none should be pinned or locked. Stop background inode reclaim here
+ * if it is still running.
*/
cancel_delayed_work_sync(&mp->m_reclaim_work);
- xfs_reclaim_inodes(mp, SYNC_WAIT);
+ xfs_reclaim_inodes(mp);
xfs_health_unmount(mp);
xfs_qm_unmount(mp);
diff --git a/fs/xfs/xfs_mount.h b/fs/xfs/xfs_mount.h
index 3725d25..a72cfca 100644
--- a/fs/xfs/xfs_mount.h
+++ b/fs/xfs/xfs_mount.h
@@ -354,7 +354,6 @@
spinlock_t pag_ici_lock; /* incore inode cache lock */
struct radix_tree_root pag_ici_root; /* incore inode cache root */
int pag_ici_reclaimable; /* reclaimable inodes */
- struct mutex pag_ici_reclaim_lock; /* serialisation point */
unsigned long pag_ici_reclaim_cursor; /* reclaim restart point */
/* buffer cache index */
diff --git a/fs/xfs/xfs_qm.c b/fs/xfs/xfs_qm.c
index d6cd833..be67570 100644
--- a/fs/xfs/xfs_qm.c
+++ b/fs/xfs/xfs_qm.c
@@ -47,7 +47,7 @@
STATIC int
xfs_qm_dquot_walk(
struct xfs_mount *mp,
- int type,
+ xfs_dqtype_t type,
int (*execute)(struct xfs_dquot *dqp, void *data),
void *data)
{
@@ -79,7 +79,7 @@
for (i = 0; i < nr_found; i++) {
struct xfs_dquot *dqp = batch[i];
- next_index = be32_to_cpu(dqp->q_core.d_id) + 1;
+ next_index = dqp->q_id + 1;
error = execute(batch[i], data);
if (error == -EAGAIN) {
@@ -124,10 +124,10 @@
int error = -EAGAIN;
xfs_dqlock(dqp);
- if ((dqp->dq_flags & XFS_DQ_FREEING) || dqp->q_nrefs != 0)
+ if ((dqp->q_flags & XFS_DQFLAG_FREEING) || dqp->q_nrefs != 0)
goto out_unlock;
- dqp->dq_flags |= XFS_DQ_FREEING;
+ dqp->q_flags |= XFS_DQFLAG_FREEING;
xfs_dqflock(dqp);
@@ -148,6 +148,7 @@
error = xfs_bwrite(bp);
xfs_buf_relse(bp);
} else if (error == -EAGAIN) {
+ dqp->q_flags &= ~XFS_DQFLAG_FREEING;
goto out_unlock;
}
xfs_dqflock(dqp);
@@ -160,8 +161,7 @@
xfs_dqfunlock(dqp);
xfs_dqunlock(dqp);
- radix_tree_delete(xfs_dquot_tree(qi, dqp->q_core.d_flags),
- be32_to_cpu(dqp->q_core.d_id));
+ radix_tree_delete(xfs_dquot_tree(qi, xfs_dquot_type(dqp)), dqp->q_id);
qi->qi_dquots--;
/*
@@ -189,11 +189,11 @@
uint flags)
{
if (flags & XFS_QMOPT_UQUOTA)
- xfs_qm_dquot_walk(mp, XFS_DQ_USER, xfs_qm_dqpurge, NULL);
+ xfs_qm_dquot_walk(mp, XFS_DQTYPE_USER, xfs_qm_dqpurge, NULL);
if (flags & XFS_QMOPT_GQUOTA)
- xfs_qm_dquot_walk(mp, XFS_DQ_GROUP, xfs_qm_dqpurge, NULL);
+ xfs_qm_dquot_walk(mp, XFS_DQTYPE_GROUP, xfs_qm_dqpurge, NULL);
if (flags & XFS_QMOPT_PQUOTA)
- xfs_qm_dquot_walk(mp, XFS_DQ_PROJ, xfs_qm_dqpurge, NULL);
+ xfs_qm_dquot_walk(mp, XFS_DQTYPE_PROJ, xfs_qm_dqpurge, NULL);
}
/*
@@ -250,7 +250,7 @@
xfs_qm_dqattach_one(
struct xfs_inode *ip,
xfs_dqid_t id,
- uint type,
+ xfs_dqtype_t type,
bool doalloc,
struct xfs_dquot **IO_idqpp)
{
@@ -331,7 +331,7 @@
if (XFS_IS_UQUOTA_ON(mp) && !ip->i_udquot) {
error = xfs_qm_dqattach_one(ip, i_uid_read(VFS_I(ip)),
- XFS_DQ_USER, doalloc, &ip->i_udquot);
+ XFS_DQTYPE_USER, doalloc, &ip->i_udquot);
if (error)
goto done;
ASSERT(ip->i_udquot);
@@ -339,14 +339,14 @@
if (XFS_IS_GQUOTA_ON(mp) && !ip->i_gdquot) {
error = xfs_qm_dqattach_one(ip, i_gid_read(VFS_I(ip)),
- XFS_DQ_GROUP, doalloc, &ip->i_gdquot);
+ XFS_DQTYPE_GROUP, doalloc, &ip->i_gdquot);
if (error)
goto done;
ASSERT(ip->i_gdquot);
}
if (XFS_IS_PQUOTA_ON(mp) && !ip->i_pdquot) {
- error = xfs_qm_dqattach_one(ip, ip->i_d.di_projid, XFS_DQ_PROJ,
+ error = xfs_qm_dqattach_one(ip, ip->i_d.di_projid, XFS_DQTYPE_PROJ,
doalloc, &ip->i_pdquot);
if (error)
goto done;
@@ -473,7 +473,7 @@
/*
* Prevent lookups now that we are past the point of no return.
*/
- dqp->dq_flags |= XFS_DQ_FREEING;
+ dqp->q_flags |= XFS_DQFLAG_FREEING;
xfs_dqunlock(dqp);
ASSERT(dqp->q_nrefs == 0);
@@ -545,31 +545,29 @@
STATIC void
xfs_qm_set_defquota(
struct xfs_mount *mp,
- uint type,
+ xfs_dqtype_t type,
struct xfs_quotainfo *qinf)
{
struct xfs_dquot *dqp;
struct xfs_def_quota *defq;
- struct xfs_disk_dquot *ddqp;
int error;
error = xfs_qm_dqget_uncached(mp, 0, type, &dqp);
if (error)
return;
- ddqp = &dqp->q_core;
defq = xfs_get_defquota(qinf, xfs_dquot_type(dqp));
/*
* Timers and warnings have been already set, let's just set the
* default limits for this quota type
*/
- defq->bhardlimit = be64_to_cpu(ddqp->d_blk_hardlimit);
- defq->bsoftlimit = be64_to_cpu(ddqp->d_blk_softlimit);
- defq->ihardlimit = be64_to_cpu(ddqp->d_ino_hardlimit);
- defq->isoftlimit = be64_to_cpu(ddqp->d_ino_softlimit);
- defq->rtbhardlimit = be64_to_cpu(ddqp->d_rtb_hardlimit);
- defq->rtbsoftlimit = be64_to_cpu(ddqp->d_rtb_softlimit);
+ defq->blk.hard = dqp->q_blk.hardlimit;
+ defq->blk.soft = dqp->q_blk.softlimit;
+ defq->ino.hard = dqp->q_ino.hardlimit;
+ defq->ino.soft = dqp->q_ino.softlimit;
+ defq->rtb.hard = dqp->q_rtb.hardlimit;
+ defq->rtb.soft = dqp->q_rtb.softlimit;
xfs_qm_dqdestroy(dqp);
}
@@ -577,22 +575,21 @@
static void
xfs_qm_init_timelimits(
struct xfs_mount *mp,
- uint type)
+ xfs_dqtype_t type)
{
struct xfs_quotainfo *qinf = mp->m_quotainfo;
struct xfs_def_quota *defq;
- struct xfs_disk_dquot *ddqp;
struct xfs_dquot *dqp;
int error;
defq = xfs_get_defquota(qinf, type);
- defq->btimelimit = XFS_QM_BTIMELIMIT;
- defq->itimelimit = XFS_QM_ITIMELIMIT;
- defq->rtbtimelimit = XFS_QM_RTBTIMELIMIT;
- defq->bwarnlimit = XFS_QM_BWARNLIMIT;
- defq->iwarnlimit = XFS_QM_IWARNLIMIT;
- defq->rtbwarnlimit = XFS_QM_RTBWARNLIMIT;
+ defq->blk.time = XFS_QM_BTIMELIMIT;
+ defq->ino.time = XFS_QM_ITIMELIMIT;
+ defq->rtb.time = XFS_QM_RTBTIMELIMIT;
+ defq->blk.warn = XFS_QM_BWARNLIMIT;
+ defq->ino.warn = XFS_QM_IWARNLIMIT;
+ defq->rtb.warn = XFS_QM_RTBWARNLIMIT;
/*
* We try to get the limits from the superuser's limits fields.
@@ -605,25 +602,23 @@
if (error)
return;
- ddqp = &dqp->q_core;
-
/*
* The warnings and timers set the grace period given to
* a user or group before he or she can not perform any
* more writing. If it is zero, a default is used.
*/
- if (ddqp->d_btimer)
- defq->btimelimit = be32_to_cpu(ddqp->d_btimer);
- if (ddqp->d_itimer)
- defq->itimelimit = be32_to_cpu(ddqp->d_itimer);
- if (ddqp->d_rtbtimer)
- defq->rtbtimelimit = be32_to_cpu(ddqp->d_rtbtimer);
- if (ddqp->d_bwarns)
- defq->bwarnlimit = be16_to_cpu(ddqp->d_bwarns);
- if (ddqp->d_iwarns)
- defq->iwarnlimit = be16_to_cpu(ddqp->d_iwarns);
- if (ddqp->d_rtbwarns)
- defq->rtbwarnlimit = be16_to_cpu(ddqp->d_rtbwarns);
+ if (dqp->q_blk.timer)
+ defq->blk.time = dqp->q_blk.timer;
+ if (dqp->q_ino.timer)
+ defq->ino.time = dqp->q_ino.timer;
+ if (dqp->q_rtb.timer)
+ defq->rtb.time = dqp->q_rtb.timer;
+ if (dqp->q_blk.warnings)
+ defq->blk.warn = dqp->q_blk.warnings;
+ if (dqp->q_ino.warnings)
+ defq->ino.warn = dqp->q_ino.warnings;
+ if (dqp->q_rtb.warnings)
+ defq->rtb.warn = dqp->q_rtb.warnings;
xfs_qm_dqdestroy(dqp);
}
@@ -669,16 +664,16 @@
mp->m_qflags |= (mp->m_sb.sb_qflags & XFS_ALL_QUOTA_CHKD);
- xfs_qm_init_timelimits(mp, XFS_DQ_USER);
- xfs_qm_init_timelimits(mp, XFS_DQ_GROUP);
- xfs_qm_init_timelimits(mp, XFS_DQ_PROJ);
+ xfs_qm_init_timelimits(mp, XFS_DQTYPE_USER);
+ xfs_qm_init_timelimits(mp, XFS_DQTYPE_GROUP);
+ xfs_qm_init_timelimits(mp, XFS_DQTYPE_PROJ);
if (XFS_IS_UQUOTA_RUNNING(mp))
- xfs_qm_set_defquota(mp, XFS_DQ_USER, qinf);
+ xfs_qm_set_defquota(mp, XFS_DQTYPE_USER, qinf);
if (XFS_IS_GQUOTA_RUNNING(mp))
- xfs_qm_set_defquota(mp, XFS_DQ_GROUP, qinf);
+ xfs_qm_set_defquota(mp, XFS_DQTYPE_GROUP, qinf);
if (XFS_IS_PQUOTA_RUNNING(mp))
- xfs_qm_set_defquota(mp, XFS_DQ_PROJ, qinf);
+ xfs_qm_set_defquota(mp, XFS_DQTYPE_PROJ, qinf);
qinf->qi_shrinker.count_objects = xfs_qm_shrink_count;
qinf->qi_shrinker.scan_objects = xfs_qm_shrink_scan;
@@ -828,14 +823,13 @@
STATIC void
xfs_qm_reset_dqcounts(
- xfs_mount_t *mp,
- xfs_buf_t *bp,
- xfs_dqid_t id,
- uint type)
+ struct xfs_mount *mp,
+ struct xfs_buf *bp,
+ xfs_dqid_t id,
+ xfs_dqtype_t type)
{
struct xfs_dqblk *dqb;
int j;
- xfs_failaddr_t fa;
trace_xfs_reset_dqcounts(bp, _RET_IP_);
@@ -860,15 +854,15 @@
* find uninitialised dquot blks. See comment in
* xfs_dquot_verify.
*/
- fa = xfs_dqblk_verify(mp, &dqb[j], id + j, type);
- if (fa)
+ if (xfs_dqblk_verify(mp, &dqb[j], id + j) ||
+ (dqb[j].dd_diskdq.d_type & XFS_DQTYPE_REC_MASK) != type)
xfs_dqblk_repair(mp, &dqb[j], id + j, type);
/*
* Reset type in case we are reusing group quota file for
* project quotas or vice versa
*/
- ddq->d_flags = type;
+ ddq->d_type = type;
ddq->d_bcount = 0;
ddq->d_icount = 0;
ddq->d_rtbcount = 0;
@@ -901,17 +895,13 @@
xfs_dqid_t firstid,
xfs_fsblock_t bno,
xfs_filblks_t blkcnt,
- uint flags,
+ xfs_dqtype_t type,
struct list_head *buffer_list)
{
struct xfs_buf *bp;
- int error;
- int type;
+ int error = 0;
ASSERT(blkcnt > 0);
- type = flags & XFS_QMOPT_UQUOTA ? XFS_DQ_USER :
- (flags & XFS_QMOPT_PQUOTA ? XFS_DQ_PROJ : XFS_DQ_GROUP);
- error = 0;
/*
* Blkcnt arg can be a very big number, and might even be
@@ -971,7 +961,7 @@
xfs_qm_reset_dqcounts_buf(
struct xfs_mount *mp,
struct xfs_inode *qip,
- uint flags,
+ xfs_dqtype_t type,
struct list_head *buffer_list)
{
struct xfs_bmbt_irec *map;
@@ -1047,7 +1037,7 @@
error = xfs_qm_reset_dqcounts_all(mp, firstid,
map[i].br_startblock,
map[i].br_blockcount,
- flags, buffer_list);
+ type, buffer_list);
if (error)
goto out;
}
@@ -1069,7 +1059,7 @@
STATIC int
xfs_qm_quotacheck_dqadjust(
struct xfs_inode *ip,
- uint type,
+ xfs_dqtype_t type,
xfs_qcnt_t nblks,
xfs_qcnt_t rtblks)
{
@@ -1095,15 +1085,15 @@
* Adjust the inode count and the block count to reflect this inode's
* resource usage.
*/
- be64_add_cpu(&dqp->q_core.d_icount, 1);
- dqp->q_res_icount++;
+ dqp->q_ino.count++;
+ dqp->q_ino.reserved++;
if (nblks) {
- be64_add_cpu(&dqp->q_core.d_bcount, nblks);
- dqp->q_res_bcount += nblks;
+ dqp->q_blk.count += nblks;
+ dqp->q_blk.reserved += nblks;
}
if (rtblks) {
- be64_add_cpu(&dqp->q_core.d_rtbcount, rtblks);
- dqp->q_res_rtbcount += rtblks;
+ dqp->q_rtb.count += rtblks;
+ dqp->q_rtb.reserved += rtblks;
}
/*
@@ -1111,12 +1101,12 @@
*
* There are no timers for the default values set in the root dquot.
*/
- if (dqp->q_core.d_id) {
- xfs_qm_adjust_dqlimits(mp, dqp);
- xfs_qm_adjust_dqtimers(mp, dqp);
+ if (dqp->q_id) {
+ xfs_qm_adjust_dqlimits(dqp);
+ xfs_qm_adjust_dqtimers(dqp);
}
- dqp->dq_flags |= XFS_DQ_DIRTY;
+ dqp->q_flags |= XFS_DQFLAG_DIRTY;
xfs_qm_dqput(dqp);
return 0;
}
@@ -1186,21 +1176,21 @@
* and quotaoffs don't race. (Quotachecks happen at mount time only).
*/
if (XFS_IS_UQUOTA_ON(mp)) {
- error = xfs_qm_quotacheck_dqadjust(ip, XFS_DQ_USER, nblks,
+ error = xfs_qm_quotacheck_dqadjust(ip, XFS_DQTYPE_USER, nblks,
rtblks);
if (error)
goto error0;
}
if (XFS_IS_GQUOTA_ON(mp)) {
- error = xfs_qm_quotacheck_dqadjust(ip, XFS_DQ_GROUP, nblks,
+ error = xfs_qm_quotacheck_dqadjust(ip, XFS_DQTYPE_GROUP, nblks,
rtblks);
if (error)
goto error0;
}
if (XFS_IS_PQUOTA_ON(mp)) {
- error = xfs_qm_quotacheck_dqadjust(ip, XFS_DQ_PROJ, nblks,
+ error = xfs_qm_quotacheck_dqadjust(ip, XFS_DQTYPE_PROJ, nblks,
rtblks);
if (error)
goto error0;
@@ -1222,7 +1212,7 @@
int error = 0;
xfs_dqlock(dqp);
- if (dqp->dq_flags & XFS_DQ_FREEING)
+ if (dqp->q_flags & XFS_DQFLAG_FREEING)
goto out_unlock;
if (!XFS_DQ_IS_DIRTY(dqp))
goto out_unlock;
@@ -1291,7 +1281,7 @@
* We don't log our changes till later.
*/
if (uip) {
- error = xfs_qm_reset_dqcounts_buf(mp, uip, XFS_QMOPT_UQUOTA,
+ error = xfs_qm_reset_dqcounts_buf(mp, uip, XFS_DQTYPE_USER,
&buffer_list);
if (error)
goto error_return;
@@ -1299,7 +1289,7 @@
}
if (gip) {
- error = xfs_qm_reset_dqcounts_buf(mp, gip, XFS_QMOPT_GQUOTA,
+ error = xfs_qm_reset_dqcounts_buf(mp, gip, XFS_DQTYPE_GROUP,
&buffer_list);
if (error)
goto error_return;
@@ -1307,7 +1297,7 @@
}
if (pip) {
- error = xfs_qm_reset_dqcounts_buf(mp, pip, XFS_QMOPT_PQUOTA,
+ error = xfs_qm_reset_dqcounts_buf(mp, pip, XFS_DQTYPE_PROJ,
&buffer_list);
if (error)
goto error_return;
@@ -1324,17 +1314,17 @@
* down to disk buffers if everything was updated successfully.
*/
if (XFS_IS_UQUOTA_ON(mp)) {
- error = xfs_qm_dquot_walk(mp, XFS_DQ_USER, xfs_qm_flush_one,
+ error = xfs_qm_dquot_walk(mp, XFS_DQTYPE_USER, xfs_qm_flush_one,
&buffer_list);
}
if (XFS_IS_GQUOTA_ON(mp)) {
- error2 = xfs_qm_dquot_walk(mp, XFS_DQ_GROUP, xfs_qm_flush_one,
+ error2 = xfs_qm_dquot_walk(mp, XFS_DQTYPE_GROUP, xfs_qm_flush_one,
&buffer_list);
if (!error)
error = error2;
}
if (XFS_IS_PQUOTA_ON(mp)) {
- error2 = xfs_qm_dquot_walk(mp, XFS_DQ_PROJ, xfs_qm_flush_one,
+ error2 = xfs_qm_dquot_walk(mp, XFS_DQTYPE_PROJ, xfs_qm_flush_one,
&buffer_list);
if (!error)
error = error2;
@@ -1597,8 +1587,7 @@
struct xfs_quotainfo *qi = mp->m_quotainfo;
mutex_lock(&qi->qi_tree_lock);
- radix_tree_delete(xfs_dquot_tree(qi, dqp->q_core.d_flags),
- be32_to_cpu(dqp->q_core.d_id));
+ radix_tree_delete(xfs_dquot_tree(qi, xfs_dquot_type(dqp)), dqp->q_id);
qi->qi_dquots--;
mutex_unlock(&qi->qi_tree_lock);
@@ -1673,7 +1662,7 @@
*/
xfs_iunlock(ip, lockflags);
error = xfs_qm_dqget(mp, from_kuid(user_ns, uid),
- XFS_DQ_USER, true, &uq);
+ XFS_DQTYPE_USER, true, &uq);
if (error) {
ASSERT(error != -ENOENT);
return error;
@@ -1697,7 +1686,7 @@
if (!gid_eq(inode->i_gid, gid)) {
xfs_iunlock(ip, lockflags);
error = xfs_qm_dqget(mp, from_kgid(user_ns, gid),
- XFS_DQ_GROUP, true, &gq);
+ XFS_DQTYPE_GROUP, true, &gq);
if (error) {
ASSERT(error != -ENOENT);
goto error_rele;
@@ -1713,8 +1702,8 @@
if ((flags & XFS_QMOPT_PQUOTA) && XFS_IS_PQUOTA_ON(mp)) {
if (ip->i_d.di_projid != prid) {
xfs_iunlock(ip, lockflags);
- error = xfs_qm_dqget(mp, (xfs_dqid_t)prid, XFS_DQ_PROJ,
- true, &pq);
+ error = xfs_qm_dqget(mp, (xfs_dqid_t)prid,
+ XFS_DQTYPE_PROJ, true, &pq);
if (error) {
ASSERT(error != -ENOENT);
goto error_rele;
@@ -1822,7 +1811,7 @@
XFS_QMOPT_RES_RTBLKS : XFS_QMOPT_RES_REGBLKS;
if (XFS_IS_UQUOTA_ON(mp) && udqp &&
- i_uid_read(VFS_I(ip)) != be32_to_cpu(udqp->q_core.d_id)) {
+ i_uid_read(VFS_I(ip)) != udqp->q_id) {
udq_delblks = udqp;
/*
* If there are delayed allocation blocks, then we have to
@@ -1835,7 +1824,7 @@
}
}
if (XFS_IS_GQUOTA_ON(ip->i_mount) && gdqp &&
- i_gid_read(VFS_I(ip)) != be32_to_cpu(gdqp->q_core.d_id)) {
+ i_gid_read(VFS_I(ip)) != gdqp->q_id) {
gdq_delblks = gdqp;
if (delblks) {
ASSERT(ip->i_gdquot);
@@ -1844,7 +1833,7 @@
}
if (XFS_IS_PQUOTA_ON(ip->i_mount) && pdqp &&
- ip->i_d.di_projid != be32_to_cpu(pdqp->q_core.d_id)) {
+ ip->i_d.di_projid != pdqp->q_id) {
pdq_delblks = pdqp;
if (delblks) {
ASSERT(ip->i_pdquot);
@@ -1928,21 +1917,21 @@
if (udqp && XFS_IS_UQUOTA_ON(mp)) {
ASSERT(ip->i_udquot == NULL);
- ASSERT(i_uid_read(VFS_I(ip)) == be32_to_cpu(udqp->q_core.d_id));
+ ASSERT(i_uid_read(VFS_I(ip)) == udqp->q_id);
ip->i_udquot = xfs_qm_dqhold(udqp);
xfs_trans_mod_dquot(tp, udqp, XFS_TRANS_DQ_ICOUNT, 1);
}
if (gdqp && XFS_IS_GQUOTA_ON(mp)) {
ASSERT(ip->i_gdquot == NULL);
- ASSERT(i_gid_read(VFS_I(ip)) == be32_to_cpu(gdqp->q_core.d_id));
+ ASSERT(i_gid_read(VFS_I(ip)) == gdqp->q_id);
ip->i_gdquot = xfs_qm_dqhold(gdqp);
xfs_trans_mod_dquot(tp, gdqp, XFS_TRANS_DQ_ICOUNT, 1);
}
if (pdqp && XFS_IS_PQUOTA_ON(mp)) {
ASSERT(ip->i_pdquot == NULL);
- ASSERT(ip->i_d.di_projid == be32_to_cpu(pdqp->q_core.d_id));
+ ASSERT(ip->i_d.di_projid == pdqp->q_id);
ip->i_pdquot = xfs_qm_dqhold(pdqp);
xfs_trans_mod_dquot(tp, pdqp, XFS_TRANS_DQ_ICOUNT, 1);
diff --git a/fs/xfs/xfs_qm.h b/fs/xfs/xfs_qm.h
index 7b0e771..9c078c3 100644
--- a/fs/xfs/xfs_qm.h
+++ b/fs/xfs/xfs_qm.h
@@ -20,41 +20,28 @@
#define XFS_DQITER_MAP_SIZE 10
#define XFS_IS_DQUOT_UNINITIALIZED(dqp) ( \
- !dqp->q_core.d_blk_hardlimit && \
- !dqp->q_core.d_blk_softlimit && \
- !dqp->q_core.d_rtb_hardlimit && \
- !dqp->q_core.d_rtb_softlimit && \
- !dqp->q_core.d_ino_hardlimit && \
- !dqp->q_core.d_ino_softlimit && \
- !dqp->q_core.d_bcount && \
- !dqp->q_core.d_rtbcount && \
- !dqp->q_core.d_icount)
+ !dqp->q_blk.hardlimit && \
+ !dqp->q_blk.softlimit && \
+ !dqp->q_rtb.hardlimit && \
+ !dqp->q_rtb.softlimit && \
+ !dqp->q_ino.hardlimit && \
+ !dqp->q_ino.softlimit && \
+ !dqp->q_blk.count && \
+ !dqp->q_rtb.count && \
+ !dqp->q_ino.count)
-/*
- * This defines the unit of allocation of dquots.
- * Currently, it is just one file system block, and a 4K blk contains 30
- * (136 * 30 = 4080) dquots. It's probably not worth trying to make
- * this more dynamic.
- * XXXsup However, if this number is changed, we have to make sure that we don't
- * implicitly assume that we do allocations in chunks of a single filesystem
- * block in the dquot/xqm code.
- */
-#define XFS_DQUOT_CLUSTER_SIZE_FSB (xfs_filblks_t)1
+struct xfs_quota_limits {
+ xfs_qcnt_t hard; /* default hard limit */
+ xfs_qcnt_t soft; /* default soft limit */
+ time64_t time; /* limit for timers */
+ xfs_qwarncnt_t warn; /* limit for warnings */
+};
/* Defaults for each quota type: time limits, warn limits, usage limits */
struct xfs_def_quota {
- time64_t btimelimit; /* limit for blks timer */
- time64_t itimelimit; /* limit for inodes timer */
- time64_t rtbtimelimit; /* limit for rt blks timer */
- xfs_qwarncnt_t bwarnlimit; /* limit for blks warnings */
- xfs_qwarncnt_t iwarnlimit; /* limit for inodes warnings */
- xfs_qwarncnt_t rtbwarnlimit; /* limit for rt blks warnings */
- xfs_qcnt_t bhardlimit; /* default data blk hard limit */
- xfs_qcnt_t bsoftlimit; /* default data blk soft limit */
- xfs_qcnt_t ihardlimit; /* default inode count hard limit */
- xfs_qcnt_t isoftlimit; /* default inode count soft limit */
- xfs_qcnt_t rtbhardlimit; /* default realtime blk hard limit */
- xfs_qcnt_t rtbsoftlimit; /* default realtime blk soft limit */
+ struct xfs_quota_limits blk;
+ struct xfs_quota_limits ino;
+ struct xfs_quota_limits rtb;
};
/*
@@ -83,14 +70,14 @@
static inline struct radix_tree_root *
xfs_dquot_tree(
struct xfs_quotainfo *qi,
- int type)
+ xfs_dqtype_t type)
{
switch (type) {
- case XFS_DQ_USER:
+ case XFS_DQTYPE_USER:
return &qi->qi_uquota_tree;
- case XFS_DQ_GROUP:
+ case XFS_DQTYPE_GROUP:
return &qi->qi_gquota_tree;
- case XFS_DQ_PROJ:
+ case XFS_DQTYPE_PROJ:
return &qi->qi_pquota_tree;
default:
ASSERT(0);
@@ -99,14 +86,14 @@
}
static inline struct xfs_inode *
-xfs_quota_inode(xfs_mount_t *mp, uint dq_flags)
+xfs_quota_inode(struct xfs_mount *mp, xfs_dqtype_t type)
{
- switch (dq_flags & XFS_DQ_ALLTYPES) {
- case XFS_DQ_USER:
+ switch (type) {
+ case XFS_DQTYPE_USER:
return mp->m_quotainfo->qi_uquotaip;
- case XFS_DQ_GROUP:
+ case XFS_DQTYPE_GROUP:
return mp->m_quotainfo->qi_gquotaip;
- case XFS_DQ_PROJ:
+ case XFS_DQTYPE_PROJ:
return mp->m_quotainfo->qi_pquotaip;
default:
ASSERT(0);
@@ -114,17 +101,6 @@
return NULL;
}
-static inline int
-xfs_dquot_type(struct xfs_dquot *dqp)
-{
- if (XFS_QM_ISUDQ(dqp))
- return XFS_DQ_USER;
- if (XFS_QM_ISGDQ(dqp))
- return XFS_DQ_GROUP;
- ASSERT(XFS_QM_ISPDQ(dqp));
- return XFS_DQ_PROJ;
-}
-
extern void xfs_trans_mod_dquot(struct xfs_trans *tp, struct xfs_dquot *dqp,
uint field, int64_t delta);
extern void xfs_trans_dqjoin(struct xfs_trans *, struct xfs_dquot *);
@@ -166,24 +142,30 @@
/* quota ops */
extern int xfs_qm_scall_trunc_qfiles(struct xfs_mount *, uint);
-extern int xfs_qm_scall_getquota(struct xfs_mount *, xfs_dqid_t,
- uint, struct qc_dqblk *);
-extern int xfs_qm_scall_getquota_next(struct xfs_mount *,
- xfs_dqid_t *, uint, struct qc_dqblk *);
-extern int xfs_qm_scall_setqlim(struct xfs_mount *, xfs_dqid_t, uint,
- struct qc_dqblk *);
+extern int xfs_qm_scall_getquota(struct xfs_mount *mp,
+ xfs_dqid_t id,
+ xfs_dqtype_t type,
+ struct qc_dqblk *dst);
+extern int xfs_qm_scall_getquota_next(struct xfs_mount *mp,
+ xfs_dqid_t *id,
+ xfs_dqtype_t type,
+ struct qc_dqblk *dst);
+extern int xfs_qm_scall_setqlim(struct xfs_mount *mp,
+ xfs_dqid_t id,
+ xfs_dqtype_t type,
+ struct qc_dqblk *newlim);
extern int xfs_qm_scall_quotaon(struct xfs_mount *, uint);
extern int xfs_qm_scall_quotaoff(struct xfs_mount *, uint);
static inline struct xfs_def_quota *
-xfs_get_defquota(struct xfs_quotainfo *qi, int type)
+xfs_get_defquota(struct xfs_quotainfo *qi, xfs_dqtype_t type)
{
switch (type) {
- case XFS_DQ_USER:
+ case XFS_DQTYPE_USER:
return &qi->qi_usr_default;
- case XFS_DQ_GROUP:
+ case XFS_DQTYPE_GROUP:
return &qi->qi_grp_default;
- case XFS_DQ_PROJ:
+ case XFS_DQTYPE_PROJ:
return &qi->qi_prj_default;
default:
ASSERT(0);
diff --git a/fs/xfs/xfs_qm_bhv.c b/fs/xfs/xfs_qm_bhv.c
index fc2fa41..6393980 100644
--- a/fs/xfs/xfs_qm_bhv.c
+++ b/fs/xfs/xfs_qm_bhv.c
@@ -23,24 +23,24 @@
{
uint64_t limit;
- limit = dqp->q_core.d_blk_softlimit ?
- be64_to_cpu(dqp->q_core.d_blk_softlimit) :
- be64_to_cpu(dqp->q_core.d_blk_hardlimit);
+ limit = dqp->q_blk.softlimit ?
+ dqp->q_blk.softlimit :
+ dqp->q_blk.hardlimit;
if (limit && statp->f_blocks > limit) {
statp->f_blocks = limit;
statp->f_bfree = statp->f_bavail =
- (statp->f_blocks > dqp->q_res_bcount) ?
- (statp->f_blocks - dqp->q_res_bcount) : 0;
+ (statp->f_blocks > dqp->q_blk.reserved) ?
+ (statp->f_blocks - dqp->q_blk.reserved) : 0;
}
- limit = dqp->q_core.d_ino_softlimit ?
- be64_to_cpu(dqp->q_core.d_ino_softlimit) :
- be64_to_cpu(dqp->q_core.d_ino_hardlimit);
+ limit = dqp->q_ino.softlimit ?
+ dqp->q_ino.softlimit :
+ dqp->q_ino.hardlimit;
if (limit && statp->f_files > limit) {
statp->f_files = limit;
statp->f_ffree =
- (statp->f_files > dqp->q_res_icount) ?
- (statp->f_files - dqp->q_res_icount) : 0;
+ (statp->f_files > dqp->q_ino.reserved) ?
+ (statp->f_files - dqp->q_ino.reserved) : 0;
}
}
@@ -60,7 +60,7 @@
struct xfs_mount *mp = ip->i_mount;
struct xfs_dquot *dqp;
- if (!xfs_qm_dqget(mp, ip->i_d.di_projid, XFS_DQ_PROJ, false, &dqp)) {
+ if (!xfs_qm_dqget(mp, ip->i_d.di_projid, XFS_DQTYPE_PROJ, false, &dqp)) {
xfs_fill_statvfs_from_dquot(statp, dqp);
xfs_qm_dqput(dqp);
}
diff --git a/fs/xfs/xfs_qm_syscalls.c b/fs/xfs/xfs_qm_syscalls.c
index 7effd7a..1c542b4 100644
--- a/fs/xfs/xfs_qm_syscalls.c
+++ b/fs/xfs/xfs_qm_syscalls.c
@@ -322,23 +322,23 @@
int error = -EINVAL;
if (!xfs_sb_version_hasquota(&mp->m_sb) || flags == 0 ||
- (flags & ~XFS_DQ_ALLTYPES)) {
+ (flags & ~XFS_QMOPT_QUOTALL)) {
xfs_debug(mp, "%s: flags=%x m_qflags=%x",
__func__, flags, mp->m_qflags);
return -EINVAL;
}
- if (flags & XFS_DQ_USER) {
+ if (flags & XFS_QMOPT_UQUOTA) {
error = xfs_qm_scall_trunc_qfile(mp, mp->m_sb.sb_uquotino);
if (error)
return error;
}
- if (flags & XFS_DQ_GROUP) {
+ if (flags & XFS_QMOPT_GQUOTA) {
error = xfs_qm_scall_trunc_qfile(mp, mp->m_sb.sb_gquotino);
if (error)
return error;
}
- if (flags & XFS_DQ_PROJ)
+ if (flags & XFS_QMOPT_PQUOTA)
error = xfs_qm_scall_trunc_qfile(mp, mp->m_sb.sb_pquotino);
return error;
@@ -437,20 +437,73 @@
(QC_LIMIT_MASK | QC_TIMER_MASK | QC_WARNS_MASK)
/*
+ * Adjust limits of this quota, and the defaults if passed in. Returns true
+ * if the new limits made sense and were applied, false otherwise.
+ */
+static inline bool
+xfs_setqlim_limits(
+ struct xfs_mount *mp,
+ struct xfs_dquot_res *res,
+ struct xfs_quota_limits *qlim,
+ xfs_qcnt_t hard,
+ xfs_qcnt_t soft,
+ const char *tag)
+{
+ /* The hard limit can't be less than the soft limit. */
+ if (hard != 0 && hard < soft) {
+ xfs_debug(mp, "%shard %lld < %ssoft %lld", tag, hard, tag,
+ soft);
+ return false;
+ }
+
+ res->hardlimit = hard;
+ res->softlimit = soft;
+ if (qlim) {
+ qlim->hard = hard;
+ qlim->soft = soft;
+ }
+
+ return true;
+}
+
+static inline void
+xfs_setqlim_warns(
+ struct xfs_dquot_res *res,
+ struct xfs_quota_limits *qlim,
+ int warns)
+{
+ res->warnings = warns;
+ if (qlim)
+ qlim->warn = warns;
+}
+
+static inline void
+xfs_setqlim_timer(
+ struct xfs_dquot_res *res,
+ struct xfs_quota_limits *qlim,
+ s64 timer)
+{
+ res->timer = timer;
+ if (qlim)
+ qlim->time = timer;
+}
+
+/*
* Adjust quota limits, and start/stop timers accordingly.
*/
int
xfs_qm_scall_setqlim(
struct xfs_mount *mp,
xfs_dqid_t id,
- uint type,
+ xfs_dqtype_t type,
struct qc_dqblk *newlim)
{
struct xfs_quotainfo *q = mp->m_quotainfo;
- struct xfs_disk_dquot *ddq;
struct xfs_dquot *dqp;
struct xfs_trans *tp;
struct xfs_def_quota *defq;
+ struct xfs_dquot_res *res;
+ struct xfs_quota_limits *qlim;
int error;
xfs_qcnt_t hard, soft;
@@ -488,82 +541,15 @@
xfs_dqlock(dqp);
xfs_trans_dqjoin(tp, dqp);
- ddq = &dqp->q_core;
/*
+ * Update quota limits, warnings, and timers, and the defaults
+ * if we're touching id == 0.
+ *
* Make sure that hardlimits are >= soft limits before changing.
- */
- hard = (newlim->d_fieldmask & QC_SPC_HARD) ?
- (xfs_qcnt_t) XFS_B_TO_FSB(mp, newlim->d_spc_hardlimit) :
- be64_to_cpu(ddq->d_blk_hardlimit);
- soft = (newlim->d_fieldmask & QC_SPC_SOFT) ?
- (xfs_qcnt_t) XFS_B_TO_FSB(mp, newlim->d_spc_softlimit) :
- be64_to_cpu(ddq->d_blk_softlimit);
- if (hard == 0 || hard >= soft) {
- ddq->d_blk_hardlimit = cpu_to_be64(hard);
- ddq->d_blk_softlimit = cpu_to_be64(soft);
- xfs_dquot_set_prealloc_limits(dqp);
- if (id == 0) {
- defq->bhardlimit = hard;
- defq->bsoftlimit = soft;
- }
- } else {
- xfs_debug(mp, "blkhard %Ld < blksoft %Ld", hard, soft);
- }
- hard = (newlim->d_fieldmask & QC_RT_SPC_HARD) ?
- (xfs_qcnt_t) XFS_B_TO_FSB(mp, newlim->d_rt_spc_hardlimit) :
- be64_to_cpu(ddq->d_rtb_hardlimit);
- soft = (newlim->d_fieldmask & QC_RT_SPC_SOFT) ?
- (xfs_qcnt_t) XFS_B_TO_FSB(mp, newlim->d_rt_spc_softlimit) :
- be64_to_cpu(ddq->d_rtb_softlimit);
- if (hard == 0 || hard >= soft) {
- ddq->d_rtb_hardlimit = cpu_to_be64(hard);
- ddq->d_rtb_softlimit = cpu_to_be64(soft);
- if (id == 0) {
- defq->rtbhardlimit = hard;
- defq->rtbsoftlimit = soft;
- }
- } else {
- xfs_debug(mp, "rtbhard %Ld < rtbsoft %Ld", hard, soft);
- }
-
- hard = (newlim->d_fieldmask & QC_INO_HARD) ?
- (xfs_qcnt_t) newlim->d_ino_hardlimit :
- be64_to_cpu(ddq->d_ino_hardlimit);
- soft = (newlim->d_fieldmask & QC_INO_SOFT) ?
- (xfs_qcnt_t) newlim->d_ino_softlimit :
- be64_to_cpu(ddq->d_ino_softlimit);
- if (hard == 0 || hard >= soft) {
- ddq->d_ino_hardlimit = cpu_to_be64(hard);
- ddq->d_ino_softlimit = cpu_to_be64(soft);
- if (id == 0) {
- defq->ihardlimit = hard;
- defq->isoftlimit = soft;
- }
- } else {
- xfs_debug(mp, "ihard %Ld < isoft %Ld", hard, soft);
- }
-
- /*
- * Update warnings counter(s) if requested
- */
- if (newlim->d_fieldmask & QC_SPC_WARNS)
- ddq->d_bwarns = cpu_to_be16(newlim->d_spc_warns);
- if (newlim->d_fieldmask & QC_INO_WARNS)
- ddq->d_iwarns = cpu_to_be16(newlim->d_ino_warns);
- if (newlim->d_fieldmask & QC_RT_SPC_WARNS)
- ddq->d_rtbwarns = cpu_to_be16(newlim->d_rt_spc_warns);
-
- if (id == 0) {
- if (newlim->d_fieldmask & QC_SPC_WARNS)
- defq->bwarnlimit = newlim->d_spc_warns;
- if (newlim->d_fieldmask & QC_INO_WARNS)
- defq->iwarnlimit = newlim->d_ino_warns;
- if (newlim->d_fieldmask & QC_RT_SPC_WARNS)
- defq->rtbwarnlimit = newlim->d_rt_spc_warns;
- }
-
- /*
+ *
+ * Update warnings counter(s) if requested.
+ *
* Timelimits for the super user set the relative time the other users
* can be over quota for this file system. If it is zero a default is
* used. Ditto for the default soft and hard limit values (already
@@ -572,21 +558,55 @@
* For other IDs, userspace can bump out the grace period if over
* the soft limit.
*/
- if (newlim->d_fieldmask & QC_SPC_TIMER)
- ddq->d_btimer = cpu_to_be32(newlim->d_spc_timer);
- if (newlim->d_fieldmask & QC_INO_TIMER)
- ddq->d_itimer = cpu_to_be32(newlim->d_ino_timer);
- if (newlim->d_fieldmask & QC_RT_SPC_TIMER)
- ddq->d_rtbtimer = cpu_to_be32(newlim->d_rt_spc_timer);
- if (id == 0) {
- if (newlim->d_fieldmask & QC_SPC_TIMER)
- defq->btimelimit = newlim->d_spc_timer;
- if (newlim->d_fieldmask & QC_INO_TIMER)
- defq->itimelimit = newlim->d_ino_timer;
- if (newlim->d_fieldmask & QC_RT_SPC_TIMER)
- defq->rtbtimelimit = newlim->d_rt_spc_timer;
- }
+ /* Blocks on the data device. */
+ hard = (newlim->d_fieldmask & QC_SPC_HARD) ?
+ (xfs_qcnt_t) XFS_B_TO_FSB(mp, newlim->d_spc_hardlimit) :
+ dqp->q_blk.hardlimit;
+ soft = (newlim->d_fieldmask & QC_SPC_SOFT) ?
+ (xfs_qcnt_t) XFS_B_TO_FSB(mp, newlim->d_spc_softlimit) :
+ dqp->q_blk.softlimit;
+ res = &dqp->q_blk;
+ qlim = id == 0 ? &defq->blk : NULL;
+
+ if (xfs_setqlim_limits(mp, res, qlim, hard, soft, "blk"))
+ xfs_dquot_set_prealloc_limits(dqp);
+ if (newlim->d_fieldmask & QC_SPC_WARNS)
+ xfs_setqlim_warns(res, qlim, newlim->d_spc_warns);
+ if (newlim->d_fieldmask & QC_SPC_TIMER)
+ xfs_setqlim_timer(res, qlim, newlim->d_spc_timer);
+
+ /* Blocks on the realtime device. */
+ hard = (newlim->d_fieldmask & QC_RT_SPC_HARD) ?
+ (xfs_qcnt_t) XFS_B_TO_FSB(mp, newlim->d_rt_spc_hardlimit) :
+ dqp->q_rtb.hardlimit;
+ soft = (newlim->d_fieldmask & QC_RT_SPC_SOFT) ?
+ (xfs_qcnt_t) XFS_B_TO_FSB(mp, newlim->d_rt_spc_softlimit) :
+ dqp->q_rtb.softlimit;
+ res = &dqp->q_rtb;
+ qlim = id == 0 ? &defq->rtb : NULL;
+
+ xfs_setqlim_limits(mp, res, qlim, hard, soft, "rtb");
+ if (newlim->d_fieldmask & QC_RT_SPC_WARNS)
+ xfs_setqlim_warns(res, qlim, newlim->d_rt_spc_warns);
+ if (newlim->d_fieldmask & QC_RT_SPC_TIMER)
+ xfs_setqlim_timer(res, qlim, newlim->d_rt_spc_timer);
+
+ /* Inodes */
+ hard = (newlim->d_fieldmask & QC_INO_HARD) ?
+ (xfs_qcnt_t) newlim->d_ino_hardlimit :
+ dqp->q_ino.hardlimit;
+ soft = (newlim->d_fieldmask & QC_INO_SOFT) ?
+ (xfs_qcnt_t) newlim->d_ino_softlimit :
+ dqp->q_ino.softlimit;
+ res = &dqp->q_ino;
+ qlim = id == 0 ? &defq->ino : NULL;
+
+ xfs_setqlim_limits(mp, res, qlim, hard, soft, "ino");
+ if (newlim->d_fieldmask & QC_INO_WARNS)
+ xfs_setqlim_warns(res, qlim, newlim->d_ino_warns);
+ if (newlim->d_fieldmask & QC_INO_TIMER)
+ xfs_setqlim_timer(res, qlim, newlim->d_ino_timer);
if (id != 0) {
/*
@@ -596,9 +616,9 @@
* is on or off. We don't really want to bother with iterating
* over all ondisk dquots and turning the timers on/off.
*/
- xfs_qm_adjust_dqtimers(mp, dqp);
+ xfs_qm_adjust_dqtimers(dqp);
}
- dqp->dq_flags |= XFS_DQ_DIRTY;
+ dqp->q_flags |= XFS_DQFLAG_DIRTY;
xfs_trans_log_dquot(tp, dqp);
error = xfs_trans_commit(tp);
@@ -614,58 +634,46 @@
static void
xfs_qm_scall_getquota_fill_qc(
struct xfs_mount *mp,
- uint type,
+ xfs_dqtype_t type,
const struct xfs_dquot *dqp,
struct qc_dqblk *dst)
{
memset(dst, 0, sizeof(*dst));
- dst->d_spc_hardlimit =
- XFS_FSB_TO_B(mp, be64_to_cpu(dqp->q_core.d_blk_hardlimit));
- dst->d_spc_softlimit =
- XFS_FSB_TO_B(mp, be64_to_cpu(dqp->q_core.d_blk_softlimit));
- dst->d_ino_hardlimit = be64_to_cpu(dqp->q_core.d_ino_hardlimit);
- dst->d_ino_softlimit = be64_to_cpu(dqp->q_core.d_ino_softlimit);
- dst->d_space = XFS_FSB_TO_B(mp, dqp->q_res_bcount);
- dst->d_ino_count = dqp->q_res_icount;
- dst->d_spc_timer = be32_to_cpu(dqp->q_core.d_btimer);
- dst->d_ino_timer = be32_to_cpu(dqp->q_core.d_itimer);
- dst->d_ino_warns = be16_to_cpu(dqp->q_core.d_iwarns);
- dst->d_spc_warns = be16_to_cpu(dqp->q_core.d_bwarns);
- dst->d_rt_spc_hardlimit =
- XFS_FSB_TO_B(mp, be64_to_cpu(dqp->q_core.d_rtb_hardlimit));
- dst->d_rt_spc_softlimit =
- XFS_FSB_TO_B(mp, be64_to_cpu(dqp->q_core.d_rtb_softlimit));
- dst->d_rt_space = XFS_FSB_TO_B(mp, dqp->q_res_rtbcount);
- dst->d_rt_spc_timer = be32_to_cpu(dqp->q_core.d_rtbtimer);
- dst->d_rt_spc_warns = be16_to_cpu(dqp->q_core.d_rtbwarns);
+ dst->d_spc_hardlimit = XFS_FSB_TO_B(mp, dqp->q_blk.hardlimit);
+ dst->d_spc_softlimit = XFS_FSB_TO_B(mp, dqp->q_blk.softlimit);
+ dst->d_ino_hardlimit = dqp->q_ino.hardlimit;
+ dst->d_ino_softlimit = dqp->q_ino.softlimit;
+ dst->d_space = XFS_FSB_TO_B(mp, dqp->q_blk.reserved);
+ dst->d_ino_count = dqp->q_ino.reserved;
+ dst->d_spc_timer = dqp->q_blk.timer;
+ dst->d_ino_timer = dqp->q_ino.timer;
+ dst->d_ino_warns = dqp->q_ino.warnings;
+ dst->d_spc_warns = dqp->q_blk.warnings;
+ dst->d_rt_spc_hardlimit = XFS_FSB_TO_B(mp, dqp->q_rtb.hardlimit);
+ dst->d_rt_spc_softlimit = XFS_FSB_TO_B(mp, dqp->q_rtb.softlimit);
+ dst->d_rt_space = XFS_FSB_TO_B(mp, dqp->q_rtb.reserved);
+ dst->d_rt_spc_timer = dqp->q_rtb.timer;
+ dst->d_rt_spc_warns = dqp->q_rtb.warnings;
/*
* Internally, we don't reset all the timers when quota enforcement
* gets turned off. No need to confuse the user level code,
* so return zeroes in that case.
*/
- if ((!XFS_IS_UQUOTA_ENFORCED(mp) &&
- dqp->q_core.d_flags == XFS_DQ_USER) ||
- (!XFS_IS_GQUOTA_ENFORCED(mp) &&
- dqp->q_core.d_flags == XFS_DQ_GROUP) ||
- (!XFS_IS_PQUOTA_ENFORCED(mp) &&
- dqp->q_core.d_flags == XFS_DQ_PROJ)) {
+ if (!xfs_dquot_is_enforced(dqp)) {
dst->d_spc_timer = 0;
dst->d_ino_timer = 0;
dst->d_rt_spc_timer = 0;
}
#ifdef DEBUG
- if (((XFS_IS_UQUOTA_ENFORCED(mp) && type == XFS_DQ_USER) ||
- (XFS_IS_GQUOTA_ENFORCED(mp) && type == XFS_DQ_GROUP) ||
- (XFS_IS_PQUOTA_ENFORCED(mp) && type == XFS_DQ_PROJ)) &&
- dqp->q_core.d_id != 0) {
+ if (xfs_dquot_is_enforced(dqp) && dqp->q_id != 0) {
if ((dst->d_space > dst->d_spc_softlimit) &&
(dst->d_spc_softlimit > 0)) {
ASSERT(dst->d_spc_timer != 0);
}
- if ((dst->d_ino_count > dst->d_ino_softlimit) &&
- (dst->d_ino_softlimit > 0)) {
+ if ((dst->d_ino_count > dqp->q_ino.softlimit) &&
+ (dqp->q_ino.softlimit > 0)) {
ASSERT(dst->d_ino_timer != 0);
}
}
@@ -677,7 +685,7 @@
xfs_qm_scall_getquota(
struct xfs_mount *mp,
xfs_dqid_t id,
- uint type,
+ xfs_dqtype_t type,
struct qc_dqblk *dst)
{
struct xfs_dquot *dqp;
@@ -715,7 +723,7 @@
xfs_qm_scall_getquota_next(
struct xfs_mount *mp,
xfs_dqid_t *id,
- uint type,
+ xfs_dqtype_t type,
struct qc_dqblk *dst)
{
struct xfs_dquot *dqp;
@@ -726,7 +734,7 @@
return error;
/* Fill in the ID we actually read from disk */
- *id = be32_to_cpu(dqp->q_core.d_id);
+ *id = dqp->q_id;
xfs_qm_scall_getquota_fill_qc(mp, type, dqp, dst);
diff --git a/fs/xfs/xfs_quota.h b/fs/xfs/xfs_quota.h
index aa8fc1f..06b22e3 100644
--- a/fs/xfs/xfs_quota.h
+++ b/fs/xfs/xfs_quota.h
@@ -13,6 +13,7 @@
*/
struct xfs_trans;
+struct xfs_buf;
/*
* This check is done typically without holding the inode lock;
@@ -38,14 +39,14 @@
static inline uint
xfs_quota_chkd_flag(
- uint dqtype)
+ xfs_dqtype_t type)
{
- switch (dqtype) {
- case XFS_DQ_USER:
+ switch (type) {
+ case XFS_DQTYPE_USER:
return XFS_UQUOTA_CHKD;
- case XFS_DQ_GROUP:
+ case XFS_DQTYPE_GROUP:
return XFS_GQUOTA_CHKD;
- case XFS_DQ_PROJ:
+ case XFS_DQTYPE_PROJ:
return XFS_PQUOTA_CHKD;
default:
return 0;
@@ -107,6 +108,8 @@
extern void xfs_qm_unmount(struct xfs_mount *);
extern void xfs_qm_unmount_quotas(struct xfs_mount *);
+void xfs_dquot_done(struct xfs_buf *);
+
#else
static inline int
xfs_qm_vop_dqalloc(struct xfs_inode *ip, kuid_t kuid, kgid_t kgid,
@@ -148,6 +151,12 @@
#define xfs_qm_mount_quotas(mp)
#define xfs_qm_unmount(mp)
#define xfs_qm_unmount_quotas(mp)
+
+static inline void xfs_dquot_done(struct xfs_buf *bp)
+{
+ return;
+}
+
#endif /* CONFIG_XFS_QUOTA */
#define xfs_trans_unreserve_quota_nblks(tp, ip, nblks, ninos, flags) \
diff --git a/fs/xfs/xfs_quotaops.c b/fs/xfs/xfs_quotaops.c
index bf809b7..d27c0e8 100644
--- a/fs/xfs/xfs_quotaops.c
+++ b/fs/xfs/xfs_quotaops.c
@@ -37,12 +37,12 @@
tstate->flags |= QCI_SYSFILE;
tstate->blocks = ip->i_d.di_nblocks;
tstate->nextents = ip->i_df.if_nextents;
- tstate->spc_timelimit = (u32)defq->btimelimit;
- tstate->ino_timelimit = (u32)defq->itimelimit;
- tstate->rt_spc_timelimit = (u32)defq->rtbtimelimit;
- tstate->spc_warnlimit = defq->bwarnlimit;
- tstate->ino_warnlimit = defq->iwarnlimit;
- tstate->rt_spc_warnlimit = defq->rtbwarnlimit;
+ tstate->spc_timelimit = (u32)defq->blk.time;
+ tstate->ino_timelimit = (u32)defq->ino.time;
+ tstate->rt_spc_timelimit = (u32)defq->rtb.time;
+ tstate->spc_warnlimit = defq->blk.warn;
+ tstate->ino_warnlimit = defq->ino.warn;
+ tstate->rt_spc_warnlimit = defq->rtb.warn;
if (tempqip)
xfs_irele(ip);
}
@@ -85,16 +85,16 @@
return 0;
}
-STATIC int
+STATIC xfs_dqtype_t
xfs_quota_type(int type)
{
switch (type) {
case USRQUOTA:
- return XFS_DQ_USER;
+ return XFS_DQTYPE_USER;
case GRPQUOTA:
- return XFS_DQ_GROUP;
+ return XFS_DQTYPE_GROUP;
default:
- return XFS_DQ_PROJ;
+ return XFS_DQTYPE_PROJ;
}
}
@@ -205,11 +205,11 @@
return -EINVAL;
if (uflags & FS_USER_QUOTA)
- flags |= XFS_DQ_USER;
+ flags |= XFS_QMOPT_UQUOTA;
if (uflags & FS_GROUP_QUOTA)
- flags |= XFS_DQ_GROUP;
+ flags |= XFS_QMOPT_GQUOTA;
if (uflags & FS_PROJ_QUOTA)
- flags |= XFS_DQ_PROJ;
+ flags |= XFS_QMOPT_PQUOTA;
return xfs_qm_scall_trunc_qfiles(mp, flags);
}
diff --git a/fs/xfs/xfs_refcount_item.c b/fs/xfs/xfs_refcount_item.c
index c816398..7b2c72b 100644
--- a/fs/xfs/xfs_refcount_item.c
+++ b/fs/xfs/xfs_refcount_item.c
@@ -143,7 +143,8 @@
cuip = kmem_zalloc(xfs_cui_log_item_sizeof(nextents),
0);
else
- cuip = kmem_zone_zalloc(xfs_cui_zone, 0);
+ cuip = kmem_cache_zalloc(xfs_cui_zone,
+ GFP_KERNEL | __GFP_NOFAIL);
xfs_log_item_init(mp, &cuip->cui_item, XFS_LI_CUI, &xfs_cui_item_ops);
cuip->cui_format.cui_nextents = nextents;
@@ -220,7 +221,7 @@
{
struct xfs_cud_log_item *cudp;
- cudp = kmem_zone_zalloc(xfs_cud_zone, 0);
+ cudp = kmem_cache_zalloc(xfs_cud_zone, GFP_KERNEL | __GFP_NOFAIL);
xfs_log_item_init(tp->t_mountp, &cudp->cud_item, XFS_LI_CUD,
&xfs_cud_item_ops);
cudp->cud_cuip = cuip;
diff --git a/fs/xfs/xfs_reflink.c b/fs/xfs/xfs_reflink.c
index 107bf2a..aac83f9 100644
--- a/fs/xfs/xfs_reflink.c
+++ b/fs/xfs/xfs_reflink.c
@@ -179,7 +179,7 @@
int error = 0;
/* Holes, unwritten, and delalloc extents cannot be shared */
- if (!xfs_is_cow_inode(ip) || !xfs_bmap_is_real_extent(irec)) {
+ if (!xfs_is_cow_inode(ip) || !xfs_bmap_is_written_extent(irec)) {
*shared = false;
return 0;
}
@@ -655,7 +655,7 @@
* preallocations can leak into the range we are called upon, and we
* need to skip them.
*/
- if (!xfs_bmap_is_real_extent(&got)) {
+ if (!xfs_bmap_is_written_extent(&got)) {
*end_fsb = del.br_startoff;
goto out_cancel;
}
@@ -984,40 +984,28 @@
}
/*
- * Unmap a range of blocks from a file, then map other blocks into the hole.
- * The range to unmap is (destoff : destoff + srcioff + irec->br_blockcount).
- * The extent irec is mapped into dest at irec->br_startoff.
+ * Remap the given extent into the file. The dmap blockcount will be set to
+ * the number of blocks that were actually remapped.
*/
STATIC int
xfs_reflink_remap_extent(
struct xfs_inode *ip,
- struct xfs_bmbt_irec *irec,
- xfs_fileoff_t destoff,
+ struct xfs_bmbt_irec *dmap,
xfs_off_t new_isize)
{
+ struct xfs_bmbt_irec smap;
struct xfs_mount *mp = ip->i_mount;
- bool real_extent = xfs_bmap_is_real_extent(irec);
struct xfs_trans *tp;
- unsigned int resblks;
- struct xfs_bmbt_irec uirec;
- xfs_filblks_t rlen;
- xfs_filblks_t unmap_len;
xfs_off_t newlen;
+ int64_t qres, qdelta;
+ unsigned int resblks;
+ bool smap_real;
+ bool dmap_written = xfs_bmap_is_written_extent(dmap);
+ int nimaps;
int error;
- unmap_len = irec->br_startoff + irec->br_blockcount - destoff;
- trace_xfs_reflink_punch_range(ip, destoff, unmap_len);
-
- /* No reflinking if we're low on space */
- if (real_extent) {
- error = xfs_reflink_ag_has_free_space(mp,
- XFS_FSB_TO_AGNO(mp, irec->br_startblock));
- if (error)
- goto out;
- }
-
/* Start a rolling transaction to switch the mappings */
- resblks = XFS_EXTENTADD_SPACE_RES(ip->i_mount, XFS_DATA_FORK);
+ resblks = XFS_EXTENTADD_SPACE_RES(mp, XFS_DATA_FORK);
error = xfs_trans_alloc(mp, &M_RES(mp)->tr_write, resblks, 0, 0, &tp);
if (error)
goto out;
@@ -1025,87 +1013,147 @@
xfs_ilock(ip, XFS_ILOCK_EXCL);
xfs_trans_ijoin(tp, ip, 0);
- /* If we're not just clearing space, then do we have enough quota? */
- if (real_extent) {
- error = xfs_trans_reserve_quota_nblks(tp, ip,
- irec->br_blockcount, 0, XFS_QMOPT_RES_REGBLKS);
+ /*
+ * Read what's currently mapped in the destination file into smap.
+ * If smap isn't a hole, we will have to remove it before we can add
+ * dmap to the destination file.
+ */
+ nimaps = 1;
+ error = xfs_bmapi_read(ip, dmap->br_startoff, dmap->br_blockcount,
+ &smap, &nimaps, 0);
+ if (error)
+ goto out_cancel;
+ ASSERT(nimaps == 1 && smap.br_startoff == dmap->br_startoff);
+ smap_real = xfs_bmap_is_real_extent(&smap);
+
+ /*
+ * We can only remap as many blocks as the smaller of the two extent
+ * maps, because we can only remap one extent at a time.
+ */
+ dmap->br_blockcount = min(dmap->br_blockcount, smap.br_blockcount);
+ ASSERT(dmap->br_blockcount == smap.br_blockcount);
+
+ trace_xfs_reflink_remap_extent_dest(ip, &smap);
+
+ /*
+ * Two extents mapped to the same physical block must not have
+ * different states; that's filesystem corruption. Move on to the next
+ * extent if they're both holes or both the same physical extent.
+ */
+ if (dmap->br_startblock == smap.br_startblock) {
+ if (dmap->br_state != smap.br_state)
+ error = -EFSCORRUPTED;
+ goto out_cancel;
+ }
+
+ /* If both extents are unwritten, leave them alone. */
+ if (dmap->br_state == XFS_EXT_UNWRITTEN &&
+ smap.br_state == XFS_EXT_UNWRITTEN)
+ goto out_cancel;
+
+ /* No reflinking if the AG of the dest mapping is low on space. */
+ if (dmap_written) {
+ error = xfs_reflink_ag_has_free_space(mp,
+ XFS_FSB_TO_AGNO(mp, dmap->br_startblock));
if (error)
goto out_cancel;
}
- trace_xfs_reflink_remap(ip, irec->br_startoff,
- irec->br_blockcount, irec->br_startblock);
-
- /* Unmap the old blocks in the data fork. */
- rlen = unmap_len;
- while (rlen) {
- ASSERT(tp->t_firstblock == NULLFSBLOCK);
- error = __xfs_bunmapi(tp, ip, destoff, &rlen, 0, 1);
+ /*
+ * Compute quota reservation if we think the quota block counter for
+ * this file could increase.
+ *
+ * Adding a written extent to the extent map can cause a bmbt split,
+ * and removing a mapped extent from the extent can cause a bmbt split.
+ * The two operations cannot both cause a split since they operate on
+ * the same index in the bmap btree, so we only need a reservation for
+ * one bmbt split if either thing is happening.
+ *
+ * If we are mapping a written extent into the file, we need to have
+ * enough quota block count reservation to handle the blocks in that
+ * extent. We log only the delta to the quota block counts, so if the
+ * extent we're unmapping also has blocks allocated to it, we don't
+ * need a quota reservation for the extent itself.
+ *
+ * Note that if we're replacing a delalloc reservation with a written
+ * extent, we have to take the full quota reservation because removing
+ * the delalloc reservation gives the block count back to the quota
+ * count. This is suboptimal, but the VFS flushed the dest range
+ * before we started. That should have removed all the delalloc
+ * reservations, but we code defensively.
+ */
+ qres = qdelta = 0;
+ if (smap_real || dmap_written)
+ qres = XFS_EXTENTADD_SPACE_RES(mp, XFS_DATA_FORK);
+ if (!smap_real && dmap_written)
+ qres += dmap->br_blockcount;
+ if (qres > 0) {
+ error = xfs_trans_reserve_quota_nblks(tp, ip, qres, 0,
+ XFS_QMOPT_RES_REGBLKS);
if (error)
goto out_cancel;
+ }
+
+ if (smap_real) {
+ /*
+ * If the extent we're unmapping is backed by storage (written
+ * or not), unmap the extent and drop its refcount.
+ */
+ xfs_bmap_unmap_extent(tp, ip, &smap);
+ xfs_refcount_decrease_extent(tp, &smap);
+ qdelta -= smap.br_blockcount;
+ } else if (smap.br_startblock == DELAYSTARTBLOCK) {
+ xfs_filblks_t len = smap.br_blockcount;
/*
- * Trim the extent to whatever got unmapped.
- * Remember, bunmapi works backwards.
+ * If the extent we're unmapping is a delalloc reservation,
+ * we can use the regular bunmapi function to release the
+ * incore state. Dropping the delalloc reservation takes care
+ * of the quota reservation for us.
*/
- uirec.br_startblock = irec->br_startblock + rlen;
- uirec.br_startoff = irec->br_startoff + rlen;
- uirec.br_blockcount = unmap_len - rlen;
- uirec.br_state = irec->br_state;
- unmap_len = rlen;
-
- /* If this isn't a real mapping, we're done. */
- if (!real_extent || uirec.br_blockcount == 0)
- goto next_extent;
-
- trace_xfs_reflink_remap(ip, uirec.br_startoff,
- uirec.br_blockcount, uirec.br_startblock);
-
- /* Update the refcount tree */
- xfs_refcount_increase_extent(tp, &uirec);
-
- /* Map the new blocks into the data fork. */
- xfs_bmap_map_extent(tp, ip, &uirec);
-
- /* Update quota accounting. */
- xfs_trans_mod_dquot_byino(tp, ip, XFS_TRANS_DQ_BCOUNT,
- uirec.br_blockcount);
-
- /* Update dest isize if needed. */
- newlen = XFS_FSB_TO_B(mp,
- uirec.br_startoff + uirec.br_blockcount);
- newlen = min_t(xfs_off_t, newlen, new_isize);
- if (newlen > i_size_read(VFS_I(ip))) {
- trace_xfs_reflink_update_inode_size(ip, newlen);
- i_size_write(VFS_I(ip), newlen);
- ip->i_d.di_size = newlen;
- xfs_trans_log_inode(tp, ip, XFS_ILOG_CORE);
- }
-
-next_extent:
- /* Process all the deferred stuff. */
- error = xfs_defer_finish(&tp);
+ error = __xfs_bunmapi(NULL, ip, smap.br_startoff, &len, 0, 1);
if (error)
goto out_cancel;
+ ASSERT(len == 0);
}
+ /*
+ * If the extent we're sharing is backed by written storage, increase
+ * its refcount and map it into the file.
+ */
+ if (dmap_written) {
+ xfs_refcount_increase_extent(tp, dmap);
+ xfs_bmap_map_extent(tp, ip, dmap);
+ qdelta += dmap->br_blockcount;
+ }
+
+ xfs_trans_mod_dquot_byino(tp, ip, XFS_TRANS_DQ_BCOUNT, qdelta);
+
+ /* Update dest isize if needed. */
+ newlen = XFS_FSB_TO_B(mp, dmap->br_startoff + dmap->br_blockcount);
+ newlen = min_t(xfs_off_t, newlen, new_isize);
+ if (newlen > i_size_read(VFS_I(ip))) {
+ trace_xfs_reflink_update_inode_size(ip, newlen);
+ i_size_write(VFS_I(ip), newlen);
+ ip->i_d.di_size = newlen;
+ xfs_trans_log_inode(tp, ip, XFS_ILOG_CORE);
+ }
+
+ /* Commit everything and unlock. */
error = xfs_trans_commit(tp);
- xfs_iunlock(ip, XFS_ILOCK_EXCL);
- if (error)
- goto out;
- return 0;
+ goto out_unlock;
out_cancel:
xfs_trans_cancel(tp);
+out_unlock:
xfs_iunlock(ip, XFS_ILOCK_EXCL);
out:
- trace_xfs_reflink_remap_extent_error(ip, error, _RET_IP_);
+ if (error)
+ trace_xfs_reflink_remap_extent_error(ip, error, _RET_IP_);
return error;
}
-/*
- * Iteratively remap one file's extents (and holes) to another's.
- */
+/* Remap a range of one file to the other. */
int
xfs_reflink_remap_blocks(
struct xfs_inode *src,
@@ -1116,25 +1164,22 @@
loff_t *remapped)
{
struct xfs_bmbt_irec imap;
- xfs_fileoff_t srcoff;
- xfs_fileoff_t destoff;
+ struct xfs_mount *mp = src->i_mount;
+ xfs_fileoff_t srcoff = XFS_B_TO_FSBT(mp, pos_in);
+ xfs_fileoff_t destoff = XFS_B_TO_FSBT(mp, pos_out);
xfs_filblks_t len;
- xfs_filblks_t range_len;
xfs_filblks_t remapped_len = 0;
xfs_off_t new_isize = pos_out + remap_len;
int nimaps;
int error = 0;
- destoff = XFS_B_TO_FSBT(src->i_mount, pos_out);
- srcoff = XFS_B_TO_FSBT(src->i_mount, pos_in);
- len = XFS_B_TO_FSB(src->i_mount, remap_len);
+ len = min_t(xfs_filblks_t, XFS_B_TO_FSB(mp, remap_len),
+ XFS_MAX_FILEOFF);
- /* drange = (destoff, destoff + len); srange = (srcoff, srcoff + len) */
- while (len) {
- uint lock_mode;
+ trace_xfs_reflink_remap_blocks(src, srcoff, len, dest, destoff);
- trace_xfs_reflink_remap_blocks_loop(src, srcoff, len,
- dest, destoff);
+ while (len > 0) {
+ unsigned int lock_mode;
/* Read extent from the source file */
nimaps = 1;
@@ -1143,18 +1188,25 @@
xfs_iunlock(src, lock_mode);
if (error)
break;
- ASSERT(nimaps == 1);
+ /*
+ * The caller supposedly flushed all dirty pages in the source
+ * file range, which means that writeback should have allocated
+ * or deleted all delalloc reservations in that range. If we
+ * find one, that's a good sign that something is seriously
+ * wrong here.
+ */
+ ASSERT(nimaps == 1 && imap.br_startoff == srcoff);
+ if (imap.br_startblock == DELAYSTARTBLOCK) {
+ ASSERT(imap.br_startblock != DELAYSTARTBLOCK);
+ error = -EFSCORRUPTED;
+ break;
+ }
- trace_xfs_reflink_remap_imap(src, srcoff, len, XFS_DATA_FORK,
- &imap);
+ trace_xfs_reflink_remap_extent_src(src, &imap);
- /* Translate imap into the destination file. */
- range_len = imap.br_startoff + imap.br_blockcount - srcoff;
- imap.br_startoff += destoff - srcoff;
-
- /* Clear dest from destoff to the end of imap and map it in. */
- error = xfs_reflink_remap_extent(dest, &imap, destoff,
- new_isize);
+ /* Remap into the destination file at the given offset. */
+ imap.br_startoff = destoff;
+ error = xfs_reflink_remap_extent(dest, &imap, new_isize);
if (error)
break;
@@ -1164,10 +1216,10 @@
}
/* Advance drange/srange */
- srcoff += range_len;
- destoff += range_len;
- len -= range_len;
- remapped_len += range_len;
+ srcoff += imap.br_blockcount;
+ destoff += imap.br_blockcount;
+ len -= imap.br_blockcount;
+ remapped_len += imap.br_blockcount;
}
if (error)
@@ -1178,81 +1230,6 @@
}
/*
- * Grab the exclusive iolock for a data copy from src to dest, making sure to
- * abide vfs locking order (lowest pointer value goes first) and breaking the
- * layout leases before proceeding. The loop is needed because we cannot call
- * the blocking break_layout() with the iolocks held, and therefore have to
- * back out both locks.
- */
-static int
-xfs_iolock_two_inodes_and_break_layout(
- struct inode *src,
- struct inode *dest)
-{
- int error;
-
- if (src > dest)
- swap(src, dest);
-
-retry:
- /* Wait to break both inodes' layouts before we start locking. */
- error = break_layout(src, true);
- if (error)
- return error;
- if (src != dest) {
- error = break_layout(dest, true);
- if (error)
- return error;
- }
-
- /* Lock one inode and make sure nobody got in and leased it. */
- inode_lock(src);
- error = break_layout(src, false);
- if (error) {
- inode_unlock(src);
- if (error == -EWOULDBLOCK)
- goto retry;
- return error;
- }
-
- if (src == dest)
- return 0;
-
- /* Lock the other inode and make sure nobody got in and leased it. */
- inode_lock_nested(dest, I_MUTEX_NONDIR2);
- error = break_layout(dest, false);
- if (error) {
- inode_unlock(src);
- inode_unlock(dest);
- if (error == -EWOULDBLOCK)
- goto retry;
- return error;
- }
-
- return 0;
-}
-
-/* Unlock both inodes after they've been prepped for a range clone. */
-void
-xfs_reflink_remap_unlock(
- struct file *file_in,
- struct file *file_out)
-{
- struct inode *inode_in = file_inode(file_in);
- struct xfs_inode *src = XFS_I(inode_in);
- struct inode *inode_out = file_inode(file_out);
- struct xfs_inode *dest = XFS_I(inode_out);
- bool same_inode = (inode_in == inode_out);
-
- xfs_iunlock(dest, XFS_MMAPLOCK_EXCL);
- if (!same_inode)
- xfs_iunlock(src, XFS_MMAPLOCK_EXCL);
- inode_unlock(inode_out);
- if (!same_inode)
- inode_unlock(inode_in);
-}
-
-/*
* If we're reflinking to a point past the destination file's EOF, we must
* zero any speculative post-EOF preallocations that sit between the old EOF
* and the destination file offset.
@@ -1314,18 +1291,12 @@
struct xfs_inode *src = XFS_I(inode_in);
struct inode *inode_out = file_inode(file_out);
struct xfs_inode *dest = XFS_I(inode_out);
- bool same_inode = (inode_in == inode_out);
- ssize_t ret;
+ int ret;
/* Lock both files against IO */
- ret = xfs_iolock_two_inodes_and_break_layout(inode_in, inode_out);
+ ret = xfs_ilock2_io_mmap(src, dest);
if (ret)
return ret;
- if (same_inode)
- xfs_ilock(src, XFS_MMAPLOCK_EXCL);
- else
- xfs_lock_two_inodes(src, XFS_MMAPLOCK_EXCL, dest,
- XFS_MMAPLOCK_EXCL);
/* Check file eligibility and prepare for block sharing. */
ret = -EINVAL;
@@ -1339,7 +1310,7 @@
ret = generic_remap_file_range_prep(file_in, pos_in, file_out, pos_out,
len, remap_flags);
- if (ret < 0 || *len == 0)
+ if (ret || *len == 0)
goto out_unlock;
/* Attach dquots to dest inode before changing block map */
@@ -1374,9 +1345,9 @@
if (ret)
goto out_unlock;
- return 1;
+ return 0;
out_unlock:
- xfs_reflink_remap_unlock(file_in, file_out);
+ xfs_iunlock2_io_mmap(src, dest);
return ret;
}
diff --git a/fs/xfs/xfs_reflink.h b/fs/xfs/xfs_reflink.h
index 3e4fd46..487b004 100644
--- a/fs/xfs/xfs_reflink.h
+++ b/fs/xfs/xfs_reflink.h
@@ -56,7 +56,5 @@
loff_t *remapped);
extern int xfs_reflink_update_dest(struct xfs_inode *dest, xfs_off_t newlen,
xfs_extlen_t cowextsize, unsigned int remap_flags);
-extern void xfs_reflink_remap_unlock(struct file *file_in,
- struct file *file_out);
#endif /* __XFS_REFLINK_H */
diff --git a/fs/xfs/xfs_rmap_item.c b/fs/xfs/xfs_rmap_item.c
index a86599d..dc5b075 100644
--- a/fs/xfs/xfs_rmap_item.c
+++ b/fs/xfs/xfs_rmap_item.c
@@ -141,7 +141,8 @@
if (nextents > XFS_RUI_MAX_FAST_EXTENTS)
ruip = kmem_zalloc(xfs_rui_log_item_sizeof(nextents), 0);
else
- ruip = kmem_zone_zalloc(xfs_rui_zone, 0);
+ ruip = kmem_cache_zalloc(xfs_rui_zone,
+ GFP_KERNEL | __GFP_NOFAIL);
xfs_log_item_init(mp, &ruip->rui_item, XFS_LI_RUI, &xfs_rui_item_ops);
ruip->rui_format.rui_nextents = nextents;
@@ -243,7 +244,7 @@
{
struct xfs_rud_log_item *rudp;
- rudp = kmem_zone_zalloc(xfs_rud_zone, 0);
+ rudp = kmem_cache_zalloc(xfs_rud_zone, GFP_KERNEL | __GFP_NOFAIL);
xfs_log_item_init(tp->t_mountp, &rudp->rud_item, XFS_LI_RUD,
&xfs_rud_item_ops);
rudp->rud_ruip = ruip;
diff --git a/fs/xfs/xfs_super.c b/fs/xfs/xfs_super.c
index 379cbff..71ac6c1 100644
--- a/fs/xfs/xfs_super.c
+++ b/fs/xfs/xfs_super.c
@@ -890,9 +890,6 @@
/* force the log to unpin objects from the now complete transactions */
xfs_log_force(mp, XFS_LOG_SYNC);
- /* reclaim inodes to do any IO before the freeze completes */
- xfs_reclaim_inodes(mp, 0);
- xfs_reclaim_inodes(mp, SYNC_WAIT);
/* Push the superblock and write an unmount record */
error = xfs_log_sbcount(mp);
@@ -913,11 +910,21 @@
struct super_block *sb)
{
struct xfs_mount *mp = XFS_M(sb);
+ unsigned int flags;
+ int ret;
+ /*
+ * The filesystem is now frozen far enough that memory reclaim
+ * cannot safely operate on the filesystem. Hence we need to
+ * set a GFP_NOFS context here to avoid recursion deadlocks.
+ */
+ flags = memalloc_nofs_save();
xfs_stop_block_reaping(mp);
xfs_save_resvblks(mp);
xfs_quiesce_attr(mp);
- return xfs_sync_sb(mp, true);
+ ret = xfs_sync_sb(mp, true);
+ memalloc_nofs_restore(flags);
+ return ret;
}
STATIC int
@@ -1714,6 +1721,10 @@
int flags = fc->sb_flags;
int error;
+ /* version 5 superblocks always support version counters. */
+ if (XFS_SB_VERSION_NUM(&mp->m_sb) == XFS_SB_VERSION_5)
+ fc->sb_flags |= SB_I_VERSION;
+
error = xfs_fc_validate_params(new_mp);
if (error)
return error;
diff --git a/fs/xfs/xfs_trace.h b/fs/xfs/xfs_trace.h
index 4601366..abb1d85 100644
--- a/fs/xfs/xfs_trace.h
+++ b/fs/xfs/xfs_trace.h
@@ -36,6 +36,7 @@
struct xfs_trans_res;
struct xfs_inobt_rec_incore;
union xfs_btree_ptr;
+struct xfs_dqtrx;
#define XFS_ATTR_FILTER_FLAGS \
{ XFS_ATTR_ROOT, "ROOT" }, \
@@ -864,44 +865,65 @@
TP_STRUCT__entry(
__field(dev_t, dev)
__field(u32, id)
+ __field(xfs_dqtype_t, type)
__field(unsigned, flags)
__field(unsigned, nrefs)
__field(unsigned long long, res_bcount)
+ __field(unsigned long long, res_rtbcount)
+ __field(unsigned long long, res_icount)
+
__field(unsigned long long, bcount)
+ __field(unsigned long long, rtbcount)
__field(unsigned long long, icount)
+
__field(unsigned long long, blk_hardlimit)
__field(unsigned long long, blk_softlimit)
+ __field(unsigned long long, rtb_hardlimit)
+ __field(unsigned long long, rtb_softlimit)
__field(unsigned long long, ino_hardlimit)
__field(unsigned long long, ino_softlimit)
- ), \
+ ),
TP_fast_assign(
__entry->dev = dqp->q_mount->m_super->s_dev;
- __entry->id = be32_to_cpu(dqp->q_core.d_id);
- __entry->flags = dqp->dq_flags;
+ __entry->id = dqp->q_id;
+ __entry->type = dqp->q_type;
+ __entry->flags = dqp->q_flags;
__entry->nrefs = dqp->q_nrefs;
- __entry->res_bcount = dqp->q_res_bcount;
- __entry->bcount = be64_to_cpu(dqp->q_core.d_bcount);
- __entry->icount = be64_to_cpu(dqp->q_core.d_icount);
- __entry->blk_hardlimit =
- be64_to_cpu(dqp->q_core.d_blk_hardlimit);
- __entry->blk_softlimit =
- be64_to_cpu(dqp->q_core.d_blk_softlimit);
- __entry->ino_hardlimit =
- be64_to_cpu(dqp->q_core.d_ino_hardlimit);
- __entry->ino_softlimit =
- be64_to_cpu(dqp->q_core.d_ino_softlimit);
+
+ __entry->res_bcount = dqp->q_blk.reserved;
+ __entry->res_rtbcount = dqp->q_rtb.reserved;
+ __entry->res_icount = dqp->q_ino.reserved;
+
+ __entry->bcount = dqp->q_blk.count;
+ __entry->rtbcount = dqp->q_rtb.count;
+ __entry->icount = dqp->q_ino.count;
+
+ __entry->blk_hardlimit = dqp->q_blk.hardlimit;
+ __entry->blk_softlimit = dqp->q_blk.softlimit;
+ __entry->rtb_hardlimit = dqp->q_rtb.hardlimit;
+ __entry->rtb_softlimit = dqp->q_rtb.softlimit;
+ __entry->ino_hardlimit = dqp->q_ino.hardlimit;
+ __entry->ino_softlimit = dqp->q_ino.softlimit;
),
- TP_printk("dev %d:%d id 0x%x flags %s nrefs %u res_bc 0x%llx "
+ TP_printk("dev %d:%d id 0x%x type %s flags %s nrefs %u "
+ "res_bc 0x%llx res_rtbc 0x%llx res_ic 0x%llx "
"bcnt 0x%llx bhardlimit 0x%llx bsoftlimit 0x%llx "
+ "rtbcnt 0x%llx rtbhardlimit 0x%llx rtbsoftlimit 0x%llx "
"icnt 0x%llx ihardlimit 0x%llx isoftlimit 0x%llx]",
MAJOR(__entry->dev), MINOR(__entry->dev),
__entry->id,
- __print_flags(__entry->flags, "|", XFS_DQ_FLAGS),
+ __print_flags(__entry->type, "|", XFS_DQTYPE_STRINGS),
+ __print_flags(__entry->flags, "|", XFS_DQFLAG_STRINGS),
__entry->nrefs,
__entry->res_bcount,
+ __entry->res_rtbcount,
+ __entry->res_icount,
__entry->bcount,
__entry->blk_hardlimit,
__entry->blk_softlimit,
+ __entry->rtbcount,
+ __entry->rtb_hardlimit,
+ __entry->rtb_softlimit,
__entry->icount,
__entry->ino_hardlimit,
__entry->ino_softlimit)
@@ -932,6 +954,125 @@
DEFINE_DQUOT_EVENT(xfs_dqflush);
DEFINE_DQUOT_EVENT(xfs_dqflush_force);
DEFINE_DQUOT_EVENT(xfs_dqflush_done);
+DEFINE_DQUOT_EVENT(xfs_trans_apply_dquot_deltas_before);
+DEFINE_DQUOT_EVENT(xfs_trans_apply_dquot_deltas_after);
+
+#define XFS_QMOPT_FLAGS \
+ { XFS_QMOPT_UQUOTA, "UQUOTA" }, \
+ { XFS_QMOPT_PQUOTA, "PQUOTA" }, \
+ { XFS_QMOPT_FORCE_RES, "FORCE_RES" }, \
+ { XFS_QMOPT_SBVERSION, "SBVERSION" }, \
+ { XFS_QMOPT_GQUOTA, "GQUOTA" }, \
+ { XFS_QMOPT_INHERIT, "INHERIT" }, \
+ { XFS_QMOPT_RES_REGBLKS, "RES_REGBLKS" }, \
+ { XFS_QMOPT_RES_RTBLKS, "RES_RTBLKS" }, \
+ { XFS_QMOPT_BCOUNT, "BCOUNT" }, \
+ { XFS_QMOPT_ICOUNT, "ICOUNT" }, \
+ { XFS_QMOPT_RTBCOUNT, "RTBCOUNT" }, \
+ { XFS_QMOPT_DELBCOUNT, "DELBCOUNT" }, \
+ { XFS_QMOPT_DELRTBCOUNT, "DELRTBCOUNT" }, \
+ { XFS_QMOPT_RES_INOS, "RES_INOS" }
+
+TRACE_EVENT(xfs_trans_mod_dquot,
+ TP_PROTO(struct xfs_trans *tp, struct xfs_dquot *dqp,
+ unsigned int field, int64_t delta),
+ TP_ARGS(tp, dqp, field, delta),
+ TP_STRUCT__entry(
+ __field(dev_t, dev)
+ __field(xfs_dqtype_t, type)
+ __field(unsigned int, flags)
+ __field(unsigned int, dqid)
+ __field(unsigned int, field)
+ __field(int64_t, delta)
+ ),
+ TP_fast_assign(
+ __entry->dev = tp->t_mountp->m_super->s_dev;
+ __entry->type = dqp->q_type;
+ __entry->flags = dqp->q_flags;
+ __entry->dqid = dqp->q_id;
+ __entry->field = field;
+ __entry->delta = delta;
+ ),
+ TP_printk("dev %d:%d dquot id 0x%x type %s flags %s field %s delta %lld",
+ MAJOR(__entry->dev), MINOR(__entry->dev),
+ __entry->dqid,
+ __print_flags(__entry->type, "|", XFS_DQTYPE_STRINGS),
+ __print_flags(__entry->flags, "|", XFS_DQFLAG_STRINGS),
+ __print_flags(__entry->field, "|", XFS_QMOPT_FLAGS),
+ __entry->delta)
+);
+
+DECLARE_EVENT_CLASS(xfs_dqtrx_class,
+ TP_PROTO(struct xfs_dqtrx *qtrx),
+ TP_ARGS(qtrx),
+ TP_STRUCT__entry(
+ __field(dev_t, dev)
+ __field(xfs_dqtype_t, type)
+ __field(unsigned int, flags)
+ __field(u32, dqid)
+
+ __field(uint64_t, blk_res)
+ __field(int64_t, bcount_delta)
+ __field(int64_t, delbcnt_delta)
+
+ __field(uint64_t, rtblk_res)
+ __field(uint64_t, rtblk_res_used)
+ __field(int64_t, rtbcount_delta)
+ __field(int64_t, delrtb_delta)
+
+ __field(uint64_t, ino_res)
+ __field(uint64_t, ino_res_used)
+ __field(int64_t, icount_delta)
+ ),
+ TP_fast_assign(
+ __entry->dev = qtrx->qt_dquot->q_mount->m_super->s_dev;
+ __entry->type = qtrx->qt_dquot->q_type;
+ __entry->flags = qtrx->qt_dquot->q_flags;
+ __entry->dqid = qtrx->qt_dquot->q_id;
+
+ __entry->blk_res = qtrx->qt_blk_res;
+ __entry->bcount_delta = qtrx->qt_bcount_delta;
+ __entry->delbcnt_delta = qtrx->qt_delbcnt_delta;
+
+ __entry->rtblk_res = qtrx->qt_rtblk_res;
+ __entry->rtblk_res_used = qtrx->qt_rtblk_res_used;
+ __entry->rtbcount_delta = qtrx->qt_rtbcount_delta;
+ __entry->delrtb_delta = qtrx->qt_delrtb_delta;
+
+ __entry->ino_res = qtrx->qt_ino_res;
+ __entry->ino_res_used = qtrx->qt_ino_res_used;
+ __entry->icount_delta = qtrx->qt_icount_delta;
+ ),
+ TP_printk("dev %d:%d dquot id 0x%x type %s flags %s"
+ "blk_res %llu bcount_delta %lld delbcnt_delta %lld "
+ "rtblk_res %llu rtblk_res_used %llu rtbcount_delta %lld delrtb_delta %lld "
+ "ino_res %llu ino_res_used %llu icount_delta %lld",
+ MAJOR(__entry->dev), MINOR(__entry->dev),
+ __entry->dqid,
+ __print_flags(__entry->type, "|", XFS_DQTYPE_STRINGS),
+ __print_flags(__entry->flags, "|", XFS_DQFLAG_STRINGS),
+
+ __entry->blk_res,
+ __entry->bcount_delta,
+ __entry->delbcnt_delta,
+
+ __entry->rtblk_res,
+ __entry->rtblk_res_used,
+ __entry->rtbcount_delta,
+ __entry->delrtb_delta,
+
+ __entry->ino_res,
+ __entry->ino_res_used,
+ __entry->icount_delta)
+)
+
+#define DEFINE_DQTRX_EVENT(name) \
+DEFINE_EVENT(xfs_dqtrx_class, name, \
+ TP_PROTO(struct xfs_dqtrx *qtrx), \
+ TP_ARGS(qtrx))
+DEFINE_DQTRX_EVENT(xfs_trans_apply_dquot_deltas);
+DEFINE_DQTRX_EVENT(xfs_trans_mod_dquot_before);
+DEFINE_DQTRX_EVENT(xfs_trans_mod_dquot_after);
DECLARE_EVENT_CLASS(xfs_loggrant_class,
TP_PROTO(struct xlog *log, struct xlog_ticket *tic),
@@ -3052,8 +3193,7 @@
DEFINE_INODE_EVENT(xfs_reflink_set_inode_flag);
DEFINE_INODE_EVENT(xfs_reflink_unset_inode_flag);
DEFINE_ITRUNC_EVENT(xfs_reflink_update_inode_size);
-DEFINE_IMAP_EVENT(xfs_reflink_remap_imap);
-TRACE_EVENT(xfs_reflink_remap_blocks_loop,
+TRACE_EVENT(xfs_reflink_remap_blocks,
TP_PROTO(struct xfs_inode *src, xfs_fileoff_t soffset,
xfs_filblks_t len, struct xfs_inode *dest,
xfs_fileoff_t doffset),
@@ -3084,59 +3224,14 @@
__entry->dest_ino,
__entry->dest_lblk)
);
-TRACE_EVENT(xfs_reflink_punch_range,
- TP_PROTO(struct xfs_inode *ip, xfs_fileoff_t lblk,
- xfs_extlen_t len),
- TP_ARGS(ip, lblk, len),
- TP_STRUCT__entry(
- __field(dev_t, dev)
- __field(xfs_ino_t, ino)
- __field(xfs_fileoff_t, lblk)
- __field(xfs_extlen_t, len)
- ),
- TP_fast_assign(
- __entry->dev = VFS_I(ip)->i_sb->s_dev;
- __entry->ino = ip->i_ino;
- __entry->lblk = lblk;
- __entry->len = len;
- ),
- TP_printk("dev %d:%d ino 0x%llx lblk 0x%llx len 0x%x",
- MAJOR(__entry->dev), MINOR(__entry->dev),
- __entry->ino,
- __entry->lblk,
- __entry->len)
-);
-TRACE_EVENT(xfs_reflink_remap,
- TP_PROTO(struct xfs_inode *ip, xfs_fileoff_t lblk,
- xfs_extlen_t len, xfs_fsblock_t new_pblk),
- TP_ARGS(ip, lblk, len, new_pblk),
- TP_STRUCT__entry(
- __field(dev_t, dev)
- __field(xfs_ino_t, ino)
- __field(xfs_fileoff_t, lblk)
- __field(xfs_extlen_t, len)
- __field(xfs_fsblock_t, new_pblk)
- ),
- TP_fast_assign(
- __entry->dev = VFS_I(ip)->i_sb->s_dev;
- __entry->ino = ip->i_ino;
- __entry->lblk = lblk;
- __entry->len = len;
- __entry->new_pblk = new_pblk;
- ),
- TP_printk("dev %d:%d ino 0x%llx lblk 0x%llx len 0x%x new_pblk %llu",
- MAJOR(__entry->dev), MINOR(__entry->dev),
- __entry->ino,
- __entry->lblk,
- __entry->len,
- __entry->new_pblk)
-);
DEFINE_DOUBLE_IO_EVENT(xfs_reflink_remap_range);
DEFINE_INODE_ERROR_EVENT(xfs_reflink_remap_range_error);
DEFINE_INODE_ERROR_EVENT(xfs_reflink_set_inode_flag_error);
DEFINE_INODE_ERROR_EVENT(xfs_reflink_update_inode_size_error);
DEFINE_INODE_ERROR_EVENT(xfs_reflink_remap_blocks_error);
DEFINE_INODE_ERROR_EVENT(xfs_reflink_remap_extent_error);
+DEFINE_INODE_IREC_EVENT(xfs_reflink_remap_extent_src);
+DEFINE_INODE_IREC_EVENT(xfs_reflink_remap_extent_dest);
/* dedupe tracepoints */
DEFINE_DOUBLE_IO_EVENT(xfs_reflink_compare_extents);
@@ -3582,7 +3677,6 @@
DEFINE_KMEM_EVENT(kmem_alloc_io);
DEFINE_KMEM_EVENT(kmem_alloc_large);
DEFINE_KMEM_EVENT(kmem_realloc);
-DEFINE_KMEM_EVENT(kmem_zone_alloc);
TRACE_EVENT(xfs_check_new_dalign,
TP_PROTO(struct xfs_mount *mp, int new_dalign, xfs_ino_t calc_rootino),
diff --git a/fs/xfs/xfs_trans.c b/fs/xfs/xfs_trans.c
index 3c94e5f..ed72867 100644
--- a/fs/xfs/xfs_trans.c
+++ b/fs/xfs/xfs_trans.c
@@ -90,7 +90,7 @@
trace_xfs_trans_dup(tp, _RET_IP_);
- ntp = kmem_zone_zalloc(xfs_trans_zone, 0);
+ ntp = kmem_cache_zalloc(xfs_trans_zone, GFP_KERNEL | __GFP_NOFAIL);
/*
* Initialize the new transaction structure.
@@ -107,7 +107,8 @@
ntp->t_flags = XFS_TRANS_PERM_LOG_RES |
(tp->t_flags & XFS_TRANS_RESERVE) |
- (tp->t_flags & XFS_TRANS_NO_WRITECOUNT);
+ (tp->t_flags & XFS_TRANS_NO_WRITECOUNT) |
+ (tp->t_flags & XFS_TRANS_RES_FDBLKS);
/* We gave our writer reference to the new transaction */
tp->t_flags |= XFS_TRANS_NO_WRITECOUNT;
ntp->t_ticket = xfs_log_ticket_get(tp->t_ticket);
@@ -262,7 +263,7 @@
* GFP_NOFS allocation context so that we avoid lockdep false positives
* by doing GFP_KERNEL allocations inside sb_start_intwrite().
*/
- tp = kmem_zone_zalloc(xfs_trans_zone, 0);
+ tp = kmem_cache_zalloc(xfs_trans_zone, GFP_KERNEL | __GFP_NOFAIL);
if (!(flags & XFS_TRANS_NO_WRITECOUNT))
sb_start_intwrite(mp->m_super);
@@ -272,6 +273,8 @@
*/
WARN_ON(resp->tr_logres > 0 &&
mp->m_super->s_writers.frozen == SB_FREEZE_COMPLETE);
+ ASSERT(!(flags & XFS_TRANS_RES_FDBLKS) ||
+ xfs_sb_version_haslazysbcount(&mp->m_sb));
tp->t_magic = XFS_TRANS_HEADER_MAGIC;
tp->t_flags = flags;
@@ -365,6 +368,20 @@
tp->t_blk_res_used += (uint)-delta;
if (tp->t_blk_res_used > tp->t_blk_res)
xfs_force_shutdown(mp, SHUTDOWN_CORRUPT_INCORE);
+ } else if (delta > 0 && (tp->t_flags & XFS_TRANS_RES_FDBLKS)) {
+ int64_t blkres_delta;
+
+ /*
+ * Return freed blocks directly to the reservation
+ * instead of the global pool, being careful not to
+ * overflow the trans counter. This is used to preserve
+ * reservation across chains of transaction rolls that
+ * repeatedly free and allocate blocks.
+ */
+ blkres_delta = min_t(int64_t, delta,
+ UINT_MAX - tp->t_blk_res);
+ tp->t_blk_res += blkres_delta;
+ delta -= blkres_delta;
}
tp->t_fdblocks_delta += delta;
if (xfs_sb_version_haslazysbcount(&mp->m_sb))
diff --git a/fs/xfs/xfs_trans.h b/fs/xfs/xfs_trans.h
index 8308bf6..b752501 100644
--- a/fs/xfs/xfs_trans.h
+++ b/fs/xfs/xfs_trans.h
@@ -37,10 +37,6 @@
unsigned long li_flags; /* misc flags */
struct xfs_buf *li_buf; /* real buffer pointer */
struct list_head li_bio_list; /* buffer item list */
- void (*li_cb)(struct xfs_buf *,
- struct xfs_log_item *);
- /* buffer item iodone */
- /* callback func */
const struct xfs_item_ops *li_ops; /* function list */
/* delayed logging */
@@ -78,7 +74,6 @@
void (*iop_committing)(struct xfs_log_item *, xfs_lsn_t commit_lsn);
void (*iop_release)(struct xfs_log_item *);
xfs_lsn_t (*iop_committed)(struct xfs_log_item *, xfs_lsn_t);
- void (*iop_error)(struct xfs_log_item *, xfs_buf_t *);
int (*iop_recover)(struct xfs_log_item *lip, struct xfs_trans *tp);
bool (*iop_match)(struct xfs_log_item *item, uint64_t id);
};
diff --git a/fs/xfs/xfs_trans_ail.c b/fs/xfs/xfs_trans_ail.c
index ac50193..0c783d3 100644
--- a/fs/xfs/xfs_trans_ail.c
+++ b/fs/xfs/xfs_trans_ail.c
@@ -377,8 +377,12 @@
}
/* protected by ail_lock */
- list_for_each_entry(lip, &bp->b_li_list, li_bio_list)
- xfs_clear_li_failed(lip);
+ list_for_each_entry(lip, &bp->b_li_list, li_bio_list) {
+ if (bp->b_flags & _XBF_INODES)
+ clear_bit(XFS_LI_FAILED, &lip->li_flags);
+ else
+ xfs_clear_li_failed(lip);
+ }
xfs_buf_unlock(bp);
return XFS_ITEM_SUCCESS;
@@ -444,16 +448,10 @@
target = ailp->ail_target;
ailp->ail_target_prev = target;
+ /* we're done if the AIL is empty or our push has reached the end */
lip = xfs_trans_ail_cursor_first(ailp, &cur, ailp->ail_last_pushed_lsn);
- if (!lip) {
- /*
- * If the AIL is empty or our push has reached the end we are
- * done now.
- */
- xfs_trans_ail_cursor_done(&cur);
- spin_unlock(&ailp->ail_lock);
+ if (!lip)
goto out_done;
- }
XFS_STATS_INC(mp, xs_push_ail);
@@ -535,6 +533,8 @@
break;
lsn = lip->li_lsn;
}
+
+out_done:
xfs_trans_ail_cursor_done(&cur);
spin_unlock(&ailp->ail_lock);
@@ -542,7 +542,6 @@
ailp->ail_log_flush++;
if (!count || XFS_LSN_CMP(lsn, target) >= 0) {
-out_done:
/*
* We reached the target or the AIL is empty, so wait a bit
* longer for I/O to complete and remove pushed items from the
@@ -634,7 +633,8 @@
*/
smp_rmb();
if (!xfs_ail_min(ailp) &&
- ailp->ail_target == ailp->ail_target_prev) {
+ ailp->ail_target == ailp->ail_target_prev &&
+ list_empty(&ailp->ail_buf_list)) {
spin_unlock(&ailp->ail_lock);
freezable_schedule();
tout = 0;
@@ -843,7 +843,6 @@
trace_xfs_ail_delete(lip, mlip->li_lsn, lip->li_lsn);
xfs_ail_delete(ailp, lip);
- xfs_clear_li_failed(lip);
clear_bit(XFS_LI_IN_AIL, &lip->li_flags);
lip->li_lsn = 0;
@@ -874,6 +873,7 @@
}
/* xfs_ail_update_finish() drops the AIL lock */
+ xfs_clear_li_failed(lip);
tail_lsn = xfs_ail_delete_one(ailp, lip);
xfs_ail_update_finish(ailp, tail_lsn);
}
diff --git a/fs/xfs/xfs_trans_buf.c b/fs/xfs/xfs_trans_buf.c
index 08174ff..11cd666 100644
--- a/fs/xfs/xfs_trans_buf.c
+++ b/fs/xfs/xfs_trans_buf.c
@@ -465,24 +465,16 @@
ASSERT(bp->b_transp == tp);
ASSERT(bip != NULL);
- ASSERT(bp->b_iodone == NULL ||
- bp->b_iodone == xfs_buf_iodone_callbacks);
/*
* Mark the buffer as needing to be written out eventually,
* and set its iodone function to remove the buffer's buf log
* item from the AIL and free it when the buffer is flushed
- * to disk. See xfs_buf_attach_iodone() for more details
- * on li_cb and xfs_buf_iodone_callbacks().
- * If we end up aborting this transaction, we trap this buffer
- * inside the b_bdstrat callback so that this won't get written to
- * disk.
+ * to disk.
*/
bp->b_flags |= XBF_DONE;
ASSERT(atomic_read(&bip->bli_refcount) > 0);
- bp->b_iodone = xfs_buf_iodone_callbacks;
- bip->bli_item.li_cb = xfs_buf_iodone;
/*
* If we invalidated the buffer within this transaction, then
@@ -626,6 +618,7 @@
ASSERT(atomic_read(&bip->bli_refcount) > 0);
bip->bli_flags |= XFS_BLI_INODE_BUF;
+ bp->b_flags |= _XBF_INODES;
xfs_trans_buf_set_type(tp, bp, XFS_BLFT_DINO_BUF);
}
@@ -650,7 +643,7 @@
ASSERT(atomic_read(&bip->bli_refcount) > 0);
bip->bli_flags |= XFS_BLI_STALE_INODE;
- bip->bli_item.li_cb = xfs_buf_iodone;
+ bp->b_flags |= _XBF_INODES;
xfs_trans_buf_set_type(tp, bp, XFS_BLFT_DINO_BUF);
}
@@ -675,6 +668,7 @@
ASSERT(atomic_read(&bip->bli_refcount) > 0);
bip->bli_flags |= XFS_BLI_INODE_ALLOC_BUF;
+ bp->b_flags |= _XBF_INODES;
xfs_trans_buf_set_type(tp, bp, XFS_BLFT_DINO_BUF);
}
@@ -785,5 +779,6 @@
break;
}
+ bp->b_flags |= _XBF_DQUOTS;
xfs_trans_buf_set_type(tp, bp, type);
}
diff --git a/fs/xfs/xfs_trans_dquot.c b/fs/xfs/xfs_trans_dquot.c
index c0f73b8..c6ba7ef 100644
--- a/fs/xfs/xfs_trans_dquot.c
+++ b/fs/xfs/xfs_trans_dquot.c
@@ -15,6 +15,7 @@
#include "xfs_trans_priv.h"
#include "xfs_quota.h"
#include "xfs_qm.h"
+#include "xfs_trace.h"
STATIC void xfs_trans_alloc_dqinfo(xfs_trans_t *);
@@ -155,14 +156,19 @@
int i;
struct xfs_dqtrx *qa;
- if (XFS_QM_ISUDQ(dqp))
+ switch (xfs_dquot_type(dqp)) {
+ case XFS_DQTYPE_USER:
qa = tp->t_dqinfo->dqs[XFS_QM_TRANS_USR];
- else if (XFS_QM_ISGDQ(dqp))
+ break;
+ case XFS_DQTYPE_GROUP:
qa = tp->t_dqinfo->dqs[XFS_QM_TRANS_GRP];
- else if (XFS_QM_ISPDQ(dqp))
+ break;
+ case XFS_DQTYPE_PROJ:
qa = tp->t_dqinfo->dqs[XFS_QM_TRANS_PRJ];
- else
+ break;
+ default:
return NULL;
+ }
for (i = 0; i < XFS_QM_TRANS_MAXDQS; i++) {
if (qa[i].qt_dquot == NULL ||
@@ -203,6 +209,11 @@
if (qtrx->qt_dquot == NULL)
qtrx->qt_dquot = dqp;
+ if (delta) {
+ trace_xfs_trans_mod_dquot_before(qtrx);
+ trace_xfs_trans_mod_dquot(tp, dqp, field, delta);
+ }
+
switch (field) {
/*
@@ -266,6 +277,10 @@
default:
ASSERT(0);
}
+
+ if (delta)
+ trace_xfs_trans_mod_dquot_after(qtrx);
+
tp->t_flags |= XFS_TRANS_DQ_DIRTY;
}
@@ -293,6 +308,37 @@
}
}
+/* Apply dqtrx changes to the quota reservation counters. */
+static inline void
+xfs_apply_quota_reservation_deltas(
+ struct xfs_dquot_res *res,
+ uint64_t reserved,
+ int64_t res_used,
+ int64_t count_delta)
+{
+ if (reserved != 0) {
+ /*
+ * Subtle math here: If reserved > res_used (the normal case),
+ * we're simply subtracting the unused transaction quota
+ * reservation from the dquot reservation.
+ *
+ * If, however, res_used > reserved, then we have allocated
+ * more quota blocks than were reserved for the transaction.
+ * We must add that excess to the dquot reservation since it
+ * tracks (usage + resv) and by definition we didn't reserve
+ * that excess.
+ */
+ res->reserved -= abs(reserved - res_used);
+ } else if (count_delta != 0) {
+ /*
+ * These blks were never reserved, either inside a transaction
+ * or outside one (in a delayed allocation). Also, this isn't
+ * always a negative number since we sometimes deliberately
+ * skip quota reservations.
+ */
+ res->reserved += count_delta;
+ }
+}
/*
* Called by xfs_trans_commit() and similar in spirit to
@@ -309,7 +355,6 @@
int i, j;
struct xfs_dquot *dqp;
struct xfs_dqtrx *qtrx, *qa;
- struct xfs_disk_dquot *d;
int64_t totalbdelta;
int64_t totalrtbdelta;
@@ -328,6 +373,8 @@
xfs_trans_dqlockedjoin(tp, qa);
for (i = 0; i < XFS_QM_TRANS_MAXDQS; i++) {
+ uint64_t blk_res_used;
+
qtrx = &qa[i];
/*
* The array of dquots is filled
@@ -341,7 +388,6 @@
/*
* adjust the actual number of blocks used
*/
- d = &dqp->q_core;
/*
* The issue here is - sometimes we don't make a blkquota
@@ -360,38 +406,46 @@
qtrx->qt_delbcnt_delta;
totalrtbdelta = qtrx->qt_rtbcount_delta +
qtrx->qt_delrtb_delta;
+
+ if (totalbdelta != 0 || totalrtbdelta != 0 ||
+ qtrx->qt_icount_delta != 0) {
+ trace_xfs_trans_apply_dquot_deltas_before(dqp);
+ trace_xfs_trans_apply_dquot_deltas(qtrx);
+ }
+
#ifdef DEBUG
if (totalbdelta < 0)
- ASSERT(be64_to_cpu(d->d_bcount) >=
- -totalbdelta);
+ ASSERT(dqp->q_blk.count >= -totalbdelta);
if (totalrtbdelta < 0)
- ASSERT(be64_to_cpu(d->d_rtbcount) >=
- -totalrtbdelta);
+ ASSERT(dqp->q_rtb.count >= -totalrtbdelta);
if (qtrx->qt_icount_delta < 0)
- ASSERT(be64_to_cpu(d->d_icount) >=
- -qtrx->qt_icount_delta);
+ ASSERT(dqp->q_ino.count >= -qtrx->qt_icount_delta);
#endif
if (totalbdelta)
- be64_add_cpu(&d->d_bcount, (xfs_qcnt_t)totalbdelta);
+ dqp->q_blk.count += totalbdelta;
if (qtrx->qt_icount_delta)
- be64_add_cpu(&d->d_icount, (xfs_qcnt_t)qtrx->qt_icount_delta);
+ dqp->q_ino.count += qtrx->qt_icount_delta;
if (totalrtbdelta)
- be64_add_cpu(&d->d_rtbcount, (xfs_qcnt_t)totalrtbdelta);
+ dqp->q_rtb.count += totalrtbdelta;
+
+ if (totalbdelta != 0 || totalrtbdelta != 0 ||
+ qtrx->qt_icount_delta != 0)
+ trace_xfs_trans_apply_dquot_deltas_after(dqp);
/*
* Get any default limits in use.
* Start/reset the timer(s) if needed.
*/
- if (d->d_id) {
- xfs_qm_adjust_dqlimits(tp->t_mountp, dqp);
- xfs_qm_adjust_dqtimers(tp->t_mountp, dqp);
+ if (dqp->q_id) {
+ xfs_qm_adjust_dqlimits(dqp);
+ xfs_qm_adjust_dqtimers(dqp);
}
- dqp->dq_flags |= XFS_DQ_DIRTY;
+ dqp->q_flags |= XFS_DQFLAG_DIRTY;
/*
* add this to the list of items to get logged
*/
@@ -401,78 +455,31 @@
* In case of delayed allocations, there's no
* reservation that a transaction structure knows of.
*/
- if (qtrx->qt_blk_res != 0) {
- uint64_t blk_res_used = 0;
+ blk_res_used = max_t(int64_t, 0, qtrx->qt_bcount_delta);
+ xfs_apply_quota_reservation_deltas(&dqp->q_blk,
+ qtrx->qt_blk_res, blk_res_used,
+ qtrx->qt_bcount_delta);
- if (qtrx->qt_bcount_delta > 0)
- blk_res_used = qtrx->qt_bcount_delta;
-
- if (qtrx->qt_blk_res != blk_res_used) {
- if (qtrx->qt_blk_res > blk_res_used)
- dqp->q_res_bcount -= (xfs_qcnt_t)
- (qtrx->qt_blk_res -
- blk_res_used);
- else
- dqp->q_res_bcount -= (xfs_qcnt_t)
- (blk_res_used -
- qtrx->qt_blk_res);
- }
- } else {
- /*
- * These blks were never reserved, either inside
- * a transaction or outside one (in a delayed
- * allocation). Also, this isn't always a
- * negative number since we sometimes
- * deliberately skip quota reservations.
- */
- if (qtrx->qt_bcount_delta) {
- dqp->q_res_bcount +=
- (xfs_qcnt_t)qtrx->qt_bcount_delta;
- }
- }
/*
* Adjust the RT reservation.
*/
- if (qtrx->qt_rtblk_res != 0) {
- if (qtrx->qt_rtblk_res != qtrx->qt_rtblk_res_used) {
- if (qtrx->qt_rtblk_res >
- qtrx->qt_rtblk_res_used)
- dqp->q_res_rtbcount -= (xfs_qcnt_t)
- (qtrx->qt_rtblk_res -
- qtrx->qt_rtblk_res_used);
- else
- dqp->q_res_rtbcount -= (xfs_qcnt_t)
- (qtrx->qt_rtblk_res_used -
- qtrx->qt_rtblk_res);
- }
- } else {
- if (qtrx->qt_rtbcount_delta)
- dqp->q_res_rtbcount +=
- (xfs_qcnt_t)qtrx->qt_rtbcount_delta;
- }
+ xfs_apply_quota_reservation_deltas(&dqp->q_rtb,
+ qtrx->qt_rtblk_res,
+ qtrx->qt_rtblk_res_used,
+ qtrx->qt_rtbcount_delta);
/*
* Adjust the inode reservation.
*/
- if (qtrx->qt_ino_res != 0) {
- ASSERT(qtrx->qt_ino_res >=
- qtrx->qt_ino_res_used);
- if (qtrx->qt_ino_res > qtrx->qt_ino_res_used)
- dqp->q_res_icount -= (xfs_qcnt_t)
- (qtrx->qt_ino_res -
- qtrx->qt_ino_res_used);
- } else {
- if (qtrx->qt_icount_delta)
- dqp->q_res_icount +=
- (xfs_qcnt_t)qtrx->qt_icount_delta;
- }
+ ASSERT(qtrx->qt_ino_res >= qtrx->qt_ino_res_used);
+ xfs_apply_quota_reservation_deltas(&dqp->q_ino,
+ qtrx->qt_ino_res,
+ qtrx->qt_ino_res_used,
+ qtrx->qt_icount_delta);
- ASSERT(dqp->q_res_bcount >=
- be64_to_cpu(dqp->q_core.d_bcount));
- ASSERT(dqp->q_res_icount >=
- be64_to_cpu(dqp->q_core.d_icount));
- ASSERT(dqp->q_res_rtbcount >=
- be64_to_cpu(dqp->q_core.d_rtbcount));
+ ASSERT(dqp->q_blk.reserved >= dqp->q_blk.count);
+ ASSERT(dqp->q_ino.reserved >= dqp->q_ino.count);
+ ASSERT(dqp->q_rtb.reserved >= dqp->q_rtb.count);
}
}
}
@@ -516,7 +523,7 @@
if (qtrx->qt_blk_res) {
xfs_dqlock(dqp);
locked = true;
- dqp->q_res_bcount -=
+ dqp->q_blk.reserved -=
(xfs_qcnt_t)qtrx->qt_blk_res;
}
if (qtrx->qt_ino_res) {
@@ -524,7 +531,7 @@
xfs_dqlock(dqp);
locked = true;
}
- dqp->q_res_icount -=
+ dqp->q_ino.reserved -=
(xfs_qcnt_t)qtrx->qt_ino_res;
}
@@ -533,7 +540,7 @@
xfs_dqlock(dqp);
locked = true;
}
- dqp->q_res_rtbcount -=
+ dqp->q_rtb.reserved -=
(xfs_qcnt_t)qtrx->qt_rtblk_res;
}
if (locked)
@@ -549,21 +556,80 @@
struct xfs_dquot *dqp,
int type)
{
- enum quota_type qtype;
+ enum quota_type qtype;
- if (dqp->dq_flags & XFS_DQ_PROJ)
+ switch (xfs_dquot_type(dqp)) {
+ case XFS_DQTYPE_PROJ:
qtype = PRJQUOTA;
- else if (dqp->dq_flags & XFS_DQ_USER)
+ break;
+ case XFS_DQTYPE_USER:
qtype = USRQUOTA;
- else
+ break;
+ case XFS_DQTYPE_GROUP:
qtype = GRPQUOTA;
+ break;
+ default:
+ return;
+ }
- quota_send_warning(make_kqid(&init_user_ns, qtype,
- be32_to_cpu(dqp->q_core.d_id)),
+ quota_send_warning(make_kqid(&init_user_ns, qtype, dqp->q_id),
mp->m_super->s_dev, type);
}
/*
+ * Decide if we can make an additional reservation against a quota resource.
+ * Returns an inode QUOTA_NL_ warning code and whether or not it's fatal.
+ *
+ * Note that we assume that the numeric difference between the inode and block
+ * warning codes will always be 3 since it's userspace ABI now, and will never
+ * decrease the quota reservation, so the *BELOW messages are irrelevant.
+ */
+static inline int
+xfs_dqresv_check(
+ struct xfs_dquot_res *res,
+ struct xfs_quota_limits *qlim,
+ int64_t delta,
+ bool *fatal)
+{
+ xfs_qcnt_t hardlimit = res->hardlimit;
+ xfs_qcnt_t softlimit = res->softlimit;
+ xfs_qcnt_t total_count = res->reserved + delta;
+
+ BUILD_BUG_ON(QUOTA_NL_BHARDWARN != QUOTA_NL_IHARDWARN + 3);
+ BUILD_BUG_ON(QUOTA_NL_BSOFTLONGWARN != QUOTA_NL_ISOFTLONGWARN + 3);
+ BUILD_BUG_ON(QUOTA_NL_BSOFTWARN != QUOTA_NL_ISOFTWARN + 3);
+
+ *fatal = false;
+ if (delta <= 0)
+ return QUOTA_NL_NOWARN;
+
+ if (!hardlimit)
+ hardlimit = qlim->hard;
+ if (!softlimit)
+ softlimit = qlim->soft;
+
+ if (hardlimit && total_count > hardlimit) {
+ *fatal = true;
+ return QUOTA_NL_IHARDWARN;
+ }
+
+ if (softlimit && total_count > softlimit) {
+ time64_t now = ktime_get_real_seconds();
+
+ if ((res->timer != 0 && now > res->timer) ||
+ (res->warnings != 0 && res->warnings >= qlim->warn)) {
+ *fatal = true;
+ return QUOTA_NL_ISOFTLONGWARN;
+ }
+
+ res->warnings++;
+ return QUOTA_NL_ISOFTWARN;
+ }
+
+ return QUOTA_NL_NOWARN;
+}
+
+/*
* This reserves disk blocks and inodes against a dquot.
* Flags indicate if the dquot is to be locked here and also
* if the blk reservation is for RT or regular blocks.
@@ -578,110 +644,58 @@
long ninos,
uint flags)
{
- xfs_qcnt_t hardlimit;
- xfs_qcnt_t softlimit;
- time64_t timer;
- xfs_qwarncnt_t warns;
- xfs_qwarncnt_t warnlimit;
- xfs_qcnt_t total_count;
- xfs_qcnt_t *resbcountp;
struct xfs_quotainfo *q = mp->m_quotainfo;
struct xfs_def_quota *defq;
-
+ struct xfs_dquot_res *blkres;
+ struct xfs_quota_limits *qlim;
xfs_dqlock(dqp);
defq = xfs_get_defquota(q, xfs_dquot_type(dqp));
if (flags & XFS_TRANS_DQ_RES_BLKS) {
- hardlimit = be64_to_cpu(dqp->q_core.d_blk_hardlimit);
- if (!hardlimit)
- hardlimit = defq->bhardlimit;
- softlimit = be64_to_cpu(dqp->q_core.d_blk_softlimit);
- if (!softlimit)
- softlimit = defq->bsoftlimit;
- timer = be32_to_cpu(dqp->q_core.d_btimer);
- warns = be16_to_cpu(dqp->q_core.d_bwarns);
- warnlimit = defq->bwarnlimit;
- resbcountp = &dqp->q_res_bcount;
+ blkres = &dqp->q_blk;
+ qlim = &defq->blk;
} else {
- ASSERT(flags & XFS_TRANS_DQ_RES_RTBLKS);
- hardlimit = be64_to_cpu(dqp->q_core.d_rtb_hardlimit);
- if (!hardlimit)
- hardlimit = defq->rtbhardlimit;
- softlimit = be64_to_cpu(dqp->q_core.d_rtb_softlimit);
- if (!softlimit)
- softlimit = defq->rtbsoftlimit;
- timer = be32_to_cpu(dqp->q_core.d_rtbtimer);
- warns = be16_to_cpu(dqp->q_core.d_rtbwarns);
- warnlimit = defq->rtbwarnlimit;
- resbcountp = &dqp->q_res_rtbcount;
+ blkres = &dqp->q_rtb;
+ qlim = &defq->rtb;
}
- if ((flags & XFS_QMOPT_FORCE_RES) == 0 &&
- dqp->q_core.d_id &&
- ((XFS_IS_UQUOTA_ENFORCED(dqp->q_mount) && XFS_QM_ISUDQ(dqp)) ||
- (XFS_IS_GQUOTA_ENFORCED(dqp->q_mount) && XFS_QM_ISGDQ(dqp)) ||
- (XFS_IS_PQUOTA_ENFORCED(dqp->q_mount) && XFS_QM_ISPDQ(dqp)))) {
- if (nblks > 0) {
+ if ((flags & XFS_QMOPT_FORCE_RES) == 0 && dqp->q_id &&
+ xfs_dquot_is_enforced(dqp)) {
+ int quota_nl;
+ bool fatal;
+
+ /*
+ * dquot is locked already. See if we'd go over the hardlimit
+ * or exceed the timelimit if we'd reserve resources.
+ */
+ quota_nl = xfs_dqresv_check(blkres, qlim, nblks, &fatal);
+ if (quota_nl != QUOTA_NL_NOWARN) {
/*
- * dquot is locked already. See if we'd go over the
- * hardlimit or exceed the timelimit if we allocate
- * nblks.
+ * Quota block warning codes are 3 more than the inode
+ * codes, which we check above.
*/
- total_count = *resbcountp + nblks;
- if (hardlimit && total_count > hardlimit) {
- xfs_quota_warn(mp, dqp, QUOTA_NL_BHARDWARN);
+ xfs_quota_warn(mp, dqp, quota_nl + 3);
+ if (fatal)
goto error_return;
- }
- if (softlimit && total_count > softlimit) {
- if ((timer != 0 &&
- ktime_get_real_seconds() > timer) ||
- (warns != 0 && warns >= warnlimit)) {
- xfs_quota_warn(mp, dqp,
- QUOTA_NL_BSOFTLONGWARN);
- goto error_return;
- }
-
- xfs_quota_warn(mp, dqp, QUOTA_NL_BSOFTWARN);
- }
}
- if (ninos > 0) {
- total_count = be64_to_cpu(dqp->q_core.d_icount) + ninos;
- timer = be32_to_cpu(dqp->q_core.d_itimer);
- warns = be16_to_cpu(dqp->q_core.d_iwarns);
- warnlimit = defq->iwarnlimit;
- hardlimit = be64_to_cpu(dqp->q_core.d_ino_hardlimit);
- if (!hardlimit)
- hardlimit = defq->ihardlimit;
- softlimit = be64_to_cpu(dqp->q_core.d_ino_softlimit);
- if (!softlimit)
- softlimit = defq->isoftlimit;
- if (hardlimit && total_count > hardlimit) {
- xfs_quota_warn(mp, dqp, QUOTA_NL_IHARDWARN);
+ quota_nl = xfs_dqresv_check(&dqp->q_ino, &defq->ino, ninos,
+ &fatal);
+ if (quota_nl != QUOTA_NL_NOWARN) {
+ xfs_quota_warn(mp, dqp, quota_nl);
+ if (fatal)
goto error_return;
- }
- if (softlimit && total_count > softlimit) {
- if ((timer != 0 &&
- ktime_get_real_seconds() > timer) ||
- (warns != 0 && warns >= warnlimit)) {
- xfs_quota_warn(mp, dqp,
- QUOTA_NL_ISOFTLONGWARN);
- goto error_return;
- }
- xfs_quota_warn(mp, dqp, QUOTA_NL_ISOFTWARN);
- }
}
}
/*
* Change the reservation, but not the actual usage.
- * Note that q_res_bcount = q_core.d_bcount + resv
+ * Note that q_blk.reserved = q_blk.count + resv
*/
- (*resbcountp) += (xfs_qcnt_t)nblks;
- if (ninos != 0)
- dqp->q_res_icount += (xfs_qcnt_t)ninos;
+ blkres->reserved += (xfs_qcnt_t)nblks;
+ dqp->q_ino.reserved += (xfs_qcnt_t)ninos;
/*
* note the reservation amt in the trans struct too,
@@ -702,16 +716,16 @@
XFS_TRANS_DQ_RES_INOS,
ninos);
}
- ASSERT(dqp->q_res_bcount >= be64_to_cpu(dqp->q_core.d_bcount));
- ASSERT(dqp->q_res_rtbcount >= be64_to_cpu(dqp->q_core.d_rtbcount));
- ASSERT(dqp->q_res_icount >= be64_to_cpu(dqp->q_core.d_icount));
+ ASSERT(dqp->q_blk.reserved >= dqp->q_blk.count);
+ ASSERT(dqp->q_rtb.reserved >= dqp->q_rtb.count);
+ ASSERT(dqp->q_ino.reserved >= dqp->q_ino.count);
xfs_dqunlock(dqp);
return 0;
error_return:
xfs_dqunlock(dqp);
- if (XFS_QM_ISPDQ(dqp))
+ if (xfs_dquot_type(dqp) == XFS_DQTYPE_PROJ)
return -ENOSPC;
return -EDQUOT;
}
@@ -860,7 +874,8 @@
xfs_trans_alloc_dqinfo(
xfs_trans_t *tp)
{
- tp->t_dqinfo = kmem_zone_zalloc(xfs_qm_dqtrxzone, 0);
+ tp->t_dqinfo = kmem_cache_zalloc(xfs_qm_dqtrxzone,
+ GFP_KERNEL | __GFP_NOFAIL);
}
void
diff --git a/fs/zonefs/super.c b/fs/zonefs/super.c
index abfb17f..4a0bff6 100644
--- a/fs/zonefs/super.c
+++ b/fs/zonefs/super.c
@@ -786,8 +786,11 @@
if (iocb->ki_pos >= ZONEFS_I(inode)->i_max_size)
return -EFBIG;
- if (iocb->ki_flags & IOCB_DIRECT)
- return zonefs_file_dio_write(iocb, from);
+ if (iocb->ki_flags & IOCB_DIRECT) {
+ ssize_t ret = zonefs_file_dio_write(iocb, from);
+ if (ret != -ENOTBLK)
+ return ret;
+ }
return zonefs_file_buffered_write(iocb, from);
}
diff --git a/include/asm-generic/barrier.h b/include/asm-generic/barrier.h
index fec97dc..798027b 100644
--- a/include/asm-generic/barrier.h
+++ b/include/asm-generic/barrier.h
@@ -240,5 +240,15 @@
})
#endif
+/*
+ * pmem_wmb() ensures that all stores for which the modification
+ * are written to persistent storage by preceding instructions have
+ * updated persistent storage before any data access or data transfer
+ * caused by subsequent instructions is initiated.
+ */
+#ifndef pmem_wmb
+#define pmem_wmb() wmb()
+#endif
+
#endif /* !__ASSEMBLY__ */
#endif /* __ASM_GENERIC_BARRIER_H */
diff --git a/include/asm-generic/qspinlock.h b/include/asm-generic/qspinlock.h
index 2b26cd7..4fe7fd0 100644
--- a/include/asm-generic/qspinlock.h
+++ b/include/asm-generic/qspinlock.h
@@ -13,6 +13,7 @@
#include <asm-generic/qspinlock_types.h>
#include <linux/atomic.h>
+#ifndef queued_spin_is_locked
/**
* queued_spin_is_locked - is the spinlock locked?
* @lock: Pointer to queued spinlock structure
@@ -26,6 +27,7 @@
*/
return atomic_read(&lock->val);
}
+#endif
/**
* queued_spin_value_unlocked - is the spinlock structure unlocked?
@@ -68,6 +70,7 @@
extern void queued_spin_lock_slowpath(struct qspinlock *lock, u32 val);
+#ifndef queued_spin_lock
/**
* queued_spin_lock - acquire a queued spinlock
* @lock: Pointer to queued spinlock structure
@@ -81,6 +84,7 @@
queued_spin_lock_slowpath(lock, val);
}
+#endif
#ifndef queued_spin_unlock
/**
diff --git a/include/linux/clock_cooling.h b/include/linux/clock_cooling.h
deleted file mode 100644
index 4b0a698..0000000
--- a/include/linux/clock_cooling.h
+++ /dev/null
@@ -1,57 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0-only */
-/*
- * linux/include/linux/clock_cooling.h
- *
- * Copyright (C) 2014 Eduardo Valentin <edubezval@gmail.com>
- *
- * Copyright (C) 2013 Texas Instruments Inc.
- * Contact: Eduardo Valentin <eduardo.valentin@ti.com>
- *
- * Highly based on cpufreq_cooling.c.
- * Copyright (C) 2012 Samsung Electronics Co., Ltd(http://www.samsung.com)
- * Copyright (C) 2012 Amit Daniel <amit.kachhap@linaro.org>
- */
-
-#ifndef __CPU_COOLING_H__
-#define __CPU_COOLING_H__
-
-#include <linux/of.h>
-#include <linux/thermal.h>
-#include <linux/cpumask.h>
-
-#ifdef CONFIG_CLOCK_THERMAL
-/**
- * clock_cooling_register - function to create clock cooling device.
- * @dev: struct device pointer to the device used as clock cooling device.
- * @clock_name: string containing the clock used as cooling mechanism.
- */
-struct thermal_cooling_device *
-clock_cooling_register(struct device *dev, const char *clock_name);
-
-/**
- * clock_cooling_unregister - function to remove clock cooling device.
- * @cdev: thermal cooling device pointer.
- */
-void clock_cooling_unregister(struct thermal_cooling_device *cdev);
-
-unsigned long clock_cooling_get_level(struct thermal_cooling_device *cdev,
- unsigned long freq);
-#else /* !CONFIG_CLOCK_THERMAL */
-static inline struct thermal_cooling_device *
-clock_cooling_register(struct device *dev, const char *clock_name)
-{
- return NULL;
-}
-static inline
-void clock_cooling_unregister(struct thermal_cooling_device *cdev)
-{
-}
-static inline
-unsigned long clock_cooling_get_level(struct thermal_cooling_device *cdev,
- unsigned long freq)
-{
- return THERMAL_CSTATE_INVALID;
-}
-#endif /* CONFIG_CLOCK_THERMAL */
-
-#endif /* __CPU_COOLING_H__ */
diff --git a/include/linux/cpuhotplug.h b/include/linux/cpuhotplug.h
index 191772d..a2710e6 100644
--- a/include/linux/cpuhotplug.h
+++ b/include/linux/cpuhotplug.h
@@ -181,6 +181,7 @@
CPUHP_AP_PERF_POWERPC_CORE_IMC_ONLINE,
CPUHP_AP_PERF_POWERPC_THREAD_IMC_ONLINE,
CPUHP_AP_PERF_POWERPC_TRACE_IMC_ONLINE,
+ CPUHP_AP_PERF_POWERPC_HV_24x7_ONLINE,
CPUHP_AP_WATCHDOG_ONLINE,
CPUHP_AP_WORKQUEUE_ONLINE,
CPUHP_AP_RCUTREE_ONLINE,
diff --git a/include/linux/devfreq_cooling.h b/include/linux/devfreq_cooling.h
index 79a6e37..9df2dfc 100644
--- a/include/linux/devfreq_cooling.h
+++ b/include/linux/devfreq_cooling.h
@@ -1,17 +1,10 @@
+/* SPDX-License-Identifier: GPL-2.0 */
/*
* devfreq_cooling: Thermal cooling device implementation for devices using
* devfreq
*
* Copyright (C) 2014-2015 ARM Limited
*
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
- *
- * This program is distributed "as is" WITHOUT ANY WARRANTY of any
- * kind, whether express or implied; without even the implied warranty
- * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU General Public License for more details.
*/
#ifndef __DEVFREQ_COOLING_H__
diff --git a/include/linux/fanotify.h b/include/linux/fanotify.h
index b79fa9bb..3e9c56e 100644
--- a/include/linux/fanotify.h
+++ b/include/linux/fanotify.h
@@ -18,8 +18,10 @@
#define FANOTIFY_CLASS_BITS (FAN_CLASS_NOTIF | FAN_CLASS_CONTENT | \
FAN_CLASS_PRE_CONTENT)
-#define FANOTIFY_INIT_FLAGS (FANOTIFY_CLASS_BITS | \
- FAN_REPORT_TID | FAN_REPORT_FID | \
+#define FANOTIFY_FID_BITS (FAN_REPORT_FID | FAN_REPORT_DFID_NAME)
+
+#define FANOTIFY_INIT_FLAGS (FANOTIFY_CLASS_BITS | FANOTIFY_FID_BITS | \
+ FAN_REPORT_TID | \
FAN_CLOEXEC | FAN_NONBLOCK | \
FAN_UNLIMITED_QUEUE | FAN_UNLIMITED_MARKS)
diff --git a/include/linux/fs.h b/include/linux/fs.h
index f5da33bc..2df72de 100644
--- a/include/linux/fs.h
+++ b/include/linux/fs.h
@@ -1712,6 +1712,10 @@
int (*f)(struct dentry *, umode_t, void *),
void *);
+int vfs_fchown(struct file *file, uid_t user, gid_t group);
+int vfs_fchmod(struct file *file, umode_t mode);
+int vfs_utimes(const struct path *path, struct timespec64 *times);
+
extern long vfs_ioctl(struct file *file, unsigned int cmd, unsigned long arg);
#ifdef CONFIG_COMPAT
diff --git a/include/linux/fsnotify.h b/include/linux/fsnotify.h
index 5ab28f6..f8acddc 100644
--- a/include/linux/fsnotify.h
+++ b/include/linux/fsnotify.h
@@ -23,19 +23,14 @@
* have changed (i.e. renamed over).
*
* Unlike fsnotify_parent(), the event will be reported regardless of the
- * FS_EVENT_ON_CHILD mask on the parent inode.
+ * FS_EVENT_ON_CHILD mask on the parent inode and will not be reported if only
+ * the child is interested and not the parent.
*/
static inline void fsnotify_name(struct inode *dir, __u32 mask,
struct inode *child,
const struct qstr *name, u32 cookie)
{
- fsnotify(dir, mask, child, FSNOTIFY_EVENT_INODE, name, cookie);
- /*
- * Send another flavor of the event without child inode data and
- * without the specific event type (e.g. FS_CREATE|FS_IS_DIR).
- * The name is relative to the dir inode the event is reported to.
- */
- fsnotify(dir, FS_DIR_MODIFY, dir, FSNOTIFY_EVENT_INODE, name, 0);
+ fsnotify(mask, child, FSNOTIFY_EVENT_INODE, dir, name, NULL, cookie);
}
static inline void fsnotify_dirent(struct inode *dir, struct dentry *dentry,
@@ -44,38 +39,55 @@
fsnotify_name(dir, mask, d_inode(dentry), &dentry->d_name, 0);
}
-/*
- * Simple wrappers to consolidate calls fsnotify_parent()/fsnotify() when
- * an event is on a file/dentry.
- */
-static inline void fsnotify_dentry(struct dentry *dentry, __u32 mask)
+static inline void fsnotify_inode(struct inode *inode, __u32 mask)
{
- struct inode *inode = d_inode(dentry);
-
if (S_ISDIR(inode->i_mode))
mask |= FS_ISDIR;
- fsnotify_parent(dentry, mask, inode, FSNOTIFY_EVENT_INODE);
- fsnotify(inode, mask, inode, FSNOTIFY_EVENT_INODE, NULL, 0);
+ fsnotify(mask, inode, FSNOTIFY_EVENT_INODE, NULL, NULL, inode, 0);
+}
+
+/* Notify this dentry's parent about a child's events. */
+static inline int fsnotify_parent(struct dentry *dentry, __u32 mask,
+ const void *data, int data_type)
+{
+ struct inode *inode = d_inode(dentry);
+
+ if (S_ISDIR(inode->i_mode)) {
+ mask |= FS_ISDIR;
+
+ /* sb/mount marks are not interested in name of directory */
+ if (!(dentry->d_flags & DCACHE_FSNOTIFY_PARENT_WATCHED))
+ goto notify_child;
+ }
+
+ /* disconnected dentry cannot notify parent */
+ if (IS_ROOT(dentry))
+ goto notify_child;
+
+ return __fsnotify_parent(dentry, mask, data, data_type);
+
+notify_child:
+ return fsnotify(mask, data, data_type, NULL, NULL, inode, 0);
+}
+
+/*
+ * Simple wrappers to consolidate calls to fsnotify_parent() when an event
+ * is on a file/dentry.
+ */
+static inline void fsnotify_dentry(struct dentry *dentry, __u32 mask)
+{
+ fsnotify_parent(dentry, mask, d_inode(dentry), FSNOTIFY_EVENT_INODE);
}
static inline int fsnotify_file(struct file *file, __u32 mask)
{
const struct path *path = &file->f_path;
- struct inode *inode = file_inode(file);
- int ret;
if (file->f_mode & FMODE_NONOTIFY)
return 0;
- if (S_ISDIR(inode->i_mode))
- mask |= FS_ISDIR;
-
- ret = fsnotify_parent(path->dentry, mask, path, FSNOTIFY_EVENT_PATH);
- if (ret)
- return ret;
-
- return fsnotify(inode, mask, path, FSNOTIFY_EVENT_PATH, NULL, 0);
+ return fsnotify_parent(path->dentry, mask, path, FSNOTIFY_EVENT_PATH);
}
/* Simple call site for access decisions */
@@ -108,12 +120,7 @@
*/
static inline void fsnotify_link_count(struct inode *inode)
{
- __u32 mask = FS_ATTRIB;
-
- if (S_ISDIR(inode->i_mode))
- mask |= FS_ISDIR;
-
- fsnotify(inode, mask, inode, FSNOTIFY_EVENT_INODE, NULL, 0);
+ fsnotify_inode(inode, FS_ATTRIB);
}
/*
@@ -128,7 +135,6 @@
u32 fs_cookie = fsnotify_get_cookie();
__u32 old_dir_mask = FS_MOVED_FROM;
__u32 new_dir_mask = FS_MOVED_TO;
- __u32 mask = FS_MOVE_SELF;
const struct qstr *new_name = &moved->d_name;
if (old_dir == new_dir)
@@ -137,7 +143,6 @@
if (isdir) {
old_dir_mask |= FS_ISDIR;
new_dir_mask |= FS_ISDIR;
- mask |= FS_ISDIR;
}
fsnotify_name(old_dir, old_dir_mask, source, old_name, fs_cookie);
@@ -145,9 +150,7 @@
if (target)
fsnotify_link_count(target);
-
- if (source)
- fsnotify(source, mask, source, FSNOTIFY_EVENT_INODE, NULL, 0);
+ fsnotify_inode(source, FS_MOVE_SELF);
audit_inode_child(new_dir, moved, AUDIT_TYPE_CHILD_CREATE);
}
@@ -172,12 +175,7 @@
*/
static inline void fsnotify_inoderemove(struct inode *inode)
{
- __u32 mask = FS_DELETE_SELF;
-
- if (S_ISDIR(inode->i_mode))
- mask |= FS_ISDIR;
-
- fsnotify(inode, mask, inode, FSNOTIFY_EVENT_INODE, NULL, 0);
+ fsnotify_inode(inode, FS_DELETE_SELF);
__fsnotify_inode_delete(inode);
}
diff --git a/include/linux/fsnotify_backend.h b/include/linux/fsnotify_backend.h
index f0c5064..f8529a3 100644
--- a/include/linux/fsnotify_backend.h
+++ b/include/linux/fsnotify_backend.h
@@ -47,11 +47,13 @@
#define FS_OPEN_PERM 0x00010000 /* open event in an permission hook */
#define FS_ACCESS_PERM 0x00020000 /* access event in a permissions hook */
#define FS_OPEN_EXEC_PERM 0x00040000 /* open/exec event in a permission hook */
-#define FS_DIR_MODIFY 0x00080000 /* Directory entry was modified */
#define FS_EXCL_UNLINK 0x04000000 /* do not send events if object is unlinked */
-/* This inode cares about things that happen to its children. Always set for
- * dnotify and inotify. */
+/*
+ * Set on inode mark that cares about things that happen to its children.
+ * Always set for dnotify and inotify.
+ * Set on inode/sb/mount marks that care about parent/name info.
+ */
#define FS_EVENT_ON_CHILD 0x08000000
#define FS_DN_RENAME 0x10000000 /* file renamed */
@@ -67,21 +69,28 @@
* The watching parent may get an FS_ATTRIB|FS_EVENT_ON_CHILD event
* when a directory entry inside a child subdir changes.
*/
-#define ALL_FSNOTIFY_DIRENT_EVENTS (FS_CREATE | FS_DELETE | FS_MOVE | \
- FS_DIR_MODIFY)
+#define ALL_FSNOTIFY_DIRENT_EVENTS (FS_CREATE | FS_DELETE | FS_MOVE)
#define ALL_FSNOTIFY_PERM_EVENTS (FS_OPEN_PERM | FS_ACCESS_PERM | \
FS_OPEN_EXEC_PERM)
/*
- * This is a list of all events that may get sent to a parent based on fs event
- * happening to inodes inside that directory.
+ * This is a list of all events that may get sent to a parent that is watching
+ * with flag FS_EVENT_ON_CHILD based on fs event on a child of that directory.
*/
#define FS_EVENTS_POSS_ON_CHILD (ALL_FSNOTIFY_PERM_EVENTS | \
FS_ACCESS | FS_MODIFY | FS_ATTRIB | \
FS_CLOSE_WRITE | FS_CLOSE_NOWRITE | \
FS_OPEN | FS_OPEN_EXEC)
+/*
+ * This is a list of all events that may get sent with the parent inode as the
+ * @to_tell argument of fsnotify().
+ * It may include events that can be sent to an inode/sb/mount mark, but cannot
+ * be sent to a parent watching children.
+ */
+#define FS_EVENTS_POSS_TO_PARENT (FS_EVENTS_POSS_ON_CHILD)
+
/* Events that can be reported to backends */
#define ALL_FSNOTIFY_EVENTS (ALL_FSNOTIFY_DIRENT_EVENTS | \
FS_EVENTS_POSS_ON_CHILD | \
@@ -108,18 +117,41 @@
* these operations for each relevant group.
*
* handle_event - main call for a group to handle an fs event
+ * @group: group to notify
+ * @mask: event type and flags
+ * @data: object that event happened on
+ * @data_type: type of object for fanotify_data_XXX() accessors
+ * @dir: optional directory associated with event -
+ * if @file_name is not NULL, this is the directory that
+ * @file_name is relative to
+ * @file_name: optional file name associated with event
+ * @cookie: inotify rename cookie
+ * @iter_info: array of marks from this group that are interested in the event
+ *
+ * handle_inode_event - simple variant of handle_event() for groups that only
+ * have inode marks and don't have ignore mask
+ * @mark: mark to notify
+ * @mask: event type and flags
+ * @inode: inode that event happened on
+ * @dir: optional directory associated with event -
+ * if @file_name is not NULL, this is the directory that
+ * @file_name is relative to.
+ * @file_name: optional file name associated with event
+ *
* free_group_priv - called when a group refcnt hits 0 to clean up the private union
* freeing_mark - called when a mark is being destroyed for some reason. The group
- * MUST be holding a reference on each mark and that reference must be
- * dropped in this function. inotify uses this function to send
- * userspace messages that marks have been removed.
+ * MUST be holding a reference on each mark and that reference must be
+ * dropped in this function. inotify uses this function to send
+ * userspace messages that marks have been removed.
*/
struct fsnotify_ops {
- int (*handle_event)(struct fsnotify_group *group,
- struct inode *inode,
- u32 mask, const void *data, int data_type,
+ int (*handle_event)(struct fsnotify_group *group, u32 mask,
+ const void *data, int data_type, struct inode *dir,
const struct qstr *file_name, u32 cookie,
struct fsnotify_iter_info *iter_info);
+ int (*handle_inode_event)(struct fsnotify_mark *mark, u32 mask,
+ struct inode *inode, struct inode *dir,
+ const struct qstr *file_name);
void (*free_group_priv)(struct fsnotify_group *group);
void (*freeing_mark)(struct fsnotify_mark *mark, struct fsnotify_group *group);
void (*free_event)(struct fsnotify_event *event);
@@ -220,12 +252,11 @@
FSNOTIFY_EVENT_INODE,
};
-static inline const struct inode *fsnotify_data_inode(const void *data,
- int data_type)
+static inline struct inode *fsnotify_data_inode(const void *data, int data_type)
{
switch (data_type) {
case FSNOTIFY_EVENT_INODE:
- return data;
+ return (struct inode *)data;
case FSNOTIFY_EVENT_PATH:
return d_inode(((const struct path *)data)->dentry);
default:
@@ -246,6 +277,7 @@
enum fsnotify_obj_type {
FSNOTIFY_OBJ_TYPE_INODE,
+ FSNOTIFY_OBJ_TYPE_CHILD,
FSNOTIFY_OBJ_TYPE_VFSMOUNT,
FSNOTIFY_OBJ_TYPE_SB,
FSNOTIFY_OBJ_TYPE_COUNT,
@@ -253,6 +285,7 @@
};
#define FSNOTIFY_OBJ_TYPE_INODE_FL (1U << FSNOTIFY_OBJ_TYPE_INODE)
+#define FSNOTIFY_OBJ_TYPE_CHILD_FL (1U << FSNOTIFY_OBJ_TYPE_CHILD)
#define FSNOTIFY_OBJ_TYPE_VFSMOUNT_FL (1U << FSNOTIFY_OBJ_TYPE_VFSMOUNT)
#define FSNOTIFY_OBJ_TYPE_SB_FL (1U << FSNOTIFY_OBJ_TYPE_SB)
#define FSNOTIFY_OBJ_ALL_TYPES_MASK ((1U << FSNOTIFY_OBJ_TYPE_COUNT) - 1)
@@ -297,6 +330,7 @@
}
FSNOTIFY_ITER_FUNCS(inode, INODE)
+FSNOTIFY_ITER_FUNCS(child, CHILD)
FSNOTIFY_ITER_FUNCS(vfsmount, VFSMOUNT)
FSNOTIFY_ITER_FUNCS(sb, SB)
@@ -377,15 +411,29 @@
/* called from the vfs helpers */
/* main fsnotify call to send events */
-extern int fsnotify(struct inode *to_tell, __u32 mask, const void *data,
- int data_type, const struct qstr *name, u32 cookie);
-extern int fsnotify_parent(struct dentry *dentry, __u32 mask, const void *data,
+extern int fsnotify(__u32 mask, const void *data, int data_type,
+ struct inode *dir, const struct qstr *name,
+ struct inode *inode, u32 cookie);
+extern int __fsnotify_parent(struct dentry *dentry, __u32 mask, const void *data,
int data_type);
extern void __fsnotify_inode_delete(struct inode *inode);
extern void __fsnotify_vfsmount_delete(struct vfsmount *mnt);
extern void fsnotify_sb_delete(struct super_block *sb);
extern u32 fsnotify_get_cookie(void);
+static inline __u32 fsnotify_parent_needed_mask(__u32 mask)
+{
+ /* FS_EVENT_ON_CHILD is set on marks that want parent/name info */
+ if (!(mask & FS_EVENT_ON_CHILD))
+ return 0;
+ /*
+ * This object might be watched by a mark that cares about parent/name
+ * info, does it care about the specific set of events that can be
+ * reported with parent/name info?
+ */
+ return mask & FS_EVENTS_POSS_TO_PARENT;
+}
+
static inline int fsnotify_inode_watches_children(struct inode *inode)
{
/* FS_EVENT_ON_CHILD is set if the inode may care */
@@ -535,13 +583,14 @@
#else
-static inline int fsnotify(struct inode *to_tell, __u32 mask, const void *data,
- int data_type, const struct qstr *name, u32 cookie)
+static inline int fsnotify(__u32 mask, const void *data, int data_type,
+ struct inode *dir, const struct qstr *name,
+ struct inode *inode, u32 cookie)
{
return 0;
}
-static inline int fsnotify_parent(struct dentry *dentry, __u32 mask,
+static inline int __fsnotify_parent(struct dentry *dentry, __u32 mask,
const void *data, int data_type)
{
return 0;
diff --git a/include/linux/init_syscalls.h b/include/linux/init_syscalls.h
new file mode 100644
index 0000000..92045d1
--- /dev/null
+++ b/include/linux/init_syscalls.h
@@ -0,0 +1,19 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+
+int __init init_mount(const char *dev_name, const char *dir_name,
+ const char *type_page, unsigned long flags, void *data_page);
+int __init init_umount(const char *name, int flags);
+int __init init_chdir(const char *filename);
+int __init init_chroot(const char *filename);
+int __init init_chown(const char *filename, uid_t user, gid_t group, int flags);
+int __init init_chmod(const char *filename, umode_t mode);
+int __init init_eaccess(const char *filename);
+int __init init_stat(const char *filename, struct kstat *stat, int flags);
+int __init init_mknod(const char *filename, umode_t mode, unsigned int dev);
+int __init init_link(const char *oldname, const char *newname);
+int __init init_symlink(const char *oldname, const char *newname);
+int __init init_unlink(const char *pathname);
+int __init init_mkdir(const char *pathname, umode_t mode);
+int __init init_rmdir(const char *pathname);
+int __init init_utimes(char *filename, struct timespec64 *ts);
+int __init init_dup(struct file *file);
diff --git a/include/linux/initrd.h b/include/linux/initrd.h
index aa59143..8db6f8c 100644
--- a/include/linux/initrd.h
+++ b/include/linux/initrd.h
@@ -2,12 +2,6 @@
#define INITRD_MINOR 250 /* shouldn't collide with /dev/ram* too soon ... */
-/* 1 = load ramdisk, 0 = don't load */
-extern int rd_doload;
-
-/* 1 = prompt for ramdisk, 0 = don't prompt */
-extern int rd_prompt;
-
/* starting block # of image */
extern int rd_image_start;
diff --git a/include/linux/kexec.h b/include/linux/kexec.h
index ea67910..9e93bef 100644
--- a/include/linux/kexec.h
+++ b/include/linux/kexec.h
@@ -183,17 +183,24 @@
bool get_value);
void *kexec_purgatory_get_symbol_addr(struct kimage *image, const char *name);
-int __weak arch_kexec_kernel_image_probe(struct kimage *image, void *buf,
- unsigned long buf_len);
-void * __weak arch_kexec_kernel_image_load(struct kimage *image);
-int __weak arch_kexec_apply_relocations_add(struct purgatory_info *pi,
- Elf_Shdr *section,
- const Elf_Shdr *relsec,
- const Elf_Shdr *symtab);
-int __weak arch_kexec_apply_relocations(struct purgatory_info *pi,
- Elf_Shdr *section,
- const Elf_Shdr *relsec,
- const Elf_Shdr *symtab);
+/* Architectures may override the below functions */
+int arch_kexec_kernel_image_probe(struct kimage *image, void *buf,
+ unsigned long buf_len);
+void *arch_kexec_kernel_image_load(struct kimage *image);
+int arch_kexec_apply_relocations_add(struct purgatory_info *pi,
+ Elf_Shdr *section,
+ const Elf_Shdr *relsec,
+ const Elf_Shdr *symtab);
+int arch_kexec_apply_relocations(struct purgatory_info *pi,
+ Elf_Shdr *section,
+ const Elf_Shdr *relsec,
+ const Elf_Shdr *symtab);
+int arch_kimage_file_post_load_cleanup(struct kimage *image);
+#ifdef CONFIG_KEXEC_SIG
+int arch_kexec_kernel_verify_sig(struct kimage *image, void *buf,
+ unsigned long buf_len);
+#endif
+int arch_kexec_locate_mem_hole(struct kexec_buf *kbuf);
extern int kexec_add_buffer(struct kexec_buf *kbuf);
int kexec_locate_mem_hole(struct kexec_buf *kbuf);
diff --git a/include/linux/mlx5/device.h b/include/linux/mlx5/device.h
index 57db125..4d3376e 100644
--- a/include/linux/mlx5/device.h
+++ b/include/linux/mlx5/device.h
@@ -276,7 +276,9 @@
MLX5_MKEY_MASK_RW = 1ull << 20,
MLX5_MKEY_MASK_A = 1ull << 21,
MLX5_MKEY_MASK_SMALL_FENCE = 1ull << 23,
- MLX5_MKEY_MASK_FREE = 1ull << 29,
+ MLX5_MKEY_MASK_RELAXED_ORDERING_WRITE = 1ull << 25,
+ MLX5_MKEY_MASK_FREE = 1ull << 29,
+ MLX5_MKEY_MASK_RELAXED_ORDERING_READ = 1ull << 47,
};
enum {
@@ -1007,7 +1009,6 @@
MLX5_MKEY_REMOTE_INVAL = 1 << 24,
MLX5_MKEY_FLAG_SYNC_UMR = 1 << 29,
MLX5_MKEY_BSF_EN = 1 << 30,
- MLX5_MKEY_LEN64 = 1 << 31,
};
struct mlx5_mkey_seg {
@@ -1361,11 +1362,11 @@
MLX5_ADDR_OF(device_event_cap, (mdev)->caps.hca_cur[MLX5_CAP_DEV_EVENT], cap)
#define MLX5_CAP_DEV_VDPA_EMULATION(mdev, cap)\
- MLX5_GET(device_virtio_emulation_cap, \
+ MLX5_GET(virtio_emulation_cap, \
(mdev)->caps.hca_cur[MLX5_CAP_VDPA_EMULATION], cap)
#define MLX5_CAP64_DEV_VDPA_EMULATION(mdev, cap)\
- MLX5_GET64(device_virtio_emulation_cap, \
+ MLX5_GET64(virtio_emulation_cap, \
(mdev)->caps.hca_cur[MLX5_CAP_VDPA_EMULATION], cap)
#define MLX5_CAP_IPSEC(mdev, cap)\
diff --git a/include/linux/mlx5/driver.h b/include/linux/mlx5/driver.h
index a0fcc4d..c145de0 100644
--- a/include/linux/mlx5/driver.h
+++ b/include/linux/mlx5/driver.h
@@ -975,6 +975,7 @@
void mlx5_unregister_debugfs(void);
void mlx5_fill_page_array(struct mlx5_frag_buf *buf, __be64 *pas);
+void mlx5_fill_page_frag_array_perm(struct mlx5_frag_buf *buf, __be64 *pas, u8 perm);
void mlx5_fill_page_frag_array(struct mlx5_frag_buf *frag_buf, __be64 *pas);
int mlx5_vector2eqn(struct mlx5_core_dev *dev, int vector, int *eqn,
unsigned int *irqn);
@@ -1057,6 +1058,7 @@
enum {
MLX5_INTERFACE_PROTOCOL_IB = 0,
MLX5_INTERFACE_PROTOCOL_ETH = 1,
+ MLX5_INTERFACE_PROTOCOL_VDPA = 2,
};
struct mlx5_interface {
diff --git a/include/linux/mlx5/mlx5_ifc.h b/include/linux/mlx5/mlx5_ifc.h
index 35e4dc6..de1ffb4 100644
--- a/include/linux/mlx5/mlx5_ifc.h
+++ b/include/linux/mlx5/mlx5_ifc.h
@@ -93,6 +93,7 @@
enum {
MLX5_OBJ_TYPE_GENEVE_TLV_OPT = 0x000b,
+ MLX5_OBJ_TYPE_VIRTIO_NET_Q = 0x000d,
MLX5_OBJ_TYPE_MKEY = 0xff01,
MLX5_OBJ_TYPE_QP = 0xff02,
MLX5_OBJ_TYPE_PSV = 0xff03,
@@ -985,17 +986,40 @@
u8 user_unaffiliated_events[4][0x40];
};
-struct mlx5_ifc_device_virtio_emulation_cap_bits {
- u8 reserved_at_0[0x20];
+struct mlx5_ifc_virtio_emulation_cap_bits {
+ u8 desc_tunnel_offload_type[0x1];
+ u8 eth_frame_offload_type[0x1];
+ u8 virtio_version_1_0[0x1];
+ u8 device_features_bits_mask[0xd];
+ u8 event_mode[0x8];
+ u8 virtio_queue_type[0x8];
- u8 reserved_at_20[0x13];
+ u8 max_tunnel_desc[0x10];
+ u8 reserved_at_30[0x3];
u8 log_doorbell_stride[0x5];
u8 reserved_at_38[0x3];
u8 log_doorbell_bar_size[0x5];
u8 doorbell_bar_offset[0x40];
- u8 reserved_at_80[0x780];
+ u8 max_emulated_devices[0x8];
+ u8 max_num_virtio_queues[0x18];
+
+ u8 reserved_at_a0[0x60];
+
+ u8 umem_1_buffer_param_a[0x20];
+
+ u8 umem_1_buffer_param_b[0x20];
+
+ u8 umem_2_buffer_param_a[0x20];
+
+ u8 umem_2_buffer_param_b[0x20];
+
+ u8 umem_3_buffer_param_a[0x20];
+
+ u8 umem_3_buffer_param_b[0x20];
+
+ u8 reserved_at_1c0[0x640];
};
enum {
@@ -1220,7 +1244,11 @@
u8 max_sgl_for_optimized_performance[0x8];
u8 log_max_cq_sz[0x8];
- u8 reserved_at_d0[0xb];
+ u8 relaxed_ordering_write_umr[0x1];
+ u8 relaxed_ordering_read_umr[0x1];
+ u8 reserved_at_d2[0x7];
+ u8 virtio_net_device_emualtion_manager[0x1];
+ u8 virtio_blk_device_emualtion_manager[0x1];
u8 log_max_cq[0x5];
u8 log_max_eq_sz[0x8];
@@ -1396,7 +1424,10 @@
u8 bf[0x1];
u8 driver_version[0x1];
u8 pad_tx_eth_packet[0x1];
- u8 reserved_at_263[0x8];
+ u8 reserved_at_263[0x3];
+ u8 mkey_by_name[0x1];
+ u8 reserved_at_267[0x4];
+
u8 log_bf_reg_size[0x5];
u8 reserved_at_270[0x8];
@@ -2953,7 +2984,7 @@
struct mlx5_ifc_fpga_cap_bits fpga_cap;
struct mlx5_ifc_tls_cap_bits tls_cap;
struct mlx5_ifc_device_mem_cap_bits device_mem_cap;
- struct mlx5_ifc_device_virtio_emulation_cap_bits virtio_emulation_cap;
+ struct mlx5_ifc_virtio_emulation_cap_bits virtio_emulation_cap;
u8 reserved_at_0[0x8000];
};
@@ -3302,15 +3333,18 @@
};
struct mlx5_ifc_rqtc_bits {
- u8 reserved_at_0[0xa0];
+ u8 reserved_at_0[0xa0];
- u8 reserved_at_a0[0x10];
- u8 rqt_max_size[0x10];
+ u8 reserved_at_a0[0x5];
+ u8 list_q_type[0x3];
+ u8 reserved_at_a8[0x8];
+ u8 rqt_max_size[0x10];
- u8 reserved_at_c0[0x10];
- u8 rqt_actual_size[0x10];
+ u8 rq_vhca_id_format[0x1];
+ u8 reserved_at_c1[0xf];
+ u8 rqt_actual_size[0x10];
- u8 reserved_at_e0[0x6a0];
+ u8 reserved_at_e0[0x6a0];
struct mlx5_ifc_rq_num_bits rq_num[];
};
@@ -7092,7 +7126,7 @@
struct mlx5_ifc_destroy_mkey_in_bits {
u8 opcode[0x10];
- u8 reserved_at_10[0x10];
+ u8 uid[0x10];
u8 reserved_at_20[0x10];
u8 op_mod[0x10];
@@ -7723,8 +7757,10 @@
u8 reserved_at_20[0x10];
u8 op_mod[0x10];
- u8 reserved_at_40[0x40];
+ u8 reserved_at_40[0x8];
+ u8 input_qpn[0x18];
+ u8 reserved_at_60[0x20];
u8 opt_param_mask[0x20];
u8 ece[0x20];
@@ -7788,7 +7824,7 @@
struct mlx5_ifc_create_mkey_in_bits {
u8 opcode[0x10];
- u8 reserved_at_10[0x10];
+ u8 uid[0x10];
u8 reserved_at_20[0x10];
u8 op_mod[0x10];
@@ -10346,6 +10382,40 @@
struct mlx5_ifc_umem_bits umem;
};
+struct mlx5_ifc_create_umem_out_bits {
+ u8 status[0x8];
+ u8 reserved_at_8[0x18];
+
+ u8 syndrome[0x20];
+
+ u8 reserved_at_40[0x8];
+ u8 umem_id[0x18];
+
+ u8 reserved_at_60[0x20];
+};
+
+struct mlx5_ifc_destroy_umem_in_bits {
+ u8 opcode[0x10];
+ u8 uid[0x10];
+
+ u8 reserved_at_20[0x10];
+ u8 op_mod[0x10];
+
+ u8 reserved_at_40[0x8];
+ u8 umem_id[0x18];
+
+ u8 reserved_at_60[0x20];
+};
+
+struct mlx5_ifc_destroy_umem_out_bits {
+ u8 status[0x8];
+ u8 reserved_at_8[0x18];
+
+ u8 syndrome[0x20];
+
+ u8 reserved_at_40[0x40];
+};
+
struct mlx5_ifc_create_uctx_in_bits {
u8 opcode[0x10];
u8 reserved_at_10[0x10];
@@ -10358,6 +10428,18 @@
struct mlx5_ifc_uctx_bits uctx;
};
+struct mlx5_ifc_create_uctx_out_bits {
+ u8 status[0x8];
+ u8 reserved_at_8[0x18];
+
+ u8 syndrome[0x20];
+
+ u8 reserved_at_40[0x10];
+ u8 uid[0x10];
+
+ u8 reserved_at_60[0x20];
+};
+
struct mlx5_ifc_destroy_uctx_in_bits {
u8 opcode[0x10];
u8 reserved_at_10[0x10];
@@ -10371,6 +10453,15 @@
u8 reserved_at_60[0x20];
};
+struct mlx5_ifc_destroy_uctx_out_bits {
+ u8 status[0x8];
+ u8 reserved_at_8[0x18];
+
+ u8 syndrome[0x20];
+
+ u8 reserved_at_40[0x40];
+};
+
struct mlx5_ifc_create_sw_icm_in_bits {
struct mlx5_ifc_general_obj_in_cmd_hdr_bits hdr;
struct mlx5_ifc_sw_icm_bits sw_icm;
@@ -10687,4 +10778,10 @@
u8 hw_offset_record_number[0x18];
};
+enum {
+ MLX5_MTT_PERM_READ = 1 << 0,
+ MLX5_MTT_PERM_WRITE = 1 << 1,
+ MLX5_MTT_PERM_RW = MLX5_MTT_PERM_READ | MLX5_MTT_PERM_WRITE,
+};
+
#endif /* MLX5_IFC_H */
diff --git a/include/linux/mm.h b/include/linux/mm.h
index 392016c..f6a82f9 100644
--- a/include/linux/mm.h
+++ b/include/linux/mm.h
@@ -319,8 +319,6 @@
#if defined(CONFIG_X86)
# define VM_PAT VM_ARCH_1 /* PAT reserves whole VMA at once (x86) */
-#elif defined(CONFIG_PPC)
-# define VM_SAO VM_ARCH_1 /* Strong Access Ordering (powerpc) */
#elif defined(CONFIG_PARISC)
# define VM_GROWSUP VM_ARCH_1
#elif defined(CONFIG_IA64)
diff --git a/include/linux/pgtable.h b/include/linux/pgtable.h
index 56c1e8e..53e97da 100644
--- a/include/linux/pgtable.h
+++ b/include/linux/pgtable.h
@@ -647,40 +647,6 @@
#define flush_tlb_fix_spurious_fault(vma, address) flush_tlb_page(vma, address)
#endif
-#ifndef pgprot_nx
-#define pgprot_nx(prot) (prot)
-#endif
-
-#ifndef pgprot_noncached
-#define pgprot_noncached(prot) (prot)
-#endif
-
-#ifndef pgprot_writecombine
-#define pgprot_writecombine pgprot_noncached
-#endif
-
-#ifndef pgprot_writethrough
-#define pgprot_writethrough pgprot_noncached
-#endif
-
-#ifndef pgprot_device
-#define pgprot_device pgprot_noncached
-#endif
-
-#ifndef pgprot_modify
-#define pgprot_modify pgprot_modify
-static inline pgprot_t pgprot_modify(pgprot_t oldprot, pgprot_t newprot)
-{
- if (pgprot_val(oldprot) == pgprot_val(pgprot_noncached(oldprot)))
- newprot = pgprot_noncached(newprot);
- if (pgprot_val(oldprot) == pgprot_val(pgprot_writecombine(oldprot)))
- newprot = pgprot_writecombine(newprot);
- if (pgprot_val(oldprot) == pgprot_val(pgprot_device(oldprot)))
- newprot = pgprot_device(newprot);
- return newprot;
-}
-#endif
-
/*
* When walking page tables, get the address of the next boundary,
* or the end address of the range if that comes earlier. Although no
@@ -840,6 +806,43 @@
* No-op macros that just return the current protection value. Defined here
* because these macros can be used used even if CONFIG_MMU is not defined.
*/
+
+#ifndef pgprot_nx
+#define pgprot_nx(prot) (prot)
+#endif
+
+#ifndef pgprot_noncached
+#define pgprot_noncached(prot) (prot)
+#endif
+
+#ifndef pgprot_writecombine
+#define pgprot_writecombine pgprot_noncached
+#endif
+
+#ifndef pgprot_writethrough
+#define pgprot_writethrough pgprot_noncached
+#endif
+
+#ifndef pgprot_device
+#define pgprot_device pgprot_noncached
+#endif
+
+#ifdef CONFIG_MMU
+#ifndef pgprot_modify
+#define pgprot_modify pgprot_modify
+static inline pgprot_t pgprot_modify(pgprot_t oldprot, pgprot_t newprot)
+{
+ if (pgprot_val(oldprot) == pgprot_val(pgprot_noncached(oldprot)))
+ newprot = pgprot_noncached(newprot);
+ if (pgprot_val(oldprot) == pgprot_val(pgprot_writecombine(oldprot)))
+ newprot = pgprot_writecombine(newprot);
+ if (pgprot_val(oldprot) == pgprot_val(pgprot_device(oldprot)))
+ newprot = pgprot_device(newprot);
+ return newprot;
+}
+#endif
+#endif /* CONFIG_MMU */
+
#ifndef pgprot_encrypted
#define pgprot_encrypted(prot) (prot)
#endif
diff --git a/include/linux/qcom_scm.h b/include/linux/qcom_scm.h
index 3d6a246..2e1193a 100644
--- a/include/linux/qcom_scm.h
+++ b/include/linux/qcom_scm.h
@@ -44,6 +44,13 @@
QCOM_SCM_ICE_DEV_ID = 20,
};
+enum qcom_scm_ice_cipher {
+ QCOM_SCM_ICE_CIPHER_AES_128_XTS = 0,
+ QCOM_SCM_ICE_CIPHER_AES_128_CBC = 1,
+ QCOM_SCM_ICE_CIPHER_AES_256_XTS = 3,
+ QCOM_SCM_ICE_CIPHER_AES_256_CBC = 4,
+};
+
#define QCOM_SCM_VMID_HLOS 0x3
#define QCOM_SCM_VMID_MSS_MSA 0xF
#define QCOM_SCM_VMID_WLAN 0x18
@@ -88,6 +95,12 @@
extern int qcom_scm_ocmem_unlock(enum qcom_scm_ocmem_client id, u32 offset,
u32 size);
+extern bool qcom_scm_ice_available(void);
+extern int qcom_scm_ice_invalidate_key(u32 index);
+extern int qcom_scm_ice_set_key(u32 index, const u8 *key, u32 key_size,
+ enum qcom_scm_ice_cipher cipher,
+ u32 data_unit_size);
+
extern bool qcom_scm_hdcp_available(void);
extern int qcom_scm_hdcp_req(struct qcom_scm_hdcp_req *req, u32 req_cnt,
u32 *resp);
@@ -138,6 +151,12 @@
static inline int qcom_scm_ocmem_unlock(enum qcom_scm_ocmem_client id,
u32 offset, u32 size) { return -ENODEV; }
+static inline bool qcom_scm_ice_available(void) { return false; }
+static inline int qcom_scm_ice_invalidate_key(u32 index) { return -ENODEV; }
+static inline int qcom_scm_ice_set_key(u32 index, const u8 *key, u32 key_size,
+ enum qcom_scm_ice_cipher cipher,
+ u32 data_unit_size) { return -ENODEV; }
+
static inline bool qcom_scm_hdcp_available(void) { return false; }
static inline int qcom_scm_hdcp_req(struct qcom_scm_hdcp_req *req, u32 req_cnt,
u32 *resp) { return -ENODEV; }
diff --git a/include/linux/raid/detect.h b/include/linux/raid/detect.h
index 37dd3f4..1f029a7 100644
--- a/include/linux/raid/detect.h
+++ b/include/linux/raid/detect.h
@@ -1,3 +1,11 @@
/* SPDX-License-Identifier: GPL-2.0 */
void md_autodetect_dev(dev_t dev);
+
+#ifdef CONFIG_BLK_DEV_MD
+void md_run_setup(void);
+#else
+static inline void md_run_setup(void)
+{
+}
+#endif
diff --git a/include/linux/raid/md_u.h b/include/linux/raid/md_u.h
deleted file mode 100644
index 8dfec08..0000000
--- a/include/linux/raid/md_u.h
+++ /dev/null
@@ -1,13 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0-or-later */
-/*
- md_u.h : user <=> kernel API between Linux raidtools and RAID drivers
- Copyright (C) 1998 Ingo Molnar
-
-*/
-#ifndef _MD_U_H
-#define _MD_U_H
-
-#include <uapi/linux/raid/md_u.h>
-
-extern int mdp_major;
-#endif
diff --git a/include/linux/regset.h b/include/linux/regset.h
index 46d6ae6..c3403f3 100644
--- a/include/linux/regset.h
+++ b/include/linux/regset.h
@@ -17,6 +17,52 @@
struct task_struct;
struct user_regset;
+struct membuf {
+ void *p;
+ size_t left;
+};
+
+static inline int membuf_zero(struct membuf *s, size_t size)
+{
+ if (s->left) {
+ if (size > s->left)
+ size = s->left;
+ memset(s->p, 0, size);
+ s->p += size;
+ s->left -= size;
+ }
+ return s->left;
+}
+
+static inline int membuf_write(struct membuf *s, const void *v, size_t size)
+{
+ if (s->left) {
+ if (size > s->left)
+ size = s->left;
+ memcpy(s->p, v, size);
+ s->p += size;
+ s->left -= size;
+ }
+ return s->left;
+}
+
+/* current s->p must be aligned for v; v must be a scalar */
+#define membuf_store(s, v) \
+({ \
+ struct membuf *__s = (s); \
+ if (__s->left) { \
+ typeof(v) __v = (v); \
+ size_t __size = sizeof(__v); \
+ if (unlikely(__size > __s->left)) { \
+ __size = __s->left; \
+ memcpy(__s->p, &__v, __size); \
+ } else { \
+ *(typeof(__v + 0) *)__s->p = __v; \
+ } \
+ __s->p += __size; \
+ __s->left -= __size; \
+ } \
+ __s->left;})
/**
* user_regset_active_fn - type of @active function in &struct user_regset
@@ -36,26 +82,9 @@
typedef int user_regset_active_fn(struct task_struct *target,
const struct user_regset *regset);
-/**
- * user_regset_get_fn - type of @get function in &struct user_regset
- * @target: thread being examined
- * @regset: regset being examined
- * @pos: offset into the regset data to access, in bytes
- * @count: amount of data to copy, in bytes
- * @kbuf: if not %NULL, a kernel-space pointer to copy into
- * @ubuf: if @kbuf is %NULL, a user-space pointer to copy into
- *
- * Fetch register values. Return %0 on success; -%EIO or -%ENODEV
- * are usual failure returns. The @pos and @count values are in
- * bytes, but must be properly aligned. If @kbuf is non-null, that
- * buffer is used and @ubuf is ignored. If @kbuf is %NULL, then
- * ubuf gives a userland pointer to access directly, and an -%EFAULT
- * return value is possible.
- */
-typedef int user_regset_get_fn(struct task_struct *target,
+typedef int user_regset_get2_fn(struct task_struct *target,
const struct user_regset *regset,
- unsigned int pos, unsigned int count,
- void *kbuf, void __user *ubuf);
+ struct membuf to);
/**
* user_regset_set_fn - type of @set function in &struct user_regset
@@ -104,28 +133,6 @@
int immediate);
/**
- * user_regset_get_size_fn - type of @get_size function in &struct user_regset
- * @target: thread being examined
- * @regset: regset being examined
- *
- * This call is optional; usually the pointer is %NULL.
- *
- * When provided, this function must return the current size of regset
- * data, as observed by the @get function in &struct user_regset. The
- * value returned must be a multiple of @size. The returned size is
- * required to be valid only until the next time (if any) @regset is
- * modified for @target.
- *
- * This function is intended for dynamically sized regsets. A regset
- * that is statically sized does not need to implement it.
- *
- * This function should not be called directly: instead, callers should
- * call regset_size() to determine the current size of a regset.
- */
-typedef unsigned int user_regset_get_size_fn(struct task_struct *target,
- const struct user_regset *regset);
-
-/**
* struct user_regset - accessible thread CPU state
* @n: Number of slots (registers).
* @size: Size in bytes of a slot (register).
@@ -136,7 +143,6 @@
* @set: Function to store values.
* @active: Function to report if regset is active, or %NULL.
* @writeback: Function to write data back to user memory, or %NULL.
- * @get_size: Function to return the regset's size, or %NULL.
*
* This data structure describes a machine resource we call a register set.
* This is part of the state of an individual thread, not necessarily
@@ -144,12 +150,7 @@
* similar slots, given by @n. Each slot is @size bytes, and aligned to
* @align bytes (which is at least @size). For dynamically-sized
* regsets, @n must contain the maximum possible number of slots for the
- * regset, and @get_size must point to a function that returns the
- * current regset size.
- *
- * Callers that need to know only the current size of the regset and do
- * not care about its internal structure should call regset_size()
- * instead of inspecting @n or calling @get_size.
+ * regset.
*
* For backward compatibility, the @get and @set methods must pad to, or
* accept, @n * @size bytes, even if the current regset size is smaller.
@@ -185,11 +186,10 @@
* omitted when there is an @active function and it returns zero.
*/
struct user_regset {
- user_regset_get_fn *get;
+ user_regset_get2_fn *regset_get;
user_regset_set_fn *set;
user_regset_active_fn *active;
user_regset_writeback_fn *writeback;
- user_regset_get_size_fn *get_size;
unsigned int n;
unsigned int size;
unsigned int align;
@@ -238,44 +238,6 @@
*/
const struct user_regset_view *task_user_regset_view(struct task_struct *tsk);
-
-/*
- * These are helpers for writing regset get/set functions in arch code.
- * Because @start_pos and @end_pos are always compile-time constants,
- * these are inlined into very little code though they look large.
- *
- * Use one or more calls sequentially for each chunk of regset data stored
- * contiguously in memory. Call with constants for @start_pos and @end_pos,
- * giving the range of byte positions in the regset that data corresponds
- * to; @end_pos can be -1 if this chunk is at the end of the regset layout.
- * Each call updates the arguments to point past its chunk.
- */
-
-static inline int user_regset_copyout(unsigned int *pos, unsigned int *count,
- void **kbuf,
- void __user **ubuf, const void *data,
- const int start_pos, const int end_pos)
-{
- if (*count == 0)
- return 0;
- BUG_ON(*pos < start_pos);
- if (end_pos < 0 || *pos < end_pos) {
- unsigned int copy = (end_pos < 0 ? *count
- : min(*count, end_pos - *pos));
- data += *pos - start_pos;
- if (*kbuf) {
- memcpy(*kbuf, data, copy);
- *kbuf += copy;
- } else if (__copy_to_user(*ubuf, data, copy))
- return -EFAULT;
- else
- *ubuf += copy;
- *pos += copy;
- *count -= copy;
- }
- return 0;
-}
-
static inline int user_regset_copyin(unsigned int *pos, unsigned int *count,
const void **kbuf,
const void __user **ubuf, void *data,
@@ -301,35 +263,6 @@
return 0;
}
-/*
- * These two parallel the two above, but for portions of a regset layout
- * that always read as all-zero or for which writes are ignored.
- */
-static inline int user_regset_copyout_zero(unsigned int *pos,
- unsigned int *count,
- void **kbuf, void __user **ubuf,
- const int start_pos,
- const int end_pos)
-{
- if (*count == 0)
- return 0;
- BUG_ON(*pos < start_pos);
- if (end_pos < 0 || *pos < end_pos) {
- unsigned int copy = (end_pos < 0 ? *count
- : min(*count, end_pos - *pos));
- if (*kbuf) {
- memset(*kbuf, 0, copy);
- *kbuf += copy;
- } else if (clear_user(*ubuf, copy))
- return -EFAULT;
- else
- *ubuf += copy;
- *pos += copy;
- *count -= copy;
- }
- return 0;
-}
-
static inline int user_regset_copyin_ignore(unsigned int *pos,
unsigned int *count,
const void **kbuf,
@@ -353,31 +286,19 @@
return 0;
}
-/**
- * copy_regset_to_user - fetch a thread's user_regset data into user memory
- * @target: thread to be examined
- * @view: &struct user_regset_view describing user thread machine state
- * @setno: index in @view->regsets
- * @offset: offset into the regset data, in bytes
- * @size: amount of data to copy, in bytes
- * @data: user-mode pointer to copy into
- */
-static inline int copy_regset_to_user(struct task_struct *target,
- const struct user_regset_view *view,
- unsigned int setno,
- unsigned int offset, unsigned int size,
- void __user *data)
-{
- const struct user_regset *regset = &view->regsets[setno];
+extern int regset_get(struct task_struct *target,
+ const struct user_regset *regset,
+ unsigned int size, void *data);
- if (!regset->get)
- return -EOPNOTSUPP;
+extern int regset_get_alloc(struct task_struct *target,
+ const struct user_regset *regset,
+ unsigned int size,
+ void **data);
- if (!access_ok(data, size))
- return -EFAULT;
-
- return regset->get(target, regset, offset, size, NULL, data);
-}
+extern int copy_regset_to_user(struct task_struct *target,
+ const struct user_regset_view *view,
+ unsigned int setno, unsigned int offset,
+ unsigned int size, void __user *data);
/**
* copy_regset_from_user - store into thread's user_regset data from user memory
@@ -405,21 +326,4 @@
return regset->set(target, regset, offset, size, NULL, data);
}
-/**
- * regset_size - determine the current size of a regset
- * @target: thread to be examined
- * @regset: regset to be examined
- *
- * Note that the returned size is valid only until the next time
- * (if any) @regset is modified for @target.
- */
-static inline unsigned int regset_size(struct task_struct *target,
- const struct user_regset *regset)
-{
- if (!regset->get_size)
- return regset->n * regset->size;
- else
- return regset->get_size(target, regset);
-}
-
#endif /* <linux/regset.h> */
diff --git a/include/linux/syscalls.h b/include/linux/syscalls.h
index e07a83f..a2429d3 100644
--- a/include/linux/syscalls.h
+++ b/include/linux/syscalls.h
@@ -1237,18 +1237,8 @@
* Instead, use one of the functions which work equivalently, such as
* the ksys_xyzyyz() functions prototyped below.
*/
-
-int ksys_umount(char __user *name, int flags);
-int ksys_dup(unsigned int fildes);
-int ksys_chroot(const char __user *filename);
ssize_t ksys_write(unsigned int fd, const char __user *buf, size_t count);
-int ksys_chdir(const char __user *filename);
-int ksys_fchmod(unsigned int fd, umode_t mode);
int ksys_fchown(unsigned int fd, uid_t user, gid_t group);
-int ksys_getdents64(unsigned int fd, struct linux_dirent64 __user *dirent,
- unsigned int count);
-int ksys_ioctl(unsigned int fd, unsigned int cmd, unsigned long arg);
-off_t ksys_lseek(unsigned int fd, off_t offset, unsigned int whence);
ssize_t ksys_read(unsigned int fd, char __user *buf, size_t count);
void ksys_sync(void);
int ksys_unshare(unsigned long unshare_flags);
@@ -1282,68 +1272,6 @@
* The following kernel syscall equivalents are just wrappers to fs-internal
* functions. Therefore, provide stubs to be inlined at the callsites.
*/
-extern long do_unlinkat(int dfd, struct filename *name);
-
-static inline long ksys_unlink(const char __user *pathname)
-{
- return do_unlinkat(AT_FDCWD, getname(pathname));
-}
-
-extern long do_rmdir(int dfd, const char __user *pathname);
-
-static inline long ksys_rmdir(const char __user *pathname)
-{
- return do_rmdir(AT_FDCWD, pathname);
-}
-
-extern long do_mkdirat(int dfd, const char __user *pathname, umode_t mode);
-
-static inline long ksys_mkdir(const char __user *pathname, umode_t mode)
-{
- return do_mkdirat(AT_FDCWD, pathname, mode);
-}
-
-extern long do_symlinkat(const char __user *oldname, int newdfd,
- const char __user *newname);
-
-static inline long ksys_symlink(const char __user *oldname,
- const char __user *newname)
-{
- return do_symlinkat(oldname, AT_FDCWD, newname);
-}
-
-extern long do_mknodat(int dfd, const char __user *filename, umode_t mode,
- unsigned int dev);
-
-static inline long ksys_mknod(const char __user *filename, umode_t mode,
- unsigned int dev)
-{
- return do_mknodat(AT_FDCWD, filename, mode, dev);
-}
-
-extern int do_linkat(int olddfd, const char __user *oldname, int newdfd,
- const char __user *newname, int flags);
-
-static inline long ksys_link(const char __user *oldname,
- const char __user *newname)
-{
- return do_linkat(AT_FDCWD, oldname, AT_FDCWD, newname, 0);
-}
-
-extern int do_fchmodat(int dfd, const char __user *filename, umode_t mode);
-
-static inline int ksys_chmod(const char __user *filename, umode_t mode)
-{
- return do_fchmodat(AT_FDCWD, filename, mode);
-}
-
-long do_faccessat(int dfd, const char __user *filename, int mode, int flags);
-
-static inline long ksys_access(const char __user *filename, int mode)
-{
- return do_faccessat(AT_FDCWD, filename, mode, 0);
-}
-
extern int do_fchownat(int dfd, const char __user *filename, uid_t user,
gid_t group, int flag);
@@ -1379,17 +1307,6 @@
return __close_fd(current->files, fd);
}
-extern long do_sys_open(int dfd, const char __user *filename, int flags,
- umode_t mode);
-
-static inline long ksys_open(const char __user *filename, int flags,
- umode_t mode)
-{
- if (force_o_largefile())
- flags |= O_LARGEFILE;
- return do_sys_open(AT_FDCWD, filename, flags, mode);
-}
-
extern long do_sys_truncate(const char __user *pathname, loff_t length);
static inline long ksys_truncate(const char __user *pathname, loff_t length)
diff --git a/include/linux/thermal.h b/include/linux/thermal.h
index 216185b..42ef807 100644
--- a/include/linux/thermal.h
+++ b/include/linux/thermal.h
@@ -37,18 +37,6 @@
struct thermal_instance;
struct thermal_attr;
-enum thermal_device_mode {
- THERMAL_DEVICE_DISABLED = 0,
- THERMAL_DEVICE_ENABLED,
-};
-
-enum thermal_trip_type {
- THERMAL_TRIP_ACTIVE = 0,
- THERMAL_TRIP_PASSIVE,
- THERMAL_TRIP_HOT,
- THERMAL_TRIP_CRITICAL,
-};
-
enum thermal_trend {
THERMAL_TREND_STABLE, /* temperature is stable */
THERMAL_TREND_RAISING, /* temperature is raising */
@@ -76,9 +64,7 @@
struct thermal_cooling_device *);
int (*get_temp) (struct thermal_zone_device *, int *);
int (*set_trips) (struct thermal_zone_device *, int, int);
- int (*get_mode) (struct thermal_zone_device *,
- enum thermal_device_mode *);
- int (*set_mode) (struct thermal_zone_device *,
+ int (*change_mode) (struct thermal_zone_device *,
enum thermal_device_mode);
int (*get_trip_type) (struct thermal_zone_device *, int,
enum thermal_trip_type *);
@@ -128,6 +114,7 @@
* @trip_temp_attrs: attributes for trip points for sysfs: trip temperature
* @trip_type_attrs: attributes for trip points for sysfs: trip type
* @trip_hyst_attrs: attributes for trip points for sysfs: trip hysteresis
+ * @mode: current mode of this thermal zone
* @devdata: private pointer for device private data
* @trips: number of trip points the thermal zone supports
* @trips_disabled; bitmap for disabled trips
@@ -170,6 +157,7 @@
struct thermal_attr *trip_temp_attrs;
struct thermal_attr *trip_type_attrs;
struct thermal_attr *trip_hyst_attrs;
+ enum thermal_device_mode mode;
void *devdata;
int trips;
unsigned long trips_disabled; /* bitmap for disabled trips */
@@ -303,11 +291,6 @@
int offset;
};
-struct thermal_genl_event {
- u32 orig;
- enum events event;
-};
-
/**
* struct thermal_zone_of_device_ops - scallbacks for handling DT based zones
*
@@ -416,6 +399,8 @@
void thermal_cdev_update(struct thermal_cooling_device *);
void thermal_notify_framework(struct thermal_zone_device *, int);
+int thermal_zone_device_enable(struct thermal_zone_device *tz);
+int thermal_zone_device_disable(struct thermal_zone_device *tz);
#else
static inline struct thermal_zone_device *thermal_zone_device_register(
const char *type, int trips, int mask, void *devdata,
@@ -463,6 +448,12 @@
static inline void thermal_notify_framework(struct thermal_zone_device *tz,
int trip)
{ }
+
+static inline int thermal_zone_device_enable(struct thermal_zone_device *tz)
+{ return -ENODEV; }
+
+static inline int thermal_zone_device_disable(struct thermal_zone_device *tz)
+{ return -ENODEV; }
#endif /* CONFIG_THERMAL */
#endif /* __THERMAL_H__ */
diff --git a/include/misc/ocxl-config.h b/include/misc/ocxl-config.h
index 3526fa9..ccfd3b4 100644
--- a/include/misc/ocxl-config.h
+++ b/include/misc/ocxl-config.h
@@ -41,5 +41,6 @@
#define OCXL_DVSEC_VENDOR_CFG_VERS 0x0C
#define OCXL_DVSEC_VENDOR_TLX_VERS 0x10
#define OCXL_DVSEC_VENDOR_DLX_VERS 0x20
+#define OCXL_DVSEC_VENDOR_RESET_RELOAD 0x38
#endif /* _OCXL_CONFIG_H_ */
diff --git a/include/misc/ocxl.h b/include/misc/ocxl.h
index 06dd583..357ef1a 100644
--- a/include/misc/ocxl.h
+++ b/include/misc/ocxl.h
@@ -62,8 +62,7 @@
// Device detection & initialisation
/**
- * Open an OpenCAPI function on an OpenCAPI device
- *
+ * ocxl_function_open() - Open an OpenCAPI function on an OpenCAPI device
* @dev: The PCI device that contains the function
*
* Returns an opaque pointer to the function, or an error pointer (check with IS_ERR)
@@ -71,8 +70,7 @@
struct ocxl_fn *ocxl_function_open(struct pci_dev *dev);
/**
- * Get the list of AFUs associated with a PCI function device
- *
+ * ocxl_function_afu_list() - Get the list of AFUs associated with a PCI function device
* Returns a list of struct ocxl_afu *
*
* @fn: The OpenCAPI function containing the AFUs
@@ -80,8 +78,7 @@
struct list_head *ocxl_function_afu_list(struct ocxl_fn *fn);
/**
- * Fetch an AFU instance from an OpenCAPI function
- *
+ * ocxl_function_fetch_afu() - Fetch an AFU instance from an OpenCAPI function
* @fn: The OpenCAPI function to get the AFU from
* @afu_idx: The index of the AFU to get
*
@@ -92,23 +89,20 @@
struct ocxl_afu *ocxl_function_fetch_afu(struct ocxl_fn *fn, u8 afu_idx);
/**
- * Take a reference to an AFU
- *
+ * ocxl_afu_get() - Take a reference to an AFU
* @afu: The AFU to increment the reference count on
*/
void ocxl_afu_get(struct ocxl_afu *afu);
/**
- * Release a reference to an AFU
- *
+ * ocxl_afu_put() - Release a reference to an AFU
* @afu: The AFU to decrement the reference count on
*/
void ocxl_afu_put(struct ocxl_afu *afu);
/**
- * Get the configuration information for an OpenCAPI function
- *
+ * ocxl_function_config() - Get the configuration information for an OpenCAPI function
* @fn: The OpenCAPI function to get the config for
*
* Returns the function config, or NULL on error
@@ -116,8 +110,7 @@
const struct ocxl_fn_config *ocxl_function_config(struct ocxl_fn *fn);
/**
- * Close an OpenCAPI function
- *
+ * ocxl_function_close() - Close an OpenCAPI function
* This will free any AFUs previously retrieved from the function, and
* detach and associated contexts. The contexts must by freed by the caller.
*
@@ -129,8 +122,7 @@
// Context allocation
/**
- * Allocate an OpenCAPI context
- *
+ * ocxl_context_alloc() - Allocate an OpenCAPI context
* @context: The OpenCAPI context to allocate, must be freed with ocxl_context_free
* @afu: The AFU the context belongs to
* @mapping: The mapping to unmap when the context is closed (may be NULL)
@@ -139,14 +131,13 @@
struct address_space *mapping);
/**
- * Free an OpenCAPI context
- *
+ * ocxl_context_free() - Free an OpenCAPI context
* @ctx: The OpenCAPI context to free
*/
void ocxl_context_free(struct ocxl_context *ctx);
/**
- * Grant access to an MM to an OpenCAPI context
+ * ocxl_context_attach() - Grant access to an MM to an OpenCAPI context
* @ctx: The OpenCAPI context to attach
* @amr: The value of the AMR register to restrict access
* @mm: The mm to attach to the context
@@ -157,7 +148,7 @@
struct mm_struct *mm);
/**
- * Detach an MM from an OpenCAPI context
+ * ocxl_context_detach() - Detach an MM from an OpenCAPI context
* @ctx: The OpenCAPI context to attach
*
* Returns 0 on success, negative on failure
@@ -167,25 +158,25 @@
// AFU IRQs
/**
- * Allocate an IRQ associated with an AFU context
+ * ocxl_afu_irq_alloc() - Allocate an IRQ associated with an AFU context
* @ctx: the AFU context
* @irq_id: out, the IRQ ID
*
* Returns 0 on success, negative on failure
*/
-extern int ocxl_afu_irq_alloc(struct ocxl_context *ctx, int *irq_id);
+int ocxl_afu_irq_alloc(struct ocxl_context *ctx, int *irq_id);
/**
- * Frees an IRQ associated with an AFU context
+ * ocxl_afu_irq_free() - Frees an IRQ associated with an AFU context
* @ctx: the AFU context
* @irq_id: the IRQ ID
*
* Returns 0 on success, negative on failure
*/
-extern int ocxl_afu_irq_free(struct ocxl_context *ctx, int irq_id);
+int ocxl_afu_irq_free(struct ocxl_context *ctx, int irq_id);
/**
- * Gets the address of the trigger page for an IRQ
+ * ocxl_afu_irq_get_addr() - Gets the address of the trigger page for an IRQ
* This can then be provided to an AFU which will write to that
* page to trigger the IRQ.
* @ctx: The AFU context that the IRQ is associated with
@@ -193,10 +184,10 @@
*
* returns the trigger page address, or 0 if the IRQ is not valid
*/
-extern u64 ocxl_afu_irq_get_addr(struct ocxl_context *ctx, int irq_id);
+u64 ocxl_afu_irq_get_addr(struct ocxl_context *ctx, int irq_id);
/**
- * Provide a callback to be called when an IRQ is triggered
+ * ocxl_irq_set_handler() - Provide a callback to be called when an IRQ is triggered
* @ctx: The AFU context that the IRQ is associated with
* @irq_id: The IRQ ID
* @handler: the callback to be called when the IRQ is triggered
@@ -213,8 +204,7 @@
// AFU Metadata
/**
- * Get a pointer to the config for an AFU
- *
+ * ocxl_afu_config() - Get a pointer to the config for an AFU
* @afu: a pointer to the AFU to get the config for
*
* Returns a pointer to the AFU config
@@ -222,27 +212,24 @@
struct ocxl_afu_config *ocxl_afu_config(struct ocxl_afu *afu);
/**
- * Assign opaque hardware specific information to an OpenCAPI AFU.
- *
- * @dev: The PCI device associated with the OpenCAPI device
+ * ocxl_afu_set_private() - Assign opaque hardware specific information to an OpenCAPI AFU.
+ * @afu: The OpenCAPI AFU
* @private: the opaque hardware specific information to assign to the driver
*/
void ocxl_afu_set_private(struct ocxl_afu *afu, void *private);
/**
- * Fetch the hardware specific information associated with an external OpenCAPI
- * AFU. This may be consumed by an external OpenCAPI driver.
- *
- * @afu: The AFU
+ * ocxl_afu_get_private() - Fetch the hardware specific information associated with
+ * an external OpenCAPI AFU. This may be consumed by an external OpenCAPI driver.
+ * @afu: The OpenCAPI AFU
*
* Returns the opaque pointer associated with the device, or NULL if not set
*/
-void *ocxl_afu_get_private(struct ocxl_afu *dev);
+void *ocxl_afu_get_private(struct ocxl_afu *afu);
// Global MMIO
/**
- * Read a 32 bit value from global MMIO
- *
+ * ocxl_global_mmio_read32() - Read a 32 bit value from global MMIO
* @afu: The AFU
* @offset: The Offset from the start of MMIO
* @endian: the endianness that the MMIO data is in
@@ -251,11 +238,10 @@
* Returns 0 for success, negative on error
*/
int ocxl_global_mmio_read32(struct ocxl_afu *afu, size_t offset,
- enum ocxl_endian endian, u32 *val);
+ enum ocxl_endian endian, u32 *val);
/**
- * Read a 64 bit value from global MMIO
- *
+ * ocxl_global_mmio_read64() - Read a 64 bit value from global MMIO
* @afu: The AFU
* @offset: The Offset from the start of MMIO
* @endian: the endianness that the MMIO data is in
@@ -264,11 +250,10 @@
* Returns 0 for success, negative on error
*/
int ocxl_global_mmio_read64(struct ocxl_afu *afu, size_t offset,
- enum ocxl_endian endian, u64 *val);
+ enum ocxl_endian endian, u64 *val);
/**
- * Write a 32 bit value to global MMIO
- *
+ * ocxl_global_mmio_write32() - Write a 32 bit value to global MMIO
* @afu: The AFU
* @offset: The Offset from the start of MMIO
* @endian: the endianness that the MMIO data is in
@@ -277,11 +262,10 @@
* Returns 0 for success, negative on error
*/
int ocxl_global_mmio_write32(struct ocxl_afu *afu, size_t offset,
- enum ocxl_endian endian, u32 val);
+ enum ocxl_endian endian, u32 val);
/**
- * Write a 64 bit value to global MMIO
- *
+ * ocxl_global_mmio_write64() - Write a 64 bit value to global MMIO
* @afu: The AFU
* @offset: The Offset from the start of MMIO
* @endian: the endianness that the MMIO data is in
@@ -290,11 +274,10 @@
* Returns 0 for success, negative on error
*/
int ocxl_global_mmio_write64(struct ocxl_afu *afu, size_t offset,
- enum ocxl_endian endian, u64 val);
+ enum ocxl_endian endian, u64 val);
/**
- * Set bits in a 32 bit global MMIO register
- *
+ * ocxl_global_mmio_set32() - Set bits in a 32 bit global MMIO register
* @afu: The AFU
* @offset: The Offset from the start of MMIO
* @endian: the endianness that the MMIO data is in
@@ -303,11 +286,10 @@
* Returns 0 for success, negative on error
*/
int ocxl_global_mmio_set32(struct ocxl_afu *afu, size_t offset,
- enum ocxl_endian endian, u32 mask);
+ enum ocxl_endian endian, u32 mask);
/**
- * Set bits in a 64 bit global MMIO register
- *
+ * ocxl_global_mmio_set64() - Set bits in a 64 bit global MMIO register
* @afu: The AFU
* @offset: The Offset from the start of MMIO
* @endian: the endianness that the MMIO data is in
@@ -316,11 +298,10 @@
* Returns 0 for success, negative on error
*/
int ocxl_global_mmio_set64(struct ocxl_afu *afu, size_t offset,
- enum ocxl_endian endian, u64 mask);
+ enum ocxl_endian endian, u64 mask);
/**
- * Set bits in a 32 bit global MMIO register
- *
+ * ocxl_global_mmio_clear32() - Set bits in a 32 bit global MMIO register
* @afu: The AFU
* @offset: The Offset from the start of MMIO
* @endian: the endianness that the MMIO data is in
@@ -329,11 +310,10 @@
* Returns 0 for success, negative on error
*/
int ocxl_global_mmio_clear32(struct ocxl_afu *afu, size_t offset,
- enum ocxl_endian endian, u32 mask);
+ enum ocxl_endian endian, u32 mask);
/**
- * Set bits in a 64 bit global MMIO register
- *
+ * ocxl_global_mmio_clear64() - Set bits in a 64 bit global MMIO register
* @afu: The AFU
* @offset: The Offset from the start of MMIO
* @endian: the endianness that the MMIO data is in
@@ -342,7 +322,7 @@
* Returns 0 for success, negative on error
*/
int ocxl_global_mmio_clear64(struct ocxl_afu *afu, size_t offset,
- enum ocxl_endian endian, u64 mask);
+ enum ocxl_endian endian, u64 mask);
// Functions left here are for compatibility with the cxlflash driver
diff --git a/include/net/sock.h b/include/net/sock.h
index 497c64f..064637d 100644
--- a/include/net/sock.h
+++ b/include/net/sock.h
@@ -2696,6 +2696,7 @@
void sock_set_keepalive(struct sock *sk);
void sock_set_priority(struct sock *sk, u32 priority);
void sock_set_rcvbuf(struct sock *sk, int val);
+void sock_set_mark(struct sock *sk, u32 val);
void sock_set_reuseaddr(struct sock *sk);
void sock_set_reuseport(struct sock *sk);
void sock_set_sndtimeo(struct sock *sk, s64 secs);
diff --git a/include/rdma/ib.h b/include/rdma/ib.h
index fe2fc9e..83139b9 100644
--- a/include/rdma/ib.h
+++ b/include/rdma/ib.h
@@ -1,36 +1,9 @@
+/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */
/*
* Copyright (c) 2010 Intel Corporation. All rights reserved.
- *
- * This software is available to you under a choice of one of two
- * licenses. You may choose to be licensed under the terms of the GNU
- * General Public License (GPL) Version 2, available from the file
- * COPYING in the main directory of this source tree, or the
- * OpenIB.org BSD license below:
- *
- * Redistribution and use in source and binary forms, with or
- * without modification, are permitted provided that the following
- * conditions are met:
- *
- * - Redistributions of source code must retain the above
- * copyright notice, this list of conditions and the following
- * disclaimer.
- *
- * - Redistributions in binary form must reproduce the above
- * copyright notice, this list of conditions and the following
- * disclaimer in the documentation and/or other materials
- * provided with the distribution.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
- * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
- * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
- * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
- * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
- * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
- * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
*/
-#if !defined(_RDMA_IB_H)
+#ifndef _RDMA_IB_H
#define _RDMA_IB_H
#include <linux/types.h>
diff --git a/include/rdma/ib_addr.h b/include/rdma/ib_addr.h
index 2734c89..b0e636a 100644
--- a/include/rdma/ib_addr.h
+++ b/include/rdma/ib_addr.h
@@ -1,37 +1,10 @@
+/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */
/*
* Copyright (c) 2005 Voltaire Inc. All rights reserved.
* Copyright (c) 2005 Intel Corporation. All rights reserved.
- *
- * This software is available to you under a choice of one of two
- * licenses. You may choose to be licensed under the terms of the GNU
- * General Public License (GPL) Version 2, available from the file
- * COPYING in the main directory of this source tree, or the
- * OpenIB.org BSD license below:
- *
- * Redistribution and use in source and binary forms, with or
- * without modification, are permitted provided that the following
- * conditions are met:
- *
- * - Redistributions of source code must retain the above
- * copyright notice, this list of conditions and the following
- * disclaimer.
- *
- * - Redistributions in binary form must reproduce the above
- * copyright notice, this list of conditions and the following
- * disclaimer in the documentation and/or other materials
- * provided with the distribution.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
- * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
- * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
- * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
- * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
- * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
- * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
*/
-#if !defined(IB_ADDR_H)
+#ifndef IB_ADDR_H
#define IB_ADDR_H
#include <linux/in.h>
diff --git a/include/rdma/ib_cache.h b/include/rdma/ib_cache.h
index e06d133..66a8f36 100644
--- a/include/rdma/ib_cache.h
+++ b/include/rdma/ib_cache.h
@@ -1,35 +1,8 @@
+/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */
/*
* Copyright (c) 2004 Topspin Communications. All rights reserved.
* Copyright (c) 2005 Intel Corporation. All rights reserved.
* Copyright (c) 2005 Sun Microsystems, Inc. All rights reserved.
- *
- * This software is available to you under a choice of one of two
- * licenses. You may choose to be licensed under the terms of the GNU
- * General Public License (GPL) Version 2, available from the file
- * COPYING in the main directory of this source tree, or the
- * OpenIB.org BSD license below:
- *
- * Redistribution and use in source and binary forms, with or
- * without modification, are permitted provided that the following
- * conditions are met:
- *
- * - Redistributions of source code must retain the above
- * copyright notice, this list of conditions and the following
- * disclaimer.
- *
- * - Redistributions in binary form must reproduce the above
- * copyright notice, this list of conditions and the following
- * disclaimer in the documentation and/or other materials
- * provided with the distribution.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
- * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
- * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
- * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
- * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
- * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
- * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
*/
#ifndef _IB_CACHE_H
diff --git a/include/rdma/ib_cm.h b/include/rdma/ib_cm.h
index 0f1ea5f..382427a 100644
--- a/include/rdma/ib_cm.h
+++ b/include/rdma/ib_cm.h
@@ -6,6 +6,7 @@
* Copyright (c) 2005 Sun Microsystems, Inc. All rights reserved.
* Copyright (c) 2019, Mellanox Technologies inc. All rights reserved.
*/
+
#ifndef IB_CM_H
#define IB_CM_H
diff --git a/include/rdma/ib_hdrs.h b/include/rdma/ib_hdrs.h
index 9a90bd0..57c1ac8 100644
--- a/include/rdma/ib_hdrs.h
+++ b/include/rdma/ib_hdrs.h
@@ -1,48 +1,6 @@
+/* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */
/*
* Copyright(c) 2016 - 2018 Intel Corporation.
- *
- * This file is provided under a dual BSD/GPLv2 license. When using or
- * redistributing this file, you may do so under either license.
- *
- * GPL LICENSE SUMMARY
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of version 2 of the GNU General Public License as
- * published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but
- * WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- * General Public License for more details.
- *
- * BSD LICENSE
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- *
- * - Redistributions of source code must retain the above copyright
- * notice, this list of conditions and the following disclaimer.
- * - Redistributions in binary form must reproduce the above copyright
- * notice, this list of conditions and the following disclaimer in
- * the documentation and/or other materials provided with the
- * distribution.
- * - Neither the name of Intel Corporation nor the names of its
- * contributors may be used to endorse or promote products derived
- * from this software without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
- * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
- * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
- * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
- * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- *
*/
#ifndef IB_HDRS_H
diff --git a/include/rdma/ib_mad.h b/include/rdma/ib_mad.h
index 8c093fc..8dfb1dd 100644
--- a/include/rdma/ib_mad.h
+++ b/include/rdma/ib_mad.h
@@ -1,40 +1,13 @@
+/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */
/*
* Copyright (c) 2004 Mellanox Technologies Ltd. All rights reserved.
* Copyright (c) 2004 Infinicon Corporation. All rights reserved.
* Copyright (c) 2004 Intel Corporation. All rights reserved.
* Copyright (c) 2004 Topspin Corporation. All rights reserved.
* Copyright (c) 2004-2006 Voltaire Corporation. All rights reserved.
- *
- * This software is available to you under a choice of one of two
- * licenses. You may choose to be licensed under the terms of the GNU
- * General Public License (GPL) Version 2, available from the file
- * COPYING in the main directory of this source tree, or the
- * OpenIB.org BSD license below:
- *
- * Redistribution and use in source and binary forms, with or
- * without modification, are permitted provided that the following
- * conditions are met:
- *
- * - Redistributions of source code must retain the above
- * copyright notice, this list of conditions and the following
- * disclaimer.
- *
- * - Redistributions in binary form must reproduce the above
- * copyright notice, this list of conditions and the following
- * disclaimer in the documentation and/or other materials
- * provided with the distribution.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
- * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
- * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
- * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
- * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
- * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
- * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
*/
-#if !defined(IB_MAD_H)
+#ifndef IB_MAD_H
#define IB_MAD_H
#include <linux/list.h>
diff --git a/include/rdma/ib_marshall.h b/include/rdma/ib_marshall.h
index 8ebf84a..1838869 100644
--- a/include/rdma/ib_marshall.h
+++ b/include/rdma/ib_marshall.h
@@ -1,36 +1,9 @@
+/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */
/*
* Copyright (c) 2005-2006 Intel Corporation. All rights reserved.
- *
- * This software is available to you under a choice of one of two
- * licenses. You may choose to be licensed under the terms of the GNU
- * General Public License (GPL) Version 2, available from the file
- * COPYING in the main directory of this source tree, or the
- * OpenIB.org BSD license below:
- *
- * Redistribution and use in source and binary forms, with or
- * without modification, are permitted provided that the following
- * conditions are met:
- *
- * - Redistributions of source code must retain the above
- * copyright notice, this list of conditions and the following
- * disclaimer.
- *
- * - Redistributions in binary form must reproduce the above
- * copyright notice, this list of conditions and the following
- * disclaimer in the documentation and/or other materials
- * provided with the distribution.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
- * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
- * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
- * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
- * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
- * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
- * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
*/
-#if !defined(IB_USER_MARSHALL_H)
+#ifndef IB_USER_MARSHALL_H
#define IB_USER_MARSHALL_H
#include <rdma/ib_verbs.h>
diff --git a/include/rdma/ib_pack.h b/include/rdma/ib_pack.h
index 7ea1382..a9162f25 100644
--- a/include/rdma/ib_pack.h
+++ b/include/rdma/ib_pack.h
@@ -1,33 +1,6 @@
+/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */
/*
* Copyright (c) 2004 Topspin Corporation. All rights reserved.
- *
- * This software is available to you under a choice of one of two
- * licenses. You may choose to be licensed under the terms of the GNU
- * General Public License (GPL) Version 2, available from the file
- * COPYING in the main directory of this source tree, or the
- * OpenIB.org BSD license below:
- *
- * Redistribution and use in source and binary forms, with or
- * without modification, are permitted provided that the following
- * conditions are met:
- *
- * - Redistributions of source code must retain the above
- * copyright notice, this list of conditions and the following
- * disclaimer.
- *
- * - Redistributions in binary form must reproduce the above
- * copyright notice, this list of conditions and the following
- * disclaimer in the documentation and/or other materials
- * provided with the distribution.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
- * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
- * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
- * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
- * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
- * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
- * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
*/
#ifndef IB_PACK_H
diff --git a/include/rdma/ib_pma.h b/include/rdma/ib_pma.h
index 2f8a65c..44c6182 100644
--- a/include/rdma/ib_pma.h
+++ b/include/rdma/ib_pma.h
@@ -1,38 +1,11 @@
+/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */
/*
* Copyright (c) 2006, 2007, 2008, 2009, 2010 QLogic Corporation.
* All rights reserved.
* Copyright (c) 2005, 2006 PathScale, Inc. All rights reserved.
- *
- * This software is available to you under a choice of one of two
- * licenses. You may choose to be licensed under the terms of the GNU
- * General Public License (GPL) Version 2, available from the file
- * COPYING in the main directory of this source tree, or the
- * OpenIB.org BSD license below:
- *
- * Redistribution and use in source and binary forms, with or
- * without modification, are permitted provided that the following
- * conditions are met:
- *
- * - Redistributions of source code must retain the above
- * copyright notice, this list of conditions and the following
- * disclaimer.
- *
- * - Redistributions in binary form must reproduce the above
- * copyright notice, this list of conditions and the following
- * disclaimer in the documentation and/or other materials
- * provided with the distribution.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
- * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
- * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
- * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
- * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
- * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
- * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
*/
-#if !defined(IB_PMA_H)
+#ifndef IB_PMA_H
#define IB_PMA_H
#include <rdma/ib_mad.h>
diff --git a/include/rdma/ib_sa.h b/include/rdma/ib_sa.h
index 1952097..693285e 100644
--- a/include/rdma/ib_sa.h
+++ b/include/rdma/ib_sa.h
@@ -1,35 +1,8 @@
+/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */
/*
* Copyright (c) 2004 Topspin Communications. All rights reserved.
* Copyright (c) 2005 Voltaire, Inc. All rights reserved.
* Copyright (c) 2006 Intel Corporation. All rights reserved.
- *
- * This software is available to you under a choice of one of two
- * licenses. You may choose to be licensed under the terms of the GNU
- * General Public License (GPL) Version 2, available from the file
- * COPYING in the main directory of this source tree, or the
- * OpenIB.org BSD license below:
- *
- * Redistribution and use in source and binary forms, with or
- * without modification, are permitted provided that the following
- * conditions are met:
- *
- * - Redistributions of source code must retain the above
- * copyright notice, this list of conditions and the following
- * disclaimer.
- *
- * - Redistributions in binary form must reproduce the above
- * copyright notice, this list of conditions and the following
- * disclaimer in the documentation and/or other materials
- * provided with the distribution.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
- * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
- * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
- * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
- * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
- * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
- * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
*/
#ifndef IB_SA_H
diff --git a/include/rdma/ib_smi.h b/include/rdma/ib_smi.h
index 7be0028..fdb8633 100644
--- a/include/rdma/ib_smi.h
+++ b/include/rdma/ib_smi.h
@@ -1,40 +1,13 @@
+/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */
/*
* Copyright (c) 2004 Mellanox Technologies Ltd. All rights reserved.
* Copyright (c) 2004 Infinicon Corporation. All rights reserved.
* Copyright (c) 2004 Intel Corporation. All rights reserved.
* Copyright (c) 2004 Topspin Corporation. All rights reserved.
* Copyright (c) 2004 Voltaire Corporation. All rights reserved.
- *
- * This software is available to you under a choice of one of two
- * licenses. You may choose to be licensed under the terms of the GNU
- * General Public License (GPL) Version 2, available from the file
- * COPYING in the main directory of this source tree, or the
- * OpenIB.org BSD license below:
- *
- * Redistribution and use in source and binary forms, with or
- * without modification, are permitted provided that the following
- * conditions are met:
- *
- * - Redistributions of source code must retain the above
- * copyright notice, this list of conditions and the following
- * disclaimer.
- *
- * - Redistributions in binary form must reproduce the above
- * copyright notice, this list of conditions and the following
- * disclaimer in the documentation and/or other materials
- * provided with the distribution.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
- * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
- * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
- * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
- * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
- * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
- * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
*/
-#if !defined(IB_SMI_H)
+#ifndef IB_SMI_H
#define IB_SMI_H
#include <rdma/ib_mad.h>
diff --git a/include/rdma/ib_umem.h b/include/rdma/ib_umem.h
index e3518fd..71f573a 100644
--- a/include/rdma/ib_umem.h
+++ b/include/rdma/ib_umem.h
@@ -1,33 +1,6 @@
+/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */
/*
* Copyright (c) 2007 Cisco Systems. All rights reserved.
- *
- * This software is available to you under a choice of one of two
- * licenses. You may choose to be licensed under the terms of the GNU
- * General Public License (GPL) Version 2, available from the file
- * COPYING in the main directory of this source tree, or the
- * OpenIB.org BSD license below:
- *
- * Redistribution and use in source and binary forms, with or
- * without modification, are permitted provided that the following
- * conditions are met:
- *
- * - Redistributions of source code must retain the above
- * copyright notice, this list of conditions and the following
- * disclaimer.
- *
- * - Redistributions in binary form must reproduce the above
- * copyright notice, this list of conditions and the following
- * disclaimer in the documentation and/or other materials
- * provided with the distribution.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
- * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
- * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
- * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
- * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
- * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
- * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
*/
#ifndef IB_UMEM_H
diff --git a/include/rdma/ib_umem_odp.h b/include/rdma/ib_umem_odp.h
index 64314ff..d16d2c1 100644
--- a/include/rdma/ib_umem_odp.h
+++ b/include/rdma/ib_umem_odp.h
@@ -1,33 +1,6 @@
+/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */
/*
* Copyright (c) 2014 Mellanox Technologies. All rights reserved.
- *
- * This software is available to you under a choice of one of two
- * licenses. You may choose to be licensed under the terms of the GNU
- * General Public License (GPL) Version 2, available from the file
- * COPYING in the main directory of this source tree, or the
- * OpenIB.org BSD license below:
- *
- * Redistribution and use in source and binary forms, with or
- * without modification, are permitted provided that the following
- * conditions are met:
- *
- * - Redistributions of source code must retain the above
- * copyright notice, this list of conditions and the following
- * disclaimer.
- *
- * - Redistributions in binary form must reproduce the above
- * copyright notice, this list of conditions and the following
- * disclaimer in the documentation and/or other materials
- * provided with the distribution.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
- * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
- * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
- * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
- * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
- * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
- * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
*/
#ifndef IB_UMEM_ODP_H
diff --git a/include/rdma/ib_verbs.h b/include/rdma/ib_verbs.h
index ef2f398..c0b2fa7 100644
--- a/include/rdma/ib_verbs.h
+++ b/include/rdma/ib_verbs.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */
/*
* Copyright (c) 2004 Mellanox Technologies Ltd. All rights reserved.
* Copyright (c) 2004 Infinicon Corporation. All rights reserved.
@@ -6,37 +7,9 @@
* Copyright (c) 2004 Voltaire Corporation. All rights reserved.
* Copyright (c) 2005 Sun Microsystems, Inc. All rights reserved.
* Copyright (c) 2005, 2006, 2007 Cisco Systems. All rights reserved.
- *
- * This software is available to you under a choice of one of two
- * licenses. You may choose to be licensed under the terms of the GNU
- * General Public License (GPL) Version 2, available from the file
- * COPYING in the main directory of this source tree, or the
- * OpenIB.org BSD license below:
- *
- * Redistribution and use in source and binary forms, with or
- * without modification, are permitted provided that the following
- * conditions are met:
- *
- * - Redistributions of source code must retain the above
- * copyright notice, this list of conditions and the following
- * disclaimer.
- *
- * - Redistributions in binary form must reproduce the above
- * copyright notice, this list of conditions and the following
- * disclaimer in the documentation and/or other materials
- * provided with the distribution.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
- * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
- * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
- * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
- * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
- * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
- * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
*/
-#if !defined(IB_VERBS_H)
+#ifndef IB_VERBS_H
#define IB_VERBS_H
#include <linux/types.h>
@@ -75,6 +48,7 @@
struct ib_uqp_object;
struct ib_usrq_object;
struct ib_uwq_object;
+struct rdma_cm_id;
extern struct workqueue_struct *ib_wq;
extern struct workqueue_struct *ib_comp_wq;
@@ -1566,9 +1540,8 @@
struct ib_device *device;
atomic_t usecnt; /* count all exposed resources */
struct inode *inode;
-
- struct mutex tgt_qp_mutex;
- struct list_head tgt_qp_list;
+ struct rw_semaphore tgt_qps_rwsem;
+ struct xarray tgt_qps;
};
struct ib_ah {
@@ -2270,6 +2243,7 @@
struct ib_odp_counters {
atomic64_t faults;
atomic64_t invalidations;
+ atomic64_t prefetch;
};
struct ib_counters {
@@ -2476,7 +2450,7 @@
struct ib_pd *pd, struct ib_udata *udata);
int (*dereg_mr)(struct ib_mr *mr, struct ib_udata *udata);
struct ib_mr *(*alloc_mr)(struct ib_pd *pd, enum ib_mr_type mr_type,
- u32 max_num_sg, struct ib_udata *udata);
+ u32 max_num_sg);
struct ib_mr *(*alloc_mr_integrity)(struct ib_pd *pd,
u32 max_num_data_sg,
u32 max_num_meta_sg);
@@ -2493,9 +2467,8 @@
int (*dealloc_mw)(struct ib_mw *mw);
int (*attach_mcast)(struct ib_qp *qp, union ib_gid *gid, u16 lid);
int (*detach_mcast)(struct ib_qp *qp, union ib_gid *gid, u16 lid);
- struct ib_xrcd *(*alloc_xrcd)(struct ib_device *device,
- struct ib_udata *udata);
- int (*dealloc_xrcd)(struct ib_xrcd *xrcd, struct ib_udata *udata);
+ int (*alloc_xrcd)(struct ib_xrcd *xrcd, struct ib_udata *udata);
+ void (*dealloc_xrcd)(struct ib_xrcd *xrcd, struct ib_udata *udata);
struct ib_flow *(*create_flow)(struct ib_qp *qp,
struct ib_flow_attr *flow_attr,
int domain, struct ib_udata *udata);
@@ -2539,9 +2512,9 @@
struct ib_mr *(*reg_dm_mr)(struct ib_pd *pd, struct ib_dm *dm,
struct ib_dm_mr_attr *attr,
struct uverbs_attr_bundle *attrs);
- struct ib_counters *(*create_counters)(
- struct ib_device *device, struct uverbs_attr_bundle *attrs);
- int (*destroy_counters)(struct ib_counters *counters);
+ int (*create_counters)(struct ib_counters *counters,
+ struct uverbs_attr_bundle *attrs);
+ void (*destroy_counters)(struct ib_counters *counters);
int (*read_counters)(struct ib_counters *counters,
struct ib_counters_read_attr *counters_read_attr,
struct uverbs_attr_bundle *attrs);
@@ -2581,8 +2554,13 @@
/**
* Allows rdma drivers to add their own restrack attributes.
*/
- int (*fill_res_entry)(struct sk_buff *msg,
- struct rdma_restrack_entry *entry);
+ int (*fill_res_mr_entry)(struct sk_buff *msg, struct ib_mr *ibmr);
+ int (*fill_res_mr_entry_raw)(struct sk_buff *msg, struct ib_mr *ibmr);
+ int (*fill_res_cq_entry)(struct sk_buff *msg, struct ib_cq *ibcq);
+ int (*fill_res_cq_entry_raw)(struct sk_buff *msg, struct ib_cq *ibcq);
+ int (*fill_res_qp_entry)(struct sk_buff *msg, struct ib_qp *ibqp);
+ int (*fill_res_qp_entry_raw)(struct sk_buff *msg, struct ib_qp *ibqp);
+ int (*fill_res_cm_id_entry)(struct sk_buff *msg, struct rdma_cm_id *id);
/* Device lifecycle callbacks */
/*
@@ -2637,14 +2615,19 @@
* Allows rdma drivers to add their own restrack attributes
* dumped via 'rdma stat' iproute2 command.
*/
- int (*fill_stat_entry)(struct sk_buff *msg,
- struct rdma_restrack_entry *entry);
+ int (*fill_stat_mr_entry)(struct sk_buff *msg, struct ib_mr *ibmr);
+
+ /* query driver for its ucontext properties */
+ int (*query_ucontext)(struct ib_ucontext *context,
+ struct uverbs_attr_bundle *attrs);
DECLARE_RDMA_OBJ_SIZE(ib_ah);
+ DECLARE_RDMA_OBJ_SIZE(ib_counters);
DECLARE_RDMA_OBJ_SIZE(ib_cq);
DECLARE_RDMA_OBJ_SIZE(ib_pd);
DECLARE_RDMA_OBJ_SIZE(ib_srq);
DECLARE_RDMA_OBJ_SIZE(ib_ucontext);
+ DECLARE_RDMA_OBJ_SIZE(ib_xrcd);
};
struct ib_core_device {
@@ -4263,14 +4246,8 @@
return ib_dereg_mr_user(mr, NULL);
}
-struct ib_mr *ib_alloc_mr_user(struct ib_pd *pd, enum ib_mr_type mr_type,
- u32 max_num_sg, struct ib_udata *udata);
-
-static inline struct ib_mr *ib_alloc_mr(struct ib_pd *pd,
- enum ib_mr_type mr_type, u32 max_num_sg)
-{
- return ib_alloc_mr_user(pd, mr_type, max_num_sg, NULL);
-}
+struct ib_mr *ib_alloc_mr(struct ib_pd *pd, enum ib_mr_type mr_type,
+ u32 max_num_sg);
struct ib_mr *ib_alloc_mr_integrity(struct ib_pd *pd,
u32 max_num_data_sg,
@@ -4321,21 +4298,9 @@
*/
int ib_detach_mcast(struct ib_qp *qp, union ib_gid *gid, u16 lid);
-/**
- * ib_alloc_xrcd - Allocates an XRC domain.
- * @device: The device on which to allocate the XRC domain.
- * @caller: Module name for kernel consumers
- */
-struct ib_xrcd *__ib_alloc_xrcd(struct ib_device *device, const char *caller);
-#define ib_alloc_xrcd(device) \
- __ib_alloc_xrcd((device), KBUILD_MODNAME)
-
-/**
- * ib_dealloc_xrcd - Deallocates an XRC domain.
- * @xrcd: The XRC domain to deallocate.
- * @udata: Valid user data or NULL for kernel object
- */
-int ib_dealloc_xrcd(struct ib_xrcd *xrcd, struct ib_udata *udata);
+struct ib_xrcd *ib_alloc_xrcd_user(struct ib_device *device,
+ struct inode *inode, struct ib_udata *udata);
+int ib_dealloc_xrcd_user(struct ib_xrcd *xrcd, struct ib_udata *udata);
static inline int ib_check_mr_access(int flags)
{
@@ -4417,9 +4382,6 @@
int ib_destroy_wq(struct ib_wq *wq, struct ib_udata *udata);
int ib_modify_wq(struct ib_wq *wq, struct ib_wq_attr *attr,
u32 wq_attr_mask);
-struct ib_rwq_ind_table *ib_create_rwq_ind_table(struct ib_device *device,
- struct ib_rwq_ind_table_init_attr*
- wq_ind_table_init_attr);
int ib_destroy_rwq_ind_table(struct ib_rwq_ind_table *wq_ind_table);
int ib_map_mr_sg(struct ib_mr *mr, struct scatterlist *sg, int sg_nents,
diff --git a/include/rdma/iw_cm.h b/include/rdma/iw_cm.h
index 5aa8a9c..9197540 100644
--- a/include/rdma/iw_cm.h
+++ b/include/rdma/iw_cm.h
@@ -1,35 +1,9 @@
+/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */
/*
* Copyright (c) 2005 Network Appliance, Inc. All rights reserved.
* Copyright (c) 2005 Open Grid Computing, Inc. All rights reserved.
- *
- * This software is available to you under a choice of one of two
- * licenses. You may choose to be licensed under the terms of the GNU
- * General Public License (GPL) Version 2, available from the file
- * COPYING in the main directory of this source tree, or the
- * OpenIB.org BSD license below:
- *
- * Redistribution and use in source and binary forms, with or
- * without modification, are permitted provided that the following
- * conditions are met:
- *
- * - Redistributions of source code must retain the above
- * copyright notice, this list of conditions and the following
- * disclaimer.
- *
- * - Redistributions in binary form must reproduce the above
- * copyright notice, this list of conditions and the following
- * disclaimer in the documentation and/or other materials
- * provided with the distribution.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
- * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
- * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
- * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
- * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
- * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
- * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
*/
+
#ifndef IW_CM_H
#define IW_CM_H
diff --git a/include/rdma/iw_portmap.h b/include/rdma/iw_portmap.h
index c895350..6dbc1a2 100644
--- a/include/rdma/iw_portmap.h
+++ b/include/rdma/iw_portmap.h
@@ -1,35 +1,9 @@
+/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */
/*
* Copyright (c) 2014 Intel Corporation. All rights reserved.
* Copyright (c) 2014 Chelsio, Inc. All rights reserved.
- *
- * This software is available to you under a choice of one of two
- * licenses. You may choose to be licensed under the terms of the GNU
- * General Public License (GPL) Version 2, available from the file
- * COPYING in the main directory of this source tree, or the
- * OpenIB.org BSD license below:
- *
- * Redistribution and use in source and binary forms, with or
- * without modification, are permitted provided that the following
- * conditions are met:
- *
- * - Redistributions of source code must retain the above
- * copyright notice, this list of conditions and the following
- * disclaimer.
- *
- * - Redistributions in binary form must reproduce the above
- * copyright notice, this list of conditions and the following
- * disclaimer in the documentation and/or other materials
- * provided with the distribution.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
- * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
- * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
- * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
- * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
- * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
- * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
*/
+
#ifndef _IW_PORTMAP_H
#define _IW_PORTMAP_H
diff --git a/include/rdma/opa_addr.h b/include/rdma/opa_addr.h
index 66d4393..c6bfa26 100644
--- a/include/rdma/opa_addr.h
+++ b/include/rdma/opa_addr.h
@@ -1,48 +1,6 @@
+/* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */
/*
* Copyright(c) 2017 Intel Corporation.
- *
- * This file is provided under a dual BSD/GPLv2 license. When using or
- * redistributing this file, you may do so under either license.
- *
- * GPL LICENSE SUMMARY
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of version 2 of the GNU General Public License as
- * published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but
- * WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- * General Public License for more details.
- *
- * BSD LICENSE
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- *
- * - Redistributions of source code must retain the above copyright
- * notice, this list of conditions and the following disclaimer.
- * - Redistributions in binary form must reproduce the above copyright
- * notice, this list of conditions and the following disclaimer in
- * the documentation and/or other materials provided with the
- * distribution.
- * - Neither the name of Intel Corporation nor the names of its
- * contributors may be used to endorse or promote products derived
- * from this software without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
- * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
- * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
- * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
- * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- *
*/
#ifndef OPA_ADDR_H
diff --git a/include/rdma/opa_port_info.h b/include/rdma/opa_port_info.h
index 0d9e6d7..73bcac9 100644
--- a/include/rdma/opa_port_info.h
+++ b/include/rdma/opa_port_info.h
@@ -1,36 +1,9 @@
+/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */
/*
* Copyright (c) 2014-2020 Intel Corporation. All rights reserved.
- *
- * This software is available to you under a choice of one of two
- * licenses. You may choose to be licensed under the terms of the GNU
- * General Public License (GPL) Version 2, available from the file
- * COPYING in the main directory of this source tree, or the
- * OpenIB.org BSD license below:
- *
- * Redistribution and use in source and binary forms, with or
- * without modification, are permitted provided that the following
- * conditions are met:
- *
- * - Redistributions of source code must retain the above
- * copyright notice, this list of conditions and the following
- * disclaimer.
- *
- * - Redistributions in binary form must reproduce the above
- * copyright notice, this list of conditions and the following
- * disclaimer in the documentation and/or other materials
- * provided with the distribution.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
- * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
- * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
- * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
- * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
- * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
- * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
*/
-#if !defined(OPA_PORT_INFO_H)
+#ifndef OPA_PORT_INFO_H
#define OPA_PORT_INFO_H
#include <rdma/opa_smi.h>
diff --git a/include/rdma/opa_smi.h b/include/rdma/opa_smi.h
index c7b2ef1..a9f1b57 100644
--- a/include/rdma/opa_smi.h
+++ b/include/rdma/opa_smi.h
@@ -1,36 +1,9 @@
+/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */
/*
* Copyright (c) 2014 Intel Corporation. All rights reserved.
- *
- * This software is available to you under a choice of one of two
- * licenses. You may choose to be licensed under the terms of the GNU
- * General Public License (GPL) Version 2, available from the file
- * COPYING in the main directory of this source tree, or the
- * OpenIB.org BSD license below:
- *
- * Redistribution and use in source and binary forms, with or
- * without modification, are permitted provided that the following
- * conditions are met:
- *
- * - Redistributions of source code must retain the above
- * copyright notice, this list of conditions and the following
- * disclaimer.
- *
- * - Redistributions in binary form must reproduce the above
- * copyright notice, this list of conditions and the following
- * disclaimer in the documentation and/or other materials
- * provided with the distribution.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
- * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
- * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
- * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
- * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
- * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
- * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
*/
-#if !defined(OPA_SMI_H)
+#ifndef OPA_SMI_H
#define OPA_SMI_H
#include <rdma/ib_mad.h>
diff --git a/include/rdma/opa_vnic.h b/include/rdma/opa_vnic.h
index 6f244e7..cbe3c28 100644
--- a/include/rdma/opa_vnic.h
+++ b/include/rdma/opa_vnic.h
@@ -1,52 +1,11 @@
-#ifndef _OPA_VNIC_H
-#define _OPA_VNIC_H
+/* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */
/*
* Copyright(c) 2017 - 2020 Intel Corporation.
- *
- * This file is provided under a dual BSD/GPLv2 license. When using or
- * redistributing this file, you may do so under either license.
- *
- * GPL LICENSE SUMMARY
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of version 2 of the GNU General Public License as
- * published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but
- * WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- * General Public License for more details.
- *
- * BSD LICENSE
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- *
- * - Redistributions of source code must retain the above copyright
- * notice, this list of conditions and the following disclaimer.
- * - Redistributions in binary form must reproduce the above copyright
- * notice, this list of conditions and the following disclaimer in
- * the documentation and/or other materials provided with the
- * distribution.
- * - Neither the name of Intel Corporation nor the names of its
- * contributors may be used to endorse or promote products derived
- * from this software without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
- * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
- * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
- * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
- * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- *
*/
+#ifndef _OPA_VNIC_H
+#define _OPA_VNIC_H
+
/*
* This file contains Intel Omni-Path (OPA) Virtual Network Interface
* Controller (VNIC) specific declarations.
diff --git a/include/rdma/rdma_cm.h b/include/rdma/rdma_cm.h
index 939d7ab..cf5da2a 100644
--- a/include/rdma/rdma_cm.h
+++ b/include/rdma/rdma_cm.h
@@ -1,37 +1,10 @@
+/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */
/*
* Copyright (c) 2005 Voltaire Inc. All rights reserved.
* Copyright (c) 2005 Intel Corporation. All rights reserved.
- *
- * This software is available to you under a choice of one of two
- * licenses. You may choose to be licensed under the terms of the GNU
- * General Public License (GPL) Version 2, available from the file
- * COPYING in the main directory of this source tree, or the
- * OpenIB.org BSD license below:
- *
- * Redistribution and use in source and binary forms, with or
- * without modification, are permitted provided that the following
- * conditions are met:
- *
- * - Redistributions of source code must retain the above
- * copyright notice, this list of conditions and the following
- * disclaimer.
- *
- * - Redistributions in binary form must reproduce the above
- * copyright notice, this list of conditions and the following
- * disclaimer in the documentation and/or other materials
- * provided with the distribution.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
- * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
- * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
- * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
- * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
- * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
- * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
*/
-#if !defined(RDMA_CM_H)
+#ifndef RDMA_CM_H
#define RDMA_CM_H
#include <linux/socket.h>
diff --git a/include/rdma/rdma_cm_ib.h b/include/rdma/rdma_cm_ib.h
index 6a69d71..8354e7d 100644
--- a/include/rdma/rdma_cm_ib.h
+++ b/include/rdma/rdma_cm_ib.h
@@ -1,36 +1,9 @@
+/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */
/*
* Copyright (c) 2006 Intel Corporation. All rights reserved.
- *
- * This software is available to you under a choice of one of two
- * licenses. You may choose to be licensed under the terms of the GNU
- * General Public License (GPL) Version 2, available from the file
- * COPYING in the main directory of this source tree, or the
- * OpenIB.org BSD license below:
- *
- * Redistribution and use in source and binary forms, with or
- * without modification, are permitted provided that the following
- * conditions are met:
- *
- * - Redistributions of source code must retain the above
- * copyright notice, this list of conditions and the following
- * disclaimer.
- *
- * - Redistributions in binary form must reproduce the above
- * copyright notice, this list of conditions and the following
- * disclaimer in the documentation and/or other materials
- * provided with the distribution.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
- * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
- * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
- * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
- * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
- * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
- * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
*/
-#if !defined(RDMA_CM_IB_H)
+#ifndef RDMA_CM_IB_H
#define RDMA_CM_IB_H
#include <rdma/rdma_cm.h>
diff --git a/include/rdma/rdma_netlink.h b/include/rdma/rdma_netlink.h
index ab22759..2758d9d 100644
--- a/include/rdma/rdma_netlink.h
+++ b/include/rdma/rdma_netlink.h
@@ -1,8 +1,8 @@
/* SPDX-License-Identifier: GPL-2.0 */
+
#ifndef _RDMA_NETLINK_H
#define _RDMA_NETLINK_H
-
#include <linux/netlink.h>
#include <uapi/rdma/rdma_netlink.h>
diff --git a/include/rdma/rdma_vt.h b/include/rdma/rdma_vt.h
index ac5a943..9fd217b 100644
--- a/include/rdma/rdma_vt.h
+++ b/include/rdma/rdma_vt.h
@@ -1,53 +1,11 @@
-#ifndef DEF_RDMA_VT_H
-#define DEF_RDMA_VT_H
-
+/* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */
/*
* Copyright(c) 2016 - 2019 Intel Corporation.
- *
- * This file is provided under a dual BSD/GPLv2 license. When using or
- * redistributing this file, you may do so under either license.
- *
- * GPL LICENSE SUMMARY
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of version 2 of the GNU General Public License as
- * published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but
- * WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- * General Public License for more details.
- *
- * BSD LICENSE
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- *
- * - Redistributions of source code must retain the above copyright
- * notice, this list of conditions and the following disclaimer.
- * - Redistributions in binary form must reproduce the above copyright
- * notice, this list of conditions and the following disclaimer in
- * the documentation and/or other materials provided with the
- * distribution.
- * - Neither the name of Intel Corporation nor the names of its
- * contributors may be used to endorse or promote products derived
- * from this software without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
- * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
- * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
- * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
- * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- *
*/
+#ifndef DEF_RDMA_VT_H
+#define DEF_RDMA_VT_H
+
/*
* Structure that low level drivers will populate in order to register with the
* rdmavt layer.
diff --git a/include/rdma/rdmavt_cq.h b/include/rdma/rdmavt_cq.h
index 574eb72..1fe2bb5 100644
--- a/include/rdma/rdmavt_cq.h
+++ b/include/rdma/rdmavt_cq.h
@@ -1,56 +1,11 @@
+/* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */
+/*
+ * Copyright(c) 2016 - 2018 Intel Corporation.
+ */
+
#ifndef DEF_RDMAVT_INCCQ_H
#define DEF_RDMAVT_INCCQ_H
-/*
- *
- * This file is provided under a dual BSD/GPLv2 license. When using or
- * redistributing this file, you may do so under either license.
- *
- * GPL LICENSE SUMMARY
- *
- * Copyright(c) 2016 - 2018 Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of version 2 of the GNU General Public License as
- * published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but
- * WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- * General Public License for more details.
- *
- * BSD LICENSE
- *
- * Copyright(c) 2015 Intel Corporation.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- *
- * - Redistributions of source code must retain the above copyright
- * notice, this list of conditions and the following disclaimer.
- * - Redistributions in binary form must reproduce the above copyright
- * notice, this list of conditions and the following disclaimer in
- * the documentation and/or other materials provided with the
- * distribution.
- * - Neither the name of Intel Corporation nor the names of its
- * contributors may be used to endorse or promote products derived
- * from this software without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
- * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
- * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
- * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
- * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- *
- */
-
#include <linux/kthread.h>
#include <rdma/ib_user_verbs.h>
#include <rdma/ib_verbs.h>
diff --git a/include/rdma/rdmavt_mr.h b/include/rdma/rdmavt_mr.h
index ce6c888..c3367e9 100644
--- a/include/rdma/rdmavt_mr.h
+++ b/include/rdma/rdmavt_mr.h
@@ -1,53 +1,11 @@
-#ifndef DEF_RDMAVT_INCMR_H
-#define DEF_RDMAVT_INCMR_H
-
+/* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */
/*
* Copyright(c) 2016 Intel Corporation.
- *
- * This file is provided under a dual BSD/GPLv2 license. When using or
- * redistributing this file, you may do so under either license.
- *
- * GPL LICENSE SUMMARY
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of version 2 of the GNU General Public License as
- * published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but
- * WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- * General Public License for more details.
- *
- * BSD LICENSE
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- *
- * - Redistributions of source code must retain the above copyright
- * notice, this list of conditions and the following disclaimer.
- * - Redistributions in binary form must reproduce the above copyright
- * notice, this list of conditions and the following disclaimer in
- * the documentation and/or other materials provided with the
- * distribution.
- * - Neither the name of Intel Corporation nor the names of its
- * contributors may be used to endorse or promote products derived
- * from this software without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
- * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
- * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
- * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
- * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- *
*/
+#ifndef DEF_RDMAVT_INCMR_H
+#define DEF_RDMAVT_INCMR_H
+
/*
* For Memory Regions. This stuff should probably be moved into rdmavt/mr.h once
* drivers no longer need access to the MR directly.
diff --git a/include/rdma/rdmavt_qp.h b/include/rdma/rdmavt_qp.h
index 2f1fc23..8275954 100644
--- a/include/rdma/rdmavt_qp.h
+++ b/include/rdma/rdmavt_qp.h
@@ -1,53 +1,11 @@
-#ifndef DEF_RDMAVT_INCQP_H
-#define DEF_RDMAVT_INCQP_H
-
+/* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */
/*
* Copyright(c) 2016 - 2020 Intel Corporation.
- *
- * This file is provided under a dual BSD/GPLv2 license. When using or
- * redistributing this file, you may do so under either license.
- *
- * GPL LICENSE SUMMARY
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of version 2 of the GNU General Public License as
- * published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but
- * WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- * General Public License for more details.
- *
- * BSD LICENSE
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- *
- * - Redistributions of source code must retain the above copyright
- * notice, this list of conditions and the following disclaimer.
- * - Redistributions in binary form must reproduce the above copyright
- * notice, this list of conditions and the following disclaimer in
- * the documentation and/or other materials provided with the
- * distribution.
- * - Neither the name of Intel Corporation nor the names of its
- * contributors may be used to endorse or promote products derived
- * from this software without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
- * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
- * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
- * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
- * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- *
*/
+#ifndef DEF_RDMAVT_INCQP_H
+#define DEF_RDMAVT_INCQP_H
+
#include <rdma/rdma_vt.h>
#include <rdma/ib_pack.h>
#include <rdma/ib_verbs.h>
diff --git a/include/rdma/uverbs_ioctl.h b/include/rdma/uverbs_ioctl.h
index 86de10e..b00270c 100644
--- a/include/rdma/uverbs_ioctl.h
+++ b/include/rdma/uverbs_ioctl.h
@@ -1,33 +1,6 @@
+/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */
/*
* Copyright (c) 2017, Mellanox Technologies inc. All rights reserved.
- *
- * This software is available to you under a choice of one of two
- * licenses. You may choose to be licensed under the terms of the GNU
- * General Public License (GPL) Version 2, available from the file
- * COPYING in the main directory of this source tree, or the
- * OpenIB.org BSD license below:
- *
- * Redistribution and use in source and binary forms, with or
- * without modification, are permitted provided that the following
- * conditions are met:
- *
- * - Redistributions of source code must retain the above
- * copyright notice, this list of conditions and the following
- * disclaimer.
- *
- * - Redistributions in binary form must reproduce the above
- * copyright notice, this list of conditions and the following
- * disclaimer in the documentation and/or other materials
- * provided with the distribution.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
- * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
- * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
- * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
- * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
- * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
- * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
*/
#ifndef _UVERBS_IOCTL_
@@ -652,6 +625,7 @@
struct ib_udata ucore;
struct ib_uverbs_file *ufile;
struct ib_ucontext *context;
+ struct ib_uobject *uobject;
DECLARE_BITMAP(attr_present, UVERBS_API_ATTR_BKEY_LEN);
struct uverbs_attr attrs[];
};
diff --git a/include/rdma/uverbs_named_ioctl.h b/include/rdma/uverbs_named_ioctl.h
index 6ae6cf8..f04f512 100644
--- a/include/rdma/uverbs_named_ioctl.h
+++ b/include/rdma/uverbs_named_ioctl.h
@@ -1,33 +1,6 @@
+/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */
/*
* Copyright (c) 2018, Mellanox Technologies inc. All rights reserved.
- *
- * This software is available to you under a choice of one of two
- * licenses. You may choose to be licensed under the terms of the GNU
- * General Public License (GPL) Version 2, available from the file
- * COPYING in the main directory of this source tree, or the
- * OpenIB.org BSD license below:
- *
- * Redistribution and use in source and binary forms, with or
- * without modification, are permitted provided that the following
- * conditions are met:
- *
- * - Redistributions of source code must retain the above
- * copyright notice, this list of conditions and the following
- * disclaimer.
- *
- * - Redistributions in binary form must reproduce the above
- * copyright notice, this list of conditions and the following
- * disclaimer in the documentation and/or other materials
- * provided with the distribution.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
- * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
- * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
- * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
- * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
- * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
- * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
*/
#ifndef _UVERBS_NAMED_IOCTL_
diff --git a/include/rdma/uverbs_std_types.h b/include/rdma/uverbs_std_types.h
index bf0392a..fe05121 100644
--- a/include/rdma/uverbs_std_types.h
+++ b/include/rdma/uverbs_std_types.h
@@ -1,33 +1,6 @@
+/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */
/*
* Copyright (c) 2017, Mellanox Technologies inc. All rights reserved.
- *
- * This software is available to you under a choice of one of two
- * licenses. You may choose to be licensed under the terms of the GNU
- * General Public License (GPL) Version 2, available from the file
- * COPYING in the main directory of this source tree, or the
- * OpenIB.org BSD license below:
- *
- * Redistribution and use in source and binary forms, with or
- * without modification, are permitted provided that the following
- * conditions are met:
- *
- * - Redistributions of source code must retain the above
- * copyright notice, this list of conditions and the following
- * disclaimer.
- *
- * - Redistributions in binary form must reproduce the above
- * copyright notice, this list of conditions and the following
- * disclaimer in the documentation and/or other materials
- * provided with the distribution.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
- * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
- * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
- * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
- * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
- * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
- * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
*/
#ifndef _UVERBS_STD_TYPES__
@@ -110,6 +83,20 @@
rdma_alloc_abort_uobject(uobj, attrs, false);
}
+static inline void uobj_finalize_uobj_create(struct ib_uobject *uobj,
+ struct uverbs_attr_bundle *attrs)
+{
+ /*
+ * Tell the core code that the write() handler has completed
+ * initializing the object and that the core should commit or
+ * abort this object based upon the return code from the write()
+ * method. Similar to what uverbs_finalize_uobj_create() does for
+ * ioctl()
+ */
+ WARN_ON(attrs->uobject);
+ attrs->uobject = uobj;
+}
+
static inline struct ib_uobject *
__uobj_alloc(const struct uverbs_api_object *obj,
struct uverbs_attr_bundle *attrs, struct ib_device **ib_dev)
diff --git a/include/rdma/uverbs_types.h b/include/rdma/uverbs_types.h
index c15b298..06db27e 100644
--- a/include/rdma/uverbs_types.h
+++ b/include/rdma/uverbs_types.h
@@ -1,33 +1,6 @@
+/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */
/*
* Copyright (c) 2017, Mellanox Technologies inc. All rights reserved.
- *
- * This software is available to you under a choice of one of two
- * licenses. You may choose to be licensed under the terms of the GNU
- * General Public License (GPL) Version 2, available from the file
- * COPYING in the main directory of this source tree, or the
- * OpenIB.org BSD license below:
- *
- * Redistribution and use in source and binary forms, with or
- * without modification, are permitted provided that the following
- * conditions are met:
- *
- * - Redistributions of source code must retain the above
- * copyright notice, this list of conditions and the following
- * disclaimer.
- *
- * - Redistributions in binary form must reproduce the above
- * copyright notice, this list of conditions and the following
- * disclaimer in the documentation and/or other materials
- * provided with the distribution.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
- * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
- * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
- * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
- * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
- * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
- * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
*/
#ifndef _UVERBS_TYPES_
diff --git a/include/scsi/fc/fc_ms.h b/include/scsi/fc/fc_ms.h
index 800d53d..9e273fe 100644
--- a/include/scsi/fc/fc_ms.h
+++ b/include/scsi/fc/fc_ms.h
@@ -63,8 +63,8 @@
* HBA Attribute Length
*/
#define FC_FDMI_HBA_ATTR_NODENAME_LEN 8
-#define FC_FDMI_HBA_ATTR_MANUFACTURER_LEN 64
-#define FC_FDMI_HBA_ATTR_SERIALNUMBER_LEN 64
+#define FC_FDMI_HBA_ATTR_MANUFACTURER_LEN 80
+#define FC_FDMI_HBA_ATTR_SERIALNUMBER_LEN 80
#define FC_FDMI_HBA_ATTR_MODEL_LEN 256
#define FC_FDMI_HBA_ATTR_MODELDESCR_LEN 256
#define FC_FDMI_HBA_ATTR_HARDWAREVERSION_LEN 256
diff --git a/include/scsi/scsi_tcq.h b/include/scsi/scsi_tcq.h
index 6053d46..ea7848e 100644
--- a/include/scsi/scsi_tcq.h
+++ b/include/scsi/scsi_tcq.h
@@ -34,7 +34,7 @@
blk_mq_unique_tag_to_tag(tag));
}
- if (!req)
+ if (!req || !blk_mq_request_started(req))
return NULL;
return blk_mq_rq_to_pdu(req);
}
diff --git a/include/scsi/scsi_transport_iscsi.h b/include/scsi/scsi_transport_iscsi.h
index bdcb6d6..8a26a2f 100644
--- a/include/scsi/scsi_transport_iscsi.h
+++ b/include/scsi/scsi_transport_iscsi.h
@@ -57,7 +57,7 @@
* When not offloading the data path, this is called
* from the scsi work queue without the session lock.
* @xmit_task Requests LLD to transfer cmd task. Returns 0 or the
- * the number of bytes transferred on success, and -Exyz
+ * number of bytes transferred on success, and -Exyz
* value on error. When offloading the data path, this
* is called from queuecommand with the session lock, or
* from the iscsi_conn_send_pdu context with the session
diff --git a/include/target/iscsi/iscsi_target_core.h b/include/target/iscsi/iscsi_target_core.h
index 4fda324..1eccb2a 100644
--- a/include/target/iscsi/iscsi_target_core.h
+++ b/include/target/iscsi/iscsi_target_core.h
@@ -556,10 +556,11 @@
struct socket *sock;
void (*orig_data_ready)(struct sock *);
void (*orig_state_change)(struct sock *);
-#define LOGIN_FLAGS_READ_ACTIVE 1
-#define LOGIN_FLAGS_CLOSED 2
-#define LOGIN_FLAGS_READY 4
-#define LOGIN_FLAGS_INITIAL_PDU 8
+#define LOGIN_FLAGS_READY 0
+#define LOGIN_FLAGS_INITIAL_PDU 1
+#define LOGIN_FLAGS_READ_ACTIVE 2
+#define LOGIN_FLAGS_WRITE_ACTIVE 3
+#define LOGIN_FLAGS_CLOSED 4
unsigned long login_flags;
struct delayed_work login_work;
struct iscsi_login *login;
diff --git a/include/target/target_core_backend.h b/include/target/target_core_backend.h
index f51452e..6336780 100644
--- a/include/target/target_core_backend.h
+++ b/include/target/target_core_backend.h
@@ -40,6 +40,8 @@
ssize_t (*show_configfs_dev_params)(struct se_device *, char *);
sense_reason_t (*parse_cdb)(struct se_cmd *cmd);
+ void (*tmr_notify)(struct se_device *se_dev, enum tcm_tmreq_table,
+ struct list_head *aborted_cmds);
u32 (*get_device_type)(struct se_device *);
sector_t (*get_blocks)(struct se_device *);
sector_t (*get_alignment_offset_lbas)(struct se_device *);
diff --git a/include/target/target_core_base.h b/include/target/target_core_base.h
index 18c3f27..549947d 100644
--- a/include/target/target_core_base.h
+++ b/include/target/target_core_base.h
@@ -207,6 +207,7 @@
TMR_LUN_RESET = 5,
TMR_TARGET_WARM_RESET = 6,
TMR_TARGET_COLD_RESET = 7,
+ TMR_LUN_RESET_PRO = 0x80,
TMR_UNKNOWN = 0xff,
};
diff --git a/include/trace/events/mmflags.h b/include/trace/events/mmflags.h
index 5fb7520..939092d 100644
--- a/include/trace/events/mmflags.h
+++ b/include/trace/events/mmflags.h
@@ -114,8 +114,6 @@
#if defined(CONFIG_X86)
#define __VM_ARCH_SPECIFIC_1 {VM_PAT, "pat" }
-#elif defined(CONFIG_PPC)
-#define __VM_ARCH_SPECIFIC_1 {VM_SAO, "sao" }
#elif defined(CONFIG_PARISC) || defined(CONFIG_IA64)
#define __VM_ARCH_SPECIFIC_1 {VM_GROWSUP, "growsup" }
#elif !defined(CONFIG_MMU)
diff --git a/include/trace/events/ufs.h b/include/trace/events/ufs.h
index 5f30073..84841b3 100644
--- a/include/trace/events/ufs.h
+++ b/include/trace/events/ufs.h
@@ -249,6 +249,37 @@
)
);
+TRACE_EVENT(ufshcd_uic_command,
+ TP_PROTO(const char *dev_name, const char *str, u32 cmd,
+ u32 arg1, u32 arg2, u32 arg3),
+
+ TP_ARGS(dev_name, str, cmd, arg1, arg2, arg3),
+
+ TP_STRUCT__entry(
+ __string(dev_name, dev_name)
+ __string(str, str)
+ __field(u32, cmd)
+ __field(u32, arg1)
+ __field(u32, arg2)
+ __field(u32, arg3)
+ ),
+
+ TP_fast_assign(
+ __assign_str(dev_name, dev_name);
+ __assign_str(str, str);
+ __entry->cmd = cmd;
+ __entry->arg1 = arg1;
+ __entry->arg2 = arg2;
+ __entry->arg3 = arg3;
+ ),
+
+ TP_printk(
+ "%s: %s: cmd: 0x%x, arg1: 0x%x, arg2: 0x%x, arg3: 0x%x",
+ __get_str(str), __get_str(dev_name), __entry->cmd,
+ __entry->arg1, __entry->arg2, __entry->arg3
+ )
+);
+
TRACE_EVENT(ufshcd_upiu,
TP_PROTO(const char *dev_name, const char *str, void *hdr, void *tsf),
diff --git a/include/uapi/linux/fanotify.h b/include/uapi/linux/fanotify.h
index a88c7c6..fbf9c5c 100644
--- a/include/uapi/linux/fanotify.h
+++ b/include/uapi/linux/fanotify.h
@@ -24,7 +24,6 @@
#define FAN_OPEN_PERM 0x00010000 /* File open in perm check */
#define FAN_ACCESS_PERM 0x00020000 /* File accessed in perm check */
#define FAN_OPEN_EXEC_PERM 0x00040000 /* File open/exec in perm check */
-#define FAN_DIR_MODIFY 0x00080000 /* Directory entry was modified */
#define FAN_EVENT_ON_CHILD 0x08000000 /* Interested in child events */
@@ -54,6 +53,11 @@
/* Flags to determine fanotify event format */
#define FAN_REPORT_TID 0x00000100 /* event->pid is thread id */
#define FAN_REPORT_FID 0x00000200 /* Report unique file id */
+#define FAN_REPORT_DIR_FID 0x00000400 /* Report unique directory id */
+#define FAN_REPORT_NAME 0x00000800 /* Report events with name */
+
+/* Convenience macro - FAN_REPORT_NAME requires FAN_REPORT_DIR_FID */
+#define FAN_REPORT_DFID_NAME (FAN_REPORT_DIR_FID | FAN_REPORT_NAME)
/* Deprecated - do not use this in programs and do not add new flags here! */
#define FAN_ALL_INIT_FLAGS (FAN_CLOEXEC | FAN_NONBLOCK | \
@@ -118,6 +122,7 @@
#define FAN_EVENT_INFO_TYPE_FID 1
#define FAN_EVENT_INFO_TYPE_DFID_NAME 2
+#define FAN_EVENT_INFO_TYPE_DFID 3
/* Variable length info record following event metadata */
struct fanotify_event_info_header {
@@ -127,10 +132,11 @@
};
/*
- * Unique file identifier info record. This is used both for
- * FAN_EVENT_INFO_TYPE_FID records and for FAN_EVENT_INFO_TYPE_DFID_NAME
- * records. For FAN_EVENT_INFO_TYPE_DFID_NAME there is additionally a null
- * terminated name immediately after the file handle.
+ * Unique file identifier info record.
+ * This structure is used for records of types FAN_EVENT_INFO_TYPE_FID,
+ * FAN_EVENT_INFO_TYPE_DFID and FAN_EVENT_INFO_TYPE_DFID_NAME.
+ * For FAN_EVENT_INFO_TYPE_DFID_NAME there is additionally a null terminated
+ * name immediately after the file handle.
*/
struct fanotify_event_info_fid {
struct fanotify_event_info_header hdr;
diff --git a/include/uapi/linux/target_core_user.h b/include/uapi/linux/target_core_user.h
index b7b5796..95b1597 100644
--- a/include/uapi/linux/target_core_user.h
+++ b/include/uapi/linux/target_core_user.h
@@ -45,6 +45,7 @@
#define ALIGN_SIZE 64 /* Should be enough for most CPUs */
#define TCMU_MAILBOX_FLAG_CAP_OOOC (1 << 0) /* Out-of-order completions */
#define TCMU_MAILBOX_FLAG_CAP_READ_LEN (1 << 1) /* Read data length */
+#define TCMU_MAILBOX_FLAG_CAP_TMR (1 << 2) /* TMR notifications */
struct tcmu_mailbox {
__u16 version;
@@ -62,6 +63,7 @@
enum tcmu_opcode {
TCMU_OP_PAD = 0,
TCMU_OP_CMD,
+ TCMU_OP_TMR,
};
/*
@@ -128,6 +130,29 @@
} __packed;
+struct tcmu_tmr_entry {
+ struct tcmu_cmd_entry_hdr hdr;
+
+#define TCMU_TMR_UNKNOWN 0
+#define TCMU_TMR_ABORT_TASK 1
+#define TCMU_TMR_ABORT_TASK_SET 2
+#define TCMU_TMR_CLEAR_ACA 3
+#define TCMU_TMR_CLEAR_TASK_SET 4
+#define TCMU_TMR_LUN_RESET 5
+#define TCMU_TMR_TARGET_WARM_RESET 6
+#define TCMU_TMR_TARGET_COLD_RESET 7
+/* Pseudo reset due to received PR OUT */
+#define TCMU_TMR_LUN_RESET_PRO 128
+ __u8 tmr_type;
+
+ __u8 __pad1;
+ __u16 __pad2;
+ __u32 cmd_cnt;
+ __u64 __pad3;
+ __u64 __pad4;
+ __u16 cmd_ids[0];
+} __packed;
+
#define TCMU_OP_ALIGN_SIZE sizeof(__u64)
enum tcmu_genl_cmd {
diff --git a/include/uapi/linux/thermal.h b/include/uapi/linux/thermal.h
index 9621837..c105054 100644
--- a/include/uapi/linux/thermal.h
+++ b/include/uapi/linux/thermal.h
@@ -4,31 +4,86 @@
#define THERMAL_NAME_LENGTH 20
-/* Adding event notification support elements */
-#define THERMAL_GENL_FAMILY_NAME "thermal_event"
-#define THERMAL_GENL_VERSION 0x01
-#define THERMAL_GENL_MCAST_GROUP_NAME "thermal_mc_grp"
-
-/* Events supported by Thermal Netlink */
-enum events {
- THERMAL_AUX0,
- THERMAL_AUX1,
- THERMAL_CRITICAL,
- THERMAL_DEV_FAULT,
+enum thermal_device_mode {
+ THERMAL_DEVICE_DISABLED = 0,
+ THERMAL_DEVICE_ENABLED,
};
-/* attributes of thermal_genl_family */
-enum {
+enum thermal_trip_type {
+ THERMAL_TRIP_ACTIVE = 0,
+ THERMAL_TRIP_PASSIVE,
+ THERMAL_TRIP_HOT,
+ THERMAL_TRIP_CRITICAL,
+};
+
+/* Adding event notification support elements */
+#define THERMAL_GENL_FAMILY_NAME "thermal"
+#define THERMAL_GENL_VERSION 0x01
+#define THERMAL_GENL_SAMPLING_GROUP_NAME "sampling"
+#define THERMAL_GENL_EVENT_GROUP_NAME "event"
+
+/* Attributes of thermal_genl_family */
+enum thermal_genl_attr {
THERMAL_GENL_ATTR_UNSPEC,
- THERMAL_GENL_ATTR_EVENT,
+ THERMAL_GENL_ATTR_TZ,
+ THERMAL_GENL_ATTR_TZ_ID,
+ THERMAL_GENL_ATTR_TZ_TEMP,
+ THERMAL_GENL_ATTR_TZ_TRIP,
+ THERMAL_GENL_ATTR_TZ_TRIP_ID,
+ THERMAL_GENL_ATTR_TZ_TRIP_TYPE,
+ THERMAL_GENL_ATTR_TZ_TRIP_TEMP,
+ THERMAL_GENL_ATTR_TZ_TRIP_HYST,
+ THERMAL_GENL_ATTR_TZ_MODE,
+ THERMAL_GENL_ATTR_TZ_NAME,
+ THERMAL_GENL_ATTR_TZ_CDEV_WEIGHT,
+ THERMAL_GENL_ATTR_TZ_GOV,
+ THERMAL_GENL_ATTR_TZ_GOV_NAME,
+ THERMAL_GENL_ATTR_CDEV,
+ THERMAL_GENL_ATTR_CDEV_ID,
+ THERMAL_GENL_ATTR_CDEV_CUR_STATE,
+ THERMAL_GENL_ATTR_CDEV_MAX_STATE,
+ THERMAL_GENL_ATTR_CDEV_NAME,
+ THERMAL_GENL_ATTR_GOV_NAME,
+
__THERMAL_GENL_ATTR_MAX,
};
#define THERMAL_GENL_ATTR_MAX (__THERMAL_GENL_ATTR_MAX - 1)
-/* commands supported by the thermal_genl_family */
-enum {
+enum thermal_genl_sampling {
+ THERMAL_GENL_SAMPLING_TEMP,
+ __THERMAL_GENL_SAMPLING_MAX,
+};
+#define THERMAL_GENL_SAMPLING_MAX (__THERMAL_GENL_SAMPLING_MAX - 1)
+
+/* Events of thermal_genl_family */
+enum thermal_genl_event {
+ THERMAL_GENL_EVENT_UNSPEC,
+ THERMAL_GENL_EVENT_TZ_CREATE, /* Thermal zone creation */
+ THERMAL_GENL_EVENT_TZ_DELETE, /* Thermal zone deletion */
+ THERMAL_GENL_EVENT_TZ_DISABLE, /* Thermal zone disabed */
+ THERMAL_GENL_EVENT_TZ_ENABLE, /* Thermal zone enabled */
+ THERMAL_GENL_EVENT_TZ_TRIP_UP, /* Trip point crossed the way up */
+ THERMAL_GENL_EVENT_TZ_TRIP_DOWN, /* Trip point crossed the way down */
+ THERMAL_GENL_EVENT_TZ_TRIP_CHANGE, /* Trip point changed */
+ THERMAL_GENL_EVENT_TZ_TRIP_ADD, /* Trip point added */
+ THERMAL_GENL_EVENT_TZ_TRIP_DELETE, /* Trip point deleted */
+ THERMAL_GENL_EVENT_CDEV_ADD, /* Cdev bound to the thermal zone */
+ THERMAL_GENL_EVENT_CDEV_DELETE, /* Cdev unbound */
+ THERMAL_GENL_EVENT_CDEV_STATE_UPDATE, /* Cdev state updated */
+ THERMAL_GENL_EVENT_TZ_GOV_CHANGE, /* Governor policy changed */
+ __THERMAL_GENL_EVENT_MAX,
+};
+#define THERMAL_GENL_EVENT_MAX (__THERMAL_GENL_EVENT_MAX - 1)
+
+/* Commands supported by the thermal_genl_family */
+enum thermal_genl_cmd {
THERMAL_GENL_CMD_UNSPEC,
- THERMAL_GENL_CMD_EVENT,
+ THERMAL_GENL_CMD_TZ_GET_ID, /* List of thermal zones id */
+ THERMAL_GENL_CMD_TZ_GET_TRIP, /* List of thermal trips */
+ THERMAL_GENL_CMD_TZ_GET_TEMP, /* Get the thermal zone temperature */
+ THERMAL_GENL_CMD_TZ_GET_GOV, /* Get the thermal zone governor */
+ THERMAL_GENL_CMD_TZ_GET_MODE, /* Get the thermal zone mode */
+ THERMAL_GENL_CMD_CDEV_GET, /* List of cdev id */
__THERMAL_GENL_CMD_MAX,
};
#define THERMAL_GENL_CMD_MAX (__THERMAL_GENL_CMD_MAX - 1)
diff --git a/include/uapi/rdma/efa-abi.h b/include/uapi/rdma/efa-abi.h
index 53b6e20..507a286 100644
--- a/include/uapi/rdma/efa-abi.h
+++ b/include/uapi/rdma/efa-abi.h
@@ -1,6 +1,6 @@
/* SPDX-License-Identifier: ((GPL-2.0 WITH Linux-syscall-note) OR BSD-2-Clause) */
/*
- * Copyright 2018-2019 Amazon.com, Inc. or its affiliates. All rights reserved.
+ * Copyright 2018-2020 Amazon.com, Inc. or its affiliates. All rights reserved.
*/
#ifndef EFA_ABI_USER_H
@@ -20,6 +20,16 @@
* hex bit offset of the field.
*/
+enum {
+ EFA_ALLOC_UCONTEXT_CMD_COMP_TX_BATCH = 1 << 0,
+ EFA_ALLOC_UCONTEXT_CMD_COMP_MIN_SQ_WR = 1 << 1,
+};
+
+struct efa_ibv_alloc_ucontext_cmd {
+ __u32 comp_mask;
+ __u8 reserved_20[4];
+};
+
enum efa_ibv_user_cmds_supp_udata {
EFA_USER_CMDS_SUPP_UDATA_QUERY_DEVICE = 1 << 0,
EFA_USER_CMDS_SUPP_UDATA_CREATE_AH = 1 << 1,
@@ -31,6 +41,9 @@
__u16 sub_cqs_per_cq;
__u16 inline_buf_size;
__u32 max_llq_size; /* bytes */
+ __u16 max_tx_batch; /* units of 64 bytes */
+ __u16 min_sq_wr;
+ __u8 reserved_a0[4];
};
struct efa_ibv_alloc_pd_resp {
diff --git a/include/uapi/rdma/ib_user_ioctl_cmds.h b/include/uapi/rdma/ib_user_ioctl_cmds.h
index 4961d5e..99dcabf 100644
--- a/include/uapi/rdma/ib_user_ioctl_cmds.h
+++ b/include/uapi/rdma/ib_user_ioctl_cmds.h
@@ -69,6 +69,7 @@
UVERBS_METHOD_INFO_HANDLES,
UVERBS_METHOD_QUERY_PORT,
UVERBS_METHOD_GET_CONTEXT,
+ UVERBS_METHOD_QUERY_CONTEXT,
};
enum uverbs_attrs_invoke_write_cmd_attr_ids {
@@ -87,6 +88,11 @@
UVERBS_ATTR_GET_CONTEXT_CORE_SUPPORT,
};
+enum uverbs_attrs_query_context_attr_ids {
+ UVERBS_ATTR_QUERY_CONTEXT_NUM_COMP_VECTORS,
+ UVERBS_ATTR_QUERY_CONTEXT_CORE_SUPPORT,
+};
+
enum uverbs_attrs_create_cq_cmd_attr_ids {
UVERBS_ATTR_CREATE_CQ_HANDLE,
UVERBS_ATTR_CREATE_CQ_CQE,
@@ -242,6 +248,7 @@
UVERBS_METHOD_DM_MR_REG,
UVERBS_METHOD_MR_DESTROY,
UVERBS_METHOD_ADVISE_MR,
+ UVERBS_METHOD_QUERY_MR,
};
enum uverbs_attrs_mr_destroy_ids {
@@ -255,6 +262,14 @@
UVERBS_ATTR_ADVISE_MR_SGE_LIST,
};
+enum uverbs_attrs_query_mr_cmd_attr_ids {
+ UVERBS_ATTR_QUERY_MR_HANDLE,
+ UVERBS_ATTR_QUERY_MR_RESP_LKEY,
+ UVERBS_ATTR_QUERY_MR_RESP_RKEY,
+ UVERBS_ATTR_QUERY_MR_RESP_LENGTH,
+ UVERBS_ATTR_QUERY_MR_RESP_IOVA,
+};
+
enum uverbs_attrs_create_counters_cmd_attr_ids {
UVERBS_ATTR_CREATE_COUNTERS_HANDLE,
};
diff --git a/include/uapi/rdma/mlx5_user_ioctl_cmds.h b/include/uapi/rdma/mlx5_user_ioctl_cmds.h
index 8e316ef..e24d66d 100644
--- a/include/uapi/rdma/mlx5_user_ioctl_cmds.h
+++ b/include/uapi/rdma/mlx5_user_ioctl_cmds.h
@@ -228,6 +228,10 @@
MLX5_IB_METHOD_FLOW_MATCHER_DESTROY,
};
+enum mlx5_ib_device_query_context_attrs {
+ MLX5_IB_ATTR_QUERY_CONTEXT_RESP_UCTX = (1U << UVERBS_ID_NS_SHIFT),
+};
+
#define MLX5_IB_DW_MATCH_PARAM 0x80
struct mlx5_ib_match_params {
@@ -259,7 +263,7 @@
MLX5_IB_ATTR_CREATE_FLOW_FLAGS,
};
-enum mlx5_ib_destoy_flow_attrs {
+enum mlx5_ib_destroy_flow_attrs {
MLX5_IB_ATTR_DESTROY_FLOW_HANDLE = (1U << UVERBS_ID_NS_SHIFT),
};
@@ -286,4 +290,14 @@
MLX5_IB_ATTR_CREATE_PACKET_REFORMAT_DATA_BUF,
};
+enum mlx5_ib_query_pd_attrs {
+ MLX5_IB_ATTR_QUERY_PD_HANDLE = (1U << UVERBS_ID_NS_SHIFT),
+ MLX5_IB_ATTR_QUERY_PD_RESP_PDN,
+};
+
+enum mlx5_ib_pd_methods {
+ MLX5_IB_METHOD_PD_QUERY = (1U << UVERBS_ID_NS_SHIFT),
+
+};
+
#endif
diff --git a/include/uapi/rdma/qedr-abi.h b/include/uapi/rdma/qedr-abi.h
index a0b83c9..bf7333b 100644
--- a/include/uapi/rdma/qedr-abi.h
+++ b/include/uapi/rdma/qedr-abi.h
@@ -39,8 +39,9 @@
/* user kernel communication data structures. */
enum qedr_alloc_ucontext_flags {
- QEDR_ALLOC_UCTX_RESERVED = 1 << 0,
- QEDR_ALLOC_UCTX_DB_REC = 1 << 1
+ QEDR_ALLOC_UCTX_EDPM_MODE = 1 << 0,
+ QEDR_ALLOC_UCTX_DB_REC = 1 << 1,
+ QEDR_SUPPORT_DPM_SIZES = 1 << 2,
};
struct qedr_alloc_ucontext_req {
@@ -50,13 +51,14 @@
#define QEDR_LDPM_MAX_SIZE (8192)
#define QEDR_EDPM_TRANS_SIZE (64)
+#define QEDR_EDPM_MAX_SIZE (ROCE_REQ_MAX_INLINE_DATA_SIZE)
enum qedr_rdma_dpm_type {
QEDR_DPM_TYPE_NONE = 0,
QEDR_DPM_TYPE_ROCE_ENHANCED = 1 << 0,
QEDR_DPM_TYPE_ROCE_LEGACY = 1 << 1,
QEDR_DPM_TYPE_IWARP_LEGACY = 1 << 2,
- QEDR_DPM_TYPE_RESERVED = 1 << 3,
+ QEDR_DPM_TYPE_ROCE_EDPM_MODE = 1 << 3,
QEDR_DPM_SIZES_SET = 1 << 4,
};
@@ -77,6 +79,8 @@
__u16 ldpm_limit_size;
__u8 edpm_trans_size;
__u8 reserved;
+ __u16 edpm_limit_size;
+ __u8 padding[6];
};
struct qedr_alloc_pd_ureq {
diff --git a/include/uapi/rdma/rdma_netlink.h b/include/uapi/rdma/rdma_netlink.h
index 8e27778..d2f5b83 100644
--- a/include/uapi/rdma/rdma_netlink.h
+++ b/include/uapi/rdma/rdma_netlink.h
@@ -287,6 +287,12 @@
RDMA_NLDEV_CMD_STAT_DEL,
+ RDMA_NLDEV_CMD_RES_QP_GET_RAW,
+
+ RDMA_NLDEV_CMD_RES_CQ_GET_RAW,
+
+ RDMA_NLDEV_CMD_RES_MR_GET_RAW,
+
RDMA_NLDEV_NUM_OPS
};
@@ -525,6 +531,8 @@
*/
RDMA_NLDEV_ATTR_DEV_DIM, /* u8 */
+ RDMA_NLDEV_ATTR_RES_RAW, /* binary */
+
/*
* Always the end
*/
@@ -561,5 +569,6 @@
*/
enum rdma_nl_counter_mask {
RDMA_COUNTER_MASK_QP_TYPE = 1,
+ RDMA_COUNTER_MASK_PID = 1 << 1,
};
#endif /* _UAPI_RDMA_NETLINK_H */
diff --git a/include/uapi/rdma/rdma_user_ioctl.h b/include/uapi/rdma/rdma_user_ioctl.h
index d92d272..53c5518 100644
--- a/include/uapi/rdma/rdma_user_ioctl.h
+++ b/include/uapi/rdma/rdma_user_ioctl.h
@@ -43,7 +43,7 @@
/*
* General blocks assignments
- * It is closed on purpose do not expose it it user space
+ * It is closed on purpose - do not expose it to user space
* #define MAD_CMD_BASE 0x00
* #define HFI1_CMD_BAS 0xE0
*/
diff --git a/include/uapi/scsi/fc/fc_els.h b/include/uapi/scsi/fc/fc_els.h
index 66318c4..8c704e5 100644
--- a/include/uapi/scsi/fc/fc_els.h
+++ b/include/uapi/scsi/fc/fc_els.h
@@ -41,6 +41,7 @@
ELS_REC = 0x13, /* read exchange concise */
ELS_SRR = 0x14, /* sequence retransmission request */
ELS_FPIN = 0x16, /* Fabric Performance Impact Notification */
+ ELS_RDP = 0x18, /* Read Diagnostic Parameters */
ELS_RDF = 0x19, /* Register Diagnostic Functions */
ELS_PRLI = 0x20, /* process login */
ELS_PRLO = 0x21, /* process logout */
@@ -110,6 +111,7 @@
[ELS_REC] = "REC", \
[ELS_SRR] = "SRR", \
[ELS_FPIN] = "FPIN", \
+ [ELS_RDP] = "RDP", \
[ELS_RDF] = "RDF", \
[ELS_PRLI] = "PRLI", \
[ELS_PRLO] = "PRLO", \
diff --git a/include/uapi/xen/gntdev.h b/include/uapi/xen/gntdev.h
index fe4423e..9ac5515 100644
--- a/include/uapi/xen/gntdev.h
+++ b/include/uapi/xen/gntdev.h
@@ -66,7 +66,7 @@
/*
* Removes the grant references from the mapping table of an instance of
- * of gntdev. N.B. munmap() must be called on the relevant virtual address(es)
+ * gntdev. N.B. munmap() must be called on the relevant virtual address(es)
* before this ioctl is called, or an error will result.
*/
#define IOCTL_GNTDEV_UNMAP_GRANT_REF \
diff --git a/include/xen/page.h b/include/xen/page.h
index df6d6b6..285677b 100644
--- a/include/xen/page.h
+++ b/include/xen/page.h
@@ -24,7 +24,6 @@
#define XEN_PFN_DOWN(x) ((x) >> XEN_PAGE_SHIFT)
#define XEN_PFN_UP(x) (((x) + XEN_PAGE_SIZE-1) >> XEN_PAGE_SHIFT)
-#define XEN_PFN_PHYS(x) ((phys_addr_t)(x) << XEN_PAGE_SHIFT)
#include <asm/xen/page.h>
diff --git a/include/xen/swiotlb-xen.h b/include/xen/swiotlb-xen.h
index ffc0d39..d5eaf9d 100644
--- a/include/xen/swiotlb-xen.h
+++ b/include/xen/swiotlb-xen.h
@@ -4,10 +4,10 @@
#include <linux/swiotlb.h>
-void xen_dma_sync_for_cpu(dma_addr_t handle, phys_addr_t paddr, size_t size,
- enum dma_data_direction dir);
-void xen_dma_sync_for_device(dma_addr_t handle, phys_addr_t paddr, size_t size,
- enum dma_data_direction dir);
+void xen_dma_sync_for_cpu(struct device *dev, dma_addr_t handle,
+ size_t size, enum dma_data_direction dir);
+void xen_dma_sync_for_device(struct device *dev, dma_addr_t handle,
+ size_t size, enum dma_data_direction dir);
extern int xen_swiotlb_init(int verbose, bool early);
extern const struct dma_map_ops xen_swiotlb_dma_ops;
diff --git a/init/Makefile b/init/Makefile
index 57499b1..6bc37f6 100644
--- a/init/Makefile
+++ b/init/Makefile
@@ -18,7 +18,6 @@
mounts-y := do_mounts.o
mounts-$(CONFIG_BLK_DEV_RAM) += do_mounts_rd.o
mounts-$(CONFIG_BLK_DEV_INITRD) += do_mounts_initrd.o
-mounts-$(CONFIG_BLK_DEV_MD) += do_mounts_md.o
# dependencies on generated files need to be listed explicitly
$(obj)/version.o: include/generated/compile.h
diff --git a/init/do_mounts.c b/init/do_mounts.c
index 29d326b..b5f9604 100644
--- a/init/do_mounts.c
+++ b/init/do_mounts.c
@@ -23,12 +23,11 @@
#include <linux/nfs_fs.h>
#include <linux/nfs_fs_sb.h>
#include <linux/nfs_mount.h>
+#include <linux/raid/detect.h>
#include <uapi/linux/mount.h>
#include "do_mounts.h"
-int __initdata rd_doload; /* 1 = load RAM disk, 0 = don't load */
-
int root_mountflags = MS_RDONLY | MS_SILENT;
static char * __initdata root_device_name;
static char __initdata saved_root_name[64];
@@ -38,7 +37,7 @@
static int __init load_ramdisk(char *str)
{
- rd_doload = simple_strtol(str,NULL,0) & 3;
+ pr_warn("ignoring the deprecated load_ramdisk= option\n");
return 1;
}
__setup("load_ramdisk=", load_ramdisk);
@@ -396,20 +395,20 @@
int ret;
if (data) {
- /* do_mount() requires a full page as fifth argument */
+ /* init_mount() requires a full page as fifth argument */
p = alloc_page(GFP_KERNEL);
if (!p)
return -ENOMEM;
data_page = page_address(p);
- /* zero-pad. do_mount() will make sure it's terminated */
+ /* zero-pad. init_mount() will make sure it's terminated */
strncpy(data_page, data, PAGE_SIZE);
}
- ret = do_mount(name, "/root", fs, flags, data_page);
+ ret = init_mount(name, "/root", fs, flags, data_page);
if (ret)
goto out;
- ksys_chdir("/root");
+ init_chdir("/root");
s = current->fs->pwd.dentry->d_sb;
ROOT_DEV = s->s_dev;
printk(KERN_INFO
@@ -552,66 +551,20 @@
}
#endif
-#if defined(CONFIG_BLK_DEV_RAM) || defined(CONFIG_BLK_DEV_FD)
-void __init change_floppy(char *fmt, ...)
-{
- struct termios termios;
- char buf[80];
- char c;
- int fd;
- va_list args;
- va_start(args, fmt);
- vsprintf(buf, fmt, args);
- va_end(args);
- fd = ksys_open("/dev/root", O_RDWR | O_NDELAY, 0);
- if (fd >= 0) {
- ksys_ioctl(fd, FDEJECT, 0);
- ksys_close(fd);
- }
- printk(KERN_NOTICE "VFS: Insert %s and press ENTER\n", buf);
- fd = ksys_open("/dev/console", O_RDWR, 0);
- if (fd >= 0) {
- ksys_ioctl(fd, TCGETS, (long)&termios);
- termios.c_lflag &= ~ICANON;
- ksys_ioctl(fd, TCSETSF, (long)&termios);
- ksys_read(fd, &c, 1);
- termios.c_lflag |= ICANON;
- ksys_ioctl(fd, TCSETSF, (long)&termios);
- ksys_close(fd);
- }
-}
-#endif
-
void __init mount_root(void)
{
#ifdef CONFIG_ROOT_NFS
if (ROOT_DEV == Root_NFS) {
- if (mount_nfs_root())
- return;
-
- printk(KERN_ERR "VFS: Unable to mount root fs via NFS, trying floppy.\n");
- ROOT_DEV = Root_FD0;
+ if (!mount_nfs_root())
+ printk(KERN_ERR "VFS: Unable to mount root fs via NFS.\n");
+ return;
}
#endif
#ifdef CONFIG_CIFS_ROOT
if (ROOT_DEV == Root_CIFS) {
- if (mount_cifs_root())
- return;
-
- printk(KERN_ERR "VFS: Unable to mount root fs via SMB, trying floppy.\n");
- ROOT_DEV = Root_FD0;
- }
-#endif
-#ifdef CONFIG_BLK_DEV_FD
- if (MAJOR(ROOT_DEV) == FLOPPY_MAJOR) {
- /* rd_doload is 2 for a dual initrd/ramload setup */
- if (rd_doload==2) {
- if (rd_load_disk(1)) {
- ROOT_DEV = Root_RAM1;
- root_device_name = NULL;
- }
- } else
- change_floppy("root floppy");
+ if (!mount_cifs_root())
+ printk(KERN_ERR "VFS: Unable to mount root fs via SMB.\n");
+ return;
}
#endif
#ifdef CONFIG_BLOCK
@@ -630,8 +583,6 @@
*/
void __init prepare_namespace(void)
{
- int is_floppy;
-
if (root_delay) {
printk(KERN_INFO "Waiting %d sec before mounting root device...\n",
root_delay);
@@ -674,16 +625,11 @@
async_synchronize_full();
}
- is_floppy = MAJOR(ROOT_DEV) == FLOPPY_MAJOR;
-
- if (is_floppy && rd_doload && rd_load_disk(0))
- ROOT_DEV = Root_RAM0;
-
mount_root();
out:
devtmpfs_mount();
- do_mount(".", "/", NULL, MS_MOVE, NULL);
- ksys_chroot(".");
+ init_mount(".", "/", NULL, MS_MOVE, NULL);
+ init_chroot(".");
}
static bool is_tmpfs;
diff --git a/init/do_mounts.h b/init/do_mounts.h
index 0bb0806..7a29ac3 100644
--- a/init/do_mounts.h
+++ b/init/do_mounts.h
@@ -8,26 +8,16 @@
#include <linux/mount.h>
#include <linux/major.h>
#include <linux/root_dev.h>
+#include <linux/init_syscalls.h>
-void change_floppy(char *fmt, ...);
void mount_block_root(char *name, int flags);
void mount_root(void);
extern int root_mountflags;
-static inline int create_dev(char *name, dev_t dev)
+static inline __init int create_dev(char *name, dev_t dev)
{
- ksys_unlink(name);
- return ksys_mknod(name, S_IFBLK|0600, new_encode_dev(dev));
-}
-
-static inline u32 bstat(char *name)
-{
- struct kstat stat;
- if (vfs_stat(name, &stat) != 0)
- return 0;
- if (!S_ISBLK(stat.mode))
- return 0;
- return stat.rdev;
+ init_unlink(name);
+ return init_mknod(name, S_IFBLK | 0600, new_encode_dev(dev));
}
#ifdef CONFIG_BLK_DEV_RAM
@@ -51,13 +41,3 @@
static inline bool initrd_load(void) { return false; }
#endif
-
-#ifdef CONFIG_BLK_DEV_MD
-
-void md_run_setup(void);
-
-#else
-
-static inline void md_run_setup(void) {}
-
-#endif
diff --git a/init/do_mounts_initrd.c b/init/do_mounts_initrd.c
index 53314d7..533d81e 100644
--- a/init/do_mounts_initrd.c
+++ b/init/do_mounts_initrd.c
@@ -51,14 +51,14 @@
}
early_param("initrd", early_initrd);
-static int init_linuxrc(struct subprocess_info *info, struct cred *new)
+static int __init init_linuxrc(struct subprocess_info *info, struct cred *new)
{
ksys_unshare(CLONE_FS | CLONE_FILES);
console_on_rootfs();
/* move initrd over / and chdir/chroot in initrd root */
- ksys_chdir("/root");
- do_mount(".", "/", NULL, MS_MOVE, NULL);
- ksys_chroot(".");
+ init_chdir("/root");
+ init_mount(".", "/", NULL, MS_MOVE, NULL);
+ init_chroot(".");
ksys_setsid();
return 0;
}
@@ -70,12 +70,14 @@
extern char *envp_init[];
int error;
+ pr_warn("using deprecated initrd support, will be removed in 2021.\n");
+
real_root_dev = new_encode_dev(ROOT_DEV);
create_dev("/dev/root.old", Root_RAM0);
/* mount initrd on rootfs' /root */
mount_block_root("/dev/root.old", root_mountflags & ~MS_RDONLY);
- ksys_mkdir("/old", 0700);
- ksys_chdir("/old");
+ init_mkdir("/old", 0700);
+ init_chdir("/old");
/*
* In case that a resume from disk is carried out by linuxrc or one of
@@ -92,39 +94,30 @@
current->flags &= ~PF_FREEZER_SKIP;
/* move initrd to rootfs' /old */
- do_mount("..", ".", NULL, MS_MOVE, NULL);
+ init_mount("..", ".", NULL, MS_MOVE, NULL);
/* switch root and cwd back to / of rootfs */
- ksys_chroot("..");
+ init_chroot("..");
if (new_decode_dev(real_root_dev) == Root_RAM0) {
- ksys_chdir("/old");
+ init_chdir("/old");
return;
}
- ksys_chdir("/");
+ init_chdir("/");
ROOT_DEV = new_decode_dev(real_root_dev);
mount_root();
printk(KERN_NOTICE "Trying to move old root to /initrd ... ");
- error = do_mount("/old", "/root/initrd", NULL, MS_MOVE, NULL);
+ error = init_mount("/old", "/root/initrd", NULL, MS_MOVE, NULL);
if (!error)
printk("okay\n");
else {
- int fd = ksys_open("/dev/root.old", O_RDWR, 0);
if (error == -ENOENT)
printk("/initrd does not exist. Ignored.\n");
else
printk("failed\n");
printk(KERN_NOTICE "Unmounting old root\n");
- ksys_umount("/old", MNT_DETACH);
- printk(KERN_NOTICE "Trying to free ramdisk memory ... ");
- if (fd < 0) {
- error = fd;
- } else {
- error = ksys_ioctl(fd, BLKFLSBUF, 0);
- ksys_close(fd);
- }
- printk(!error ? "okay\n" : "failed\n");
+ init_umount("/old", MNT_DETACH);
}
}
@@ -139,11 +132,11 @@
* mounted in the normal path.
*/
if (rd_load_image("/initrd.image") && ROOT_DEV != Root_RAM0) {
- ksys_unlink("/initrd.image");
+ init_unlink("/initrd.image");
handle_initrd();
return true;
}
}
- ksys_unlink("/initrd.image");
+ init_unlink("/initrd.image");
return false;
}
diff --git a/init/do_mounts_md.c b/init/do_mounts_md.c
deleted file mode 100644
index b840315..0000000
--- a/init/do_mounts_md.c
+++ /dev/null
@@ -1,304 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0
-#include <linux/delay.h>
-#include <linux/raid/md_u.h>
-#include <linux/raid/md_p.h>
-
-#include "do_mounts.h"
-
-/*
- * When md (and any require personalities) are compiled into the kernel
- * (not a module), arrays can be assembles are boot time using with AUTODETECT
- * where specially marked partitions are registered with md_autodetect_dev(),
- * and with MD_BOOT where devices to be collected are given on the boot line
- * with md=.....
- * The code for that is here.
- */
-
-#ifdef CONFIG_MD_AUTODETECT
-static int __initdata raid_noautodetect;
-#else
-static int __initdata raid_noautodetect=1;
-#endif
-static int __initdata raid_autopart;
-
-static struct {
- int minor;
- int partitioned;
- int level;
- int chunk;
- char *device_names;
-} md_setup_args[256] __initdata;
-
-static int md_setup_ents __initdata;
-
-/*
- * Parse the command-line parameters given our kernel, but do not
- * actually try to invoke the MD device now; that is handled by
- * md_setup_drive after the low-level disk drivers have initialised.
- *
- * 27/11/1999: Fixed to work correctly with the 2.3 kernel (which
- * assigns the task of parsing integer arguments to the
- * invoked program now). Added ability to initialise all
- * the MD devices (by specifying multiple "md=" lines)
- * instead of just one. -- KTK
- * 18May2000: Added support for persistent-superblock arrays:
- * md=n,0,factor,fault,device-list uses RAID0 for device n
- * md=n,-1,factor,fault,device-list uses LINEAR for device n
- * md=n,device-list reads a RAID superblock from the devices
- * elements in device-list are read by name_to_kdev_t so can be
- * a hex number or something like /dev/hda1 /dev/sdb
- * 2001-06-03: Dave Cinege <dcinege@psychosis.com>
- * Shifted name_to_kdev_t() and related operations to md_set_drive()
- * for later execution. Rewrote section to make devfs compatible.
- */
-static int __init md_setup(char *str)
-{
- int minor, level, factor, fault, partitioned = 0;
- char *pername = "";
- char *str1;
- int ent;
-
- if (*str == 'd') {
- partitioned = 1;
- str++;
- }
- if (get_option(&str, &minor) != 2) { /* MD Number */
- printk(KERN_WARNING "md: Too few arguments supplied to md=.\n");
- return 0;
- }
- str1 = str;
- for (ent=0 ; ent< md_setup_ents ; ent++)
- if (md_setup_args[ent].minor == minor &&
- md_setup_args[ent].partitioned == partitioned) {
- printk(KERN_WARNING "md: md=%s%d, Specified more than once. "
- "Replacing previous definition.\n", partitioned?"d":"", minor);
- break;
- }
- if (ent >= ARRAY_SIZE(md_setup_args)) {
- printk(KERN_WARNING "md: md=%s%d - too many md initialisations\n", partitioned?"d":"", minor);
- return 0;
- }
- if (ent >= md_setup_ents)
- md_setup_ents++;
- switch (get_option(&str, &level)) { /* RAID level */
- case 2: /* could be 0 or -1.. */
- if (level == 0 || level == LEVEL_LINEAR) {
- if (get_option(&str, &factor) != 2 || /* Chunk Size */
- get_option(&str, &fault) != 2) {
- printk(KERN_WARNING "md: Too few arguments supplied to md=.\n");
- return 0;
- }
- md_setup_args[ent].level = level;
- md_setup_args[ent].chunk = 1 << (factor+12);
- if (level == LEVEL_LINEAR)
- pername = "linear";
- else
- pername = "raid0";
- break;
- }
- /* FALL THROUGH */
- case 1: /* the first device is numeric */
- str = str1;
- /* FALL THROUGH */
- case 0:
- md_setup_args[ent].level = LEVEL_NONE;
- pername="super-block";
- }
-
- printk(KERN_INFO "md: Will configure md%d (%s) from %s, below.\n",
- minor, pername, str);
- md_setup_args[ent].device_names = str;
- md_setup_args[ent].partitioned = partitioned;
- md_setup_args[ent].minor = minor;
-
- return 1;
-}
-
-static void __init md_setup_drive(void)
-{
- int minor, i, ent, partitioned;
- dev_t dev;
- dev_t devices[MD_SB_DISKS+1];
-
- for (ent = 0; ent < md_setup_ents ; ent++) {
- int fd;
- int err = 0;
- char *devname;
- mdu_disk_info_t dinfo;
- char name[16];
-
- minor = md_setup_args[ent].minor;
- partitioned = md_setup_args[ent].partitioned;
- devname = md_setup_args[ent].device_names;
-
- sprintf(name, "/dev/md%s%d", partitioned?"_d":"", minor);
- if (partitioned)
- dev = MKDEV(mdp_major, minor << MdpMinorShift);
- else
- dev = MKDEV(MD_MAJOR, minor);
- create_dev(name, dev);
- for (i = 0; i < MD_SB_DISKS && devname != NULL; i++) {
- char *p;
- char comp_name[64];
- u32 rdev;
-
- p = strchr(devname, ',');
- if (p)
- *p++ = 0;
-
- dev = name_to_dev_t(devname);
- if (strncmp(devname, "/dev/", 5) == 0)
- devname += 5;
- snprintf(comp_name, 63, "/dev/%s", devname);
- rdev = bstat(comp_name);
- if (rdev)
- dev = new_decode_dev(rdev);
- if (!dev) {
- printk(KERN_WARNING "md: Unknown device name: %s\n", devname);
- break;
- }
-
- devices[i] = dev;
-
- devname = p;
- }
- devices[i] = 0;
-
- if (!i)
- continue;
-
- printk(KERN_INFO "md: Loading md%s%d: %s\n",
- partitioned ? "_d" : "", minor,
- md_setup_args[ent].device_names);
-
- fd = ksys_open(name, 0, 0);
- if (fd < 0) {
- printk(KERN_ERR "md: open failed - cannot start "
- "array %s\n", name);
- continue;
- }
- if (ksys_ioctl(fd, SET_ARRAY_INFO, 0) == -EBUSY) {
- printk(KERN_WARNING
- "md: Ignoring md=%d, already autodetected. (Use raid=noautodetect)\n",
- minor);
- ksys_close(fd);
- continue;
- }
-
- if (md_setup_args[ent].level != LEVEL_NONE) {
- /* non-persistent */
- mdu_array_info_t ainfo;
- ainfo.level = md_setup_args[ent].level;
- ainfo.size = 0;
- ainfo.nr_disks =0;
- ainfo.raid_disks =0;
- while (devices[ainfo.raid_disks])
- ainfo.raid_disks++;
- ainfo.md_minor =minor;
- ainfo.not_persistent = 1;
-
- ainfo.state = (1 << MD_SB_CLEAN);
- ainfo.layout = 0;
- ainfo.chunk_size = md_setup_args[ent].chunk;
- err = ksys_ioctl(fd, SET_ARRAY_INFO, (long)&ainfo);
- for (i = 0; !err && i <= MD_SB_DISKS; i++) {
- dev = devices[i];
- if (!dev)
- break;
- dinfo.number = i;
- dinfo.raid_disk = i;
- dinfo.state = (1<<MD_DISK_ACTIVE)|(1<<MD_DISK_SYNC);
- dinfo.major = MAJOR(dev);
- dinfo.minor = MINOR(dev);
- err = ksys_ioctl(fd, ADD_NEW_DISK,
- (long)&dinfo);
- }
- } else {
- /* persistent */
- for (i = 0; i <= MD_SB_DISKS; i++) {
- dev = devices[i];
- if (!dev)
- break;
- dinfo.major = MAJOR(dev);
- dinfo.minor = MINOR(dev);
- ksys_ioctl(fd, ADD_NEW_DISK, (long)&dinfo);
- }
- }
- if (!err)
- err = ksys_ioctl(fd, RUN_ARRAY, 0);
- if (err)
- printk(KERN_WARNING "md: starting md%d failed\n", minor);
- else {
- /* reread the partition table.
- * I (neilb) and not sure why this is needed, but I cannot
- * boot a kernel with devfs compiled in from partitioned md
- * array without it
- */
- ksys_close(fd);
- fd = ksys_open(name, 0, 0);
- ksys_ioctl(fd, BLKRRPART, 0);
- }
- ksys_close(fd);
- }
-}
-
-static int __init raid_setup(char *str)
-{
- int len, pos;
-
- len = strlen(str) + 1;
- pos = 0;
-
- while (pos < len) {
- char *comma = strchr(str+pos, ',');
- int wlen;
- if (comma)
- wlen = (comma-str)-pos;
- else wlen = (len-1)-pos;
-
- if (!strncmp(str, "noautodetect", wlen))
- raid_noautodetect = 1;
- if (!strncmp(str, "autodetect", wlen))
- raid_noautodetect = 0;
- if (strncmp(str, "partitionable", wlen)==0)
- raid_autopart = 1;
- if (strncmp(str, "part", wlen)==0)
- raid_autopart = 1;
- pos += wlen+1;
- }
- return 1;
-}
-
-__setup("raid=", raid_setup);
-__setup("md=", md_setup);
-
-static void __init autodetect_raid(void)
-{
- int fd;
-
- /*
- * Since we don't want to detect and use half a raid array, we need to
- * wait for the known devices to complete their probing
- */
- printk(KERN_INFO "md: Waiting for all devices to be available before autodetect\n");
- printk(KERN_INFO "md: If you don't use raid, use raid=noautodetect\n");
-
- wait_for_device_probe();
-
- fd = ksys_open("/dev/md0", 0, 0);
- if (fd >= 0) {
- ksys_ioctl(fd, RAID_AUTORUN, raid_autopart);
- ksys_close(fd);
- }
-}
-
-void __init md_run_setup(void)
-{
- create_dev("/dev/md0", MKDEV(MD_MAJOR, 0));
-
- if (raid_noautodetect)
- printk(KERN_INFO "md: Skipping autodetection of RAID arrays. (raid=autodetect will force)\n");
- else
- autodetect_raid();
- md_setup_drive();
-}
diff --git a/init/do_mounts_rd.c b/init/do_mounts_rd.c
index 32fb049d..ac021ae 100644
--- a/init/do_mounts_rd.c
+++ b/init/do_mounts_rd.c
@@ -14,12 +14,12 @@
#include <linux/decompress/generic.h>
-
-int __initdata rd_prompt = 1;/* 1 = prompt for RAM disk, 0 = don't prompt */
+static struct file *in_file, *out_file;
+static loff_t in_pos, out_pos;
static int __init prompt_ramdisk(char *str)
{
- rd_prompt = simple_strtol(str,NULL,0) & 1;
+ pr_warn("ignoring the deprecated prompt_ramdisk= option\n");
return 1;
}
__setup("prompt_ramdisk=", prompt_ramdisk);
@@ -33,7 +33,7 @@
}
__setup("ramdisk_start=", ramdisk_start_setup);
-static int __init crd_load(int in_fd, int out_fd, decompress_fn deco);
+static int __init crd_load(decompress_fn deco);
/*
* This routine tries to find a RAM disk image to load, and returns the
@@ -55,7 +55,8 @@
* lz4
*/
static int __init
-identify_ramdisk_image(int fd, int start_block, decompress_fn *decompressor)
+identify_ramdisk_image(struct file *file, loff_t pos,
+ decompress_fn *decompressor)
{
const int size = 512;
struct minix_super_block *minixsb;
@@ -66,6 +67,7 @@
unsigned char *buf;
const char *compress_name;
unsigned long n;
+ int start_block = rd_image_start;
buf = kmalloc(size, GFP_KERNEL);
if (!buf)
@@ -80,8 +82,8 @@
/*
* Read block 0 to test for compressed kernel
*/
- ksys_lseek(fd, start_block * BLOCK_SIZE, 0);
- ksys_read(fd, buf, size);
+ pos = start_block * BLOCK_SIZE;
+ kernel_read(file, buf, size, &pos);
*decompressor = decompress_method(buf, size, &compress_name);
if (compress_name) {
@@ -126,8 +128,8 @@
/*
* Read 512 bytes further to check if cramfs is padded
*/
- ksys_lseek(fd, start_block * BLOCK_SIZE + 0x200, 0);
- ksys_read(fd, buf, size);
+ pos = start_block * BLOCK_SIZE + 0x200;
+ kernel_read(file, buf, size, &pos);
if (cramfsb->magic == CRAMFS_MAGIC) {
printk(KERN_NOTICE
@@ -140,8 +142,8 @@
/*
* Read block 1 to test for minix and ext2 superblock
*/
- ksys_lseek(fd, (start_block+1) * BLOCK_SIZE, 0);
- ksys_read(fd, buf, size);
+ pos = (start_block + 1) * BLOCK_SIZE;
+ kernel_read(file, buf, size, &pos);
/* Try minix */
if (minixsb->s_magic == MINIX_SUPER_MAGIC ||
@@ -168,17 +170,24 @@
start_block);
done:
- ksys_lseek(fd, start_block * BLOCK_SIZE, 0);
kfree(buf);
return nblocks;
}
+static unsigned long nr_blocks(struct file *file)
+{
+ struct inode *inode = file->f_mapping->host;
+
+ if (!S_ISBLK(inode->i_mode))
+ return 0;
+ return i_size_read(inode) >> 10;
+}
+
int __init rd_load_image(char *from)
{
int res = 0;
- int in_fd, out_fd;
unsigned long rd_blocks, devblocks;
- int nblocks, i, disk;
+ int nblocks, i;
char *buf = NULL;
unsigned short rotate = 0;
decompress_fn decompressor = NULL;
@@ -186,20 +195,21 @@
char rotator[4] = { '|' , '/' , '-' , '\\' };
#endif
- out_fd = ksys_open("/dev/ram", O_RDWR, 0);
- if (out_fd < 0)
+ out_file = filp_open("/dev/ram", O_RDWR, 0);
+ if (IS_ERR(out_file))
goto out;
- in_fd = ksys_open(from, O_RDONLY, 0);
- if (in_fd < 0)
+ in_file = filp_open(from, O_RDONLY, 0);
+ if (IS_ERR(in_file))
goto noclose_input;
- nblocks = identify_ramdisk_image(in_fd, rd_image_start, &decompressor);
+ in_pos = rd_image_start * BLOCK_SIZE;
+ nblocks = identify_ramdisk_image(in_file, in_pos, &decompressor);
if (nblocks < 0)
goto done;
if (nblocks == 0) {
- if (crd_load(in_fd, out_fd, decompressor) == 0)
+ if (crd_load(decompressor) == 0)
goto successful_load;
goto done;
}
@@ -208,11 +218,7 @@
* NOTE NOTE: nblocks is not actually blocks but
* the number of kibibytes of data to load into a ramdisk.
*/
- if (ksys_ioctl(out_fd, BLKGETSIZE, (unsigned long)&rd_blocks) < 0)
- rd_blocks = 0;
- else
- rd_blocks >>= 1;
-
+ rd_blocks = nr_blocks(out_file);
if (nblocks > rd_blocks) {
printk("RAMDISK: image too big! (%dKiB/%ldKiB)\n",
nblocks, rd_blocks);
@@ -222,13 +228,10 @@
/*
* OK, time to copy in the data
*/
- if (ksys_ioctl(in_fd, BLKGETSIZE, (unsigned long)&devblocks) < 0)
- devblocks = 0;
- else
- devblocks >>= 1;
-
if (strcmp(from, "/initrd.image") == 0)
devblocks = nblocks;
+ else
+ devblocks = nr_blocks(in_file);
if (devblocks == 0) {
printk(KERN_ERR "RAMDISK: could not determine device size\n");
@@ -243,24 +246,15 @@
printk(KERN_NOTICE "RAMDISK: Loading %dKiB [%ld disk%s] into ram disk... ",
nblocks, ((nblocks-1)/devblocks)+1, nblocks>devblocks ? "s" : "");
- for (i = 0, disk = 1; i < nblocks; i++) {
+ for (i = 0; i < nblocks; i++) {
if (i && (i % devblocks == 0)) {
- pr_cont("done disk #%d.\n", disk++);
+ pr_cont("done disk #1.\n");
rotate = 0;
- if (ksys_close(in_fd)) {
- printk("Error closing the disk.\n");
- goto noclose_input;
- }
- change_floppy("disk #%d", disk);
- in_fd = ksys_open(from, O_RDONLY, 0);
- if (in_fd < 0) {
- printk("Error opening disk.\n");
- goto noclose_input;
- }
- printk("Loading disk #%d... ", disk);
+ fput(in_file);
+ break;
}
- ksys_read(in_fd, buf, BLOCK_SIZE);
- ksys_write(out_fd, buf, BLOCK_SIZE);
+ kernel_read(in_file, buf, BLOCK_SIZE, &in_pos);
+ kernel_write(out_file, buf, BLOCK_SIZE, &out_pos);
#if !defined(CONFIG_S390)
if (!(i % 16)) {
pr_cont("%c\b", rotator[rotate & 0x3]);
@@ -273,19 +267,17 @@
successful_load:
res = 1;
done:
- ksys_close(in_fd);
+ fput(in_file);
noclose_input:
- ksys_close(out_fd);
+ fput(out_file);
out:
kfree(buf);
- ksys_unlink("/dev/ram");
+ init_unlink("/dev/ram");
return res;
}
int __init rd_load_disk(int n)
{
- if (rd_prompt)
- change_floppy("root floppy disk to be loaded into RAM disk");
create_dev("/dev/root", ROOT_DEV);
create_dev("/dev/ram", MKDEV(RAMDISK_MAJOR, n));
return rd_load_image("/dev/root");
@@ -293,11 +285,10 @@
static int exit_code;
static int decompress_error;
-static int crd_infd, crd_outfd;
static long __init compr_fill(void *buf, unsigned long len)
{
- long r = ksys_read(crd_infd, buf, len);
+ long r = kernel_read(in_file, buf, len, &in_pos);
if (r < 0)
printk(KERN_ERR "RAMDISK: error while reading compressed data");
else if (r == 0)
@@ -307,7 +298,7 @@
static long __init compr_flush(void *window, unsigned long outcnt)
{
- long written = ksys_write(crd_outfd, window, outcnt);
+ long written = kernel_write(out_file, window, outcnt, &out_pos);
if (written != outcnt) {
if (decompress_error == 0)
printk(KERN_ERR
@@ -326,11 +317,9 @@
decompress_error = 1;
}
-static int __init crd_load(int in_fd, int out_fd, decompress_fn deco)
+static int __init crd_load(decompress_fn deco)
{
int result;
- crd_infd = in_fd;
- crd_outfd = out_fd;
if (!deco) {
pr_emerg("Invalid ramdisk decompression routine. "
diff --git a/init/initramfs.c b/init/initramfs.c
index 7a38012..e6dbfb7 100644
--- a/init/initramfs.c
+++ b/init/initramfs.c
@@ -11,14 +11,17 @@
#include <linux/utime.h>
#include <linux/file.h>
#include <linux/memblock.h>
+#include <linux/namei.h>
+#include <linux/init_syscalls.h>
-static ssize_t __init xwrite(int fd, const char *p, size_t count)
+static ssize_t __init xwrite(struct file *file, const char *p, size_t count,
+ loff_t *pos)
{
ssize_t out = 0;
/* sys_write only can write MAX_RW_COUNT aka 2G-4K bytes at most */
while (count) {
- ssize_t rv = ksys_write(fd, p, count);
+ ssize_t rv = kernel_write(file, p, count, pos);
if (rv < 0) {
if (rv == -EINTR || rv == -EAGAIN)
@@ -108,8 +111,7 @@
t[0].tv_nsec = 0;
t[1].tv_sec = mtime;
t[1].tv_nsec = 0;
-
- return do_utimes(AT_FDCWD, filename, t, AT_SYMLINK_NOFOLLOW);
+ return init_utimes(filename, t);
}
static __initdata LIST_HEAD(dir_list);
@@ -200,7 +202,6 @@
byte_count -= n;
}
-static __initdata char *vcollected;
static __initdata char *collected;
static long remains __initdata;
static __initdata char *collect;
@@ -296,11 +297,12 @@
{
struct kstat st;
- if (!vfs_lstat(path, &st) && (st.mode ^ fmode) & S_IFMT) {
+ if (init_stat(path, &st, AT_SYMLINK_NOFOLLOW) &&
+ (st.mode ^ fmode) & S_IFMT) {
if (S_ISDIR(st.mode))
- ksys_rmdir(path);
+ init_rmdir(path);
else
- ksys_unlink(path);
+ init_unlink(path);
}
}
@@ -310,13 +312,14 @@
char *old = find_link(major, minor, ino, mode, collected);
if (old) {
clean_path(collected, 0);
- return (ksys_link(old, collected) < 0) ? -1 : 1;
+ return (init_link(old, collected) < 0) ? -1 : 1;
}
}
return 0;
}
-static __initdata int wfd;
+static __initdata struct file *wfile;
+static __initdata loff_t wfile_pos;
static int __init do_name(void)
{
@@ -333,28 +336,28 @@
int openflags = O_WRONLY|O_CREAT;
if (ml != 1)
openflags |= O_TRUNC;
- wfd = ksys_open(collected, openflags, mode);
+ wfile = filp_open(collected, openflags, mode);
+ if (IS_ERR(wfile))
+ return 0;
+ wfile_pos = 0;
- if (wfd >= 0) {
- ksys_fchown(wfd, uid, gid);
- ksys_fchmod(wfd, mode);
- if (body_len)
- ksys_ftruncate(wfd, body_len);
- vcollected = kstrdup(collected, GFP_KERNEL);
- state = CopyFile;
- }
+ vfs_fchown(wfile, uid, gid);
+ vfs_fchmod(wfile, mode);
+ if (body_len)
+ vfs_truncate(&wfile->f_path, body_len);
+ state = CopyFile;
}
} else if (S_ISDIR(mode)) {
- ksys_mkdir(collected, mode);
- ksys_chown(collected, uid, gid);
- ksys_chmod(collected, mode);
+ init_mkdir(collected, mode);
+ init_chown(collected, uid, gid, 0);
+ init_chmod(collected, mode);
dir_add(collected, mtime);
} else if (S_ISBLK(mode) || S_ISCHR(mode) ||
S_ISFIFO(mode) || S_ISSOCK(mode)) {
if (maybe_link() == 0) {
- ksys_mknod(collected, mode, rdev);
- ksys_chown(collected, uid, gid);
- ksys_chmod(collected, mode);
+ init_mknod(collected, mode, rdev);
+ init_chown(collected, uid, gid, 0);
+ init_chmod(collected, mode);
do_utime(collected, mtime);
}
}
@@ -364,16 +367,20 @@
static int __init do_copy(void)
{
if (byte_count >= body_len) {
- if (xwrite(wfd, victim, body_len) != body_len)
+ struct timespec64 t[2] = { };
+ if (xwrite(wfile, victim, body_len, &wfile_pos) != body_len)
error("write error");
- ksys_close(wfd);
- do_utime(vcollected, mtime);
- kfree(vcollected);
+
+ t[0].tv_sec = mtime;
+ t[1].tv_sec = mtime;
+ vfs_utimes(&wfile->f_path, t);
+
+ fput(wfile);
eat(body_len);
state = SkipIt;
return 0;
} else {
- if (xwrite(wfd, victim, byte_count) != byte_count)
+ if (xwrite(wfile, victim, byte_count, &wfile_pos) != byte_count)
error("write error");
body_len -= byte_count;
eat(byte_count);
@@ -385,8 +392,8 @@
{
collected[N_ALIGN(name_len) + body_len] = '\0';
clean_path(collected, 0);
- ksys_symlink(collected + N_ALIGN(name_len), collected);
- ksys_lchown(collected, uid, gid);
+ init_symlink(collected + N_ALIGN(name_len), collected);
+ init_chown(collected, uid, gid, AT_SYMLINK_NOFOLLOW);
do_utime(collected, mtime);
state = SkipIt;
next_state = Reset;
@@ -572,82 +579,26 @@
#endif /* CONFIG_KEXEC_CORE */
#ifdef CONFIG_BLK_DEV_RAM
-#define BUF_SIZE 1024
-static void __init clean_rootfs(void)
-{
- int fd;
- void *buf;
- struct linux_dirent64 *dirp;
- int num;
-
- fd = ksys_open("/", O_RDONLY, 0);
- WARN_ON(fd < 0);
- if (fd < 0)
- return;
- buf = kzalloc(BUF_SIZE, GFP_KERNEL);
- WARN_ON(!buf);
- if (!buf) {
- ksys_close(fd);
- return;
- }
-
- dirp = buf;
- num = ksys_getdents64(fd, dirp, BUF_SIZE);
- while (num > 0) {
- while (num > 0) {
- struct kstat st;
- int ret;
-
- ret = vfs_lstat(dirp->d_name, &st);
- WARN_ON_ONCE(ret);
- if (!ret) {
- if (S_ISDIR(st.mode))
- ksys_rmdir(dirp->d_name);
- else
- ksys_unlink(dirp->d_name);
- }
-
- num -= dirp->d_reclen;
- dirp = (void *)dirp + dirp->d_reclen;
- }
- dirp = buf;
- memset(buf, 0, BUF_SIZE);
- num = ksys_getdents64(fd, dirp, BUF_SIZE);
- }
-
- ksys_close(fd);
- kfree(buf);
-}
-#else
-static inline void clean_rootfs(void)
-{
-}
-#endif /* CONFIG_BLK_DEV_RAM */
-
-#ifdef CONFIG_BLK_DEV_RAM
static void __init populate_initrd_image(char *err)
{
ssize_t written;
- int fd;
+ struct file *file;
+ loff_t pos = 0;
unpack_to_rootfs(__initramfs_start, __initramfs_size);
printk(KERN_INFO "rootfs image is not initramfs (%s); looks like an initrd\n",
err);
- fd = ksys_open("/initrd.image", O_WRONLY | O_CREAT, 0700);
- if (fd < 0)
+ file = filp_open("/initrd.image", O_WRONLY | O_CREAT, 0700);
+ if (IS_ERR(file))
return;
- written = xwrite(fd, (char *)initrd_start, initrd_end - initrd_start);
+ written = xwrite(file, (char *)initrd_start, initrd_end - initrd_start,
+ &pos);
if (written != initrd_end - initrd_start)
pr_err("/initrd.image: incomplete write (%zd != %ld)\n",
written, initrd_end - initrd_start);
- ksys_close(fd);
-}
-#else
-static void __init populate_initrd_image(char *err)
-{
- printk(KERN_EMERG "Initramfs unpacking failed: %s\n", err);
+ fput(file);
}
#endif /* CONFIG_BLK_DEV_RAM */
@@ -668,8 +619,11 @@
err = unpack_to_rootfs((char *)initrd_start, initrd_end - initrd_start);
if (err) {
- clean_rootfs();
+#ifdef CONFIG_BLK_DEV_RAM
populate_initrd_image(err);
+#else
+ printk(KERN_EMERG "Initramfs unpacking failed: %s\n", err);
+#endif
}
done:
diff --git a/init/main.c b/init/main.c
index 92d2579..de2f9fa 100644
--- a/init/main.c
+++ b/init/main.c
@@ -96,6 +96,7 @@
#include <linux/jump_label.h>
#include <linux/mem_encrypt.h>
#include <linux/kcsan.h>
+#include <linux/init_syscalls.h>
#include <asm/io.h>
#include <asm/bugs.h>
@@ -154,7 +155,7 @@
#endif
static char *execute_command;
-static char *ramdisk_execute_command;
+static char *ramdisk_execute_command = "/init";
/*
* Used to generate warnings if static_key manipulation functions are used
@@ -1457,15 +1458,19 @@
"See Linux Documentation/admin-guide/init.rst for guidance.");
}
-void console_on_rootfs(void)
+/* Open /dev/console, for stdin/stdout/stderr, this should never fail */
+void __init console_on_rootfs(void)
{
- /* Open the /dev/console as stdin, this should never fail */
- if (ksys_open((const char __user *) "/dev/console", O_RDWR, 0) < 0)
- pr_err("Warning: unable to open an initial console.\n");
+ struct file *file = filp_open("/dev/console", O_RDWR, 0);
- /* create stdout/stderr */
- (void) ksys_dup(0);
- (void) ksys_dup(0);
+ if (IS_ERR(file)) {
+ pr_err("Warning: unable to open an initial console.\n");
+ return;
+ }
+ init_dup(file);
+ init_dup(file);
+ init_dup(file);
+ fput(file);
}
static noinline void __init kernel_init_freeable(void)
@@ -1510,12 +1515,7 @@
* check if there is an early userspace init. If yes, let it do all
* the work
*/
-
- if (!ramdisk_execute_command)
- ramdisk_execute_command = "/init";
-
- if (ksys_access((const char __user *)
- ramdisk_execute_command, 0) != 0) {
+ if (init_eaccess(ramdisk_execute_command) != 0) {
ramdisk_execute_command = NULL;
prepare_namespace();
}
diff --git a/init/noinitramfs.c b/init/noinitramfs.c
index fa9cdfa..3d62b07 100644
--- a/init/noinitramfs.c
+++ b/init/noinitramfs.c
@@ -9,6 +9,7 @@
#include <linux/stat.h>
#include <linux/kdev_t.h>
#include <linux/syscalls.h>
+#include <linux/init_syscalls.h>
/*
* Create a simple rootfs that is similar to the default initramfs
@@ -17,17 +18,16 @@
{
int err;
- err = ksys_mkdir((const char __user __force *) "/dev", 0755);
+ err = init_mkdir("/dev", 0755);
if (err < 0)
goto out;
- err = ksys_mknod((const char __user __force *) "/dev/console",
- S_IFCHR | S_IRUSR | S_IWUSR,
+ err = init_mknod("/dev/console", S_IFCHR | S_IRUSR | S_IWUSR,
new_encode_dev(MKDEV(5, 1)));
if (err < 0)
goto out;
- err = ksys_mkdir((const char __user __force *) "/root", 0700);
+ err = init_mkdir("/root", 0700);
if (err < 0)
goto out;
diff --git a/kernel/Makefile b/kernel/Makefile
index bdeb77e..5350fd2 100644
--- a/kernel/Makefile
+++ b/kernel/Makefile
@@ -10,7 +10,7 @@
extable.o params.o \
kthread.o sys_ni.o nsproxy.o \
notifier.o ksysfs.o cred.o reboot.o \
- async.o range.o smpboot.o ucount.o
+ async.o range.o smpboot.o ucount.o regset.o
obj-$(CONFIG_BPFILTER) += usermode_driver.o
obj-$(CONFIG_MODULES) += kmod.o
diff --git a/kernel/audit_fsnotify.c b/kernel/audit_fsnotify.c
index 3596448..bfcfcd6 100644
--- a/kernel/audit_fsnotify.c
+++ b/kernel/audit_fsnotify.c
@@ -36,7 +36,7 @@
/* fsnotify events we care about. */
#define AUDIT_FS_EVENTS (FS_MOVE | FS_CREATE | FS_DELETE | FS_DELETE_SELF |\
- FS_MOVE_SELF | FS_EVENT_ON_CHILD)
+ FS_MOVE_SELF)
static void audit_fsnotify_mark_free(struct audit_fsnotify_mark *audit_mark)
{
@@ -152,35 +152,31 @@
}
/* Update mark data in audit rules based on fsnotify events. */
-static int audit_mark_handle_event(struct fsnotify_group *group,
- struct inode *to_tell,
- u32 mask, const void *data, int data_type,
- const struct qstr *dname, u32 cookie,
- struct fsnotify_iter_info *iter_info)
+static int audit_mark_handle_event(struct fsnotify_mark *inode_mark, u32 mask,
+ struct inode *inode, struct inode *dir,
+ const struct qstr *dname)
{
- struct fsnotify_mark *inode_mark = fsnotify_iter_inode_mark(iter_info);
struct audit_fsnotify_mark *audit_mark;
- const struct inode *inode = fsnotify_data_inode(data, data_type);
audit_mark = container_of(inode_mark, struct audit_fsnotify_mark, mark);
- BUG_ON(group != audit_fsnotify_group);
-
- if (WARN_ON(!inode))
+ if (WARN_ON_ONCE(inode_mark->group != audit_fsnotify_group) ||
+ WARN_ON_ONCE(!inode))
return 0;
if (mask & (FS_CREATE|FS_MOVED_TO|FS_DELETE|FS_MOVED_FROM)) {
if (audit_compare_dname_path(dname, audit_mark->path, AUDIT_NAME_FULL))
return 0;
audit_update_mark(audit_mark, inode);
- } else if (mask & (FS_DELETE_SELF|FS_UNMOUNT|FS_MOVE_SELF))
+ } else if (mask & (FS_DELETE_SELF|FS_UNMOUNT|FS_MOVE_SELF)) {
audit_autoremove_mark_rule(audit_mark);
+ }
return 0;
}
static const struct fsnotify_ops audit_mark_fsnotify_ops = {
- .handle_event = audit_mark_handle_event,
+ .handle_inode_event = audit_mark_handle_event,
.free_mark = audit_fsnotify_free_mark,
};
diff --git a/kernel/audit_tree.c b/kernel/audit_tree.c
index 1b7a2f0..83e1c07 100644
--- a/kernel/audit_tree.c
+++ b/kernel/audit_tree.c
@@ -1035,11 +1035,9 @@
audit_schedule_prune();
}
-static int audit_tree_handle_event(struct fsnotify_group *group,
- struct inode *to_tell,
- u32 mask, const void *data, int data_type,
- const struct qstr *file_name, u32 cookie,
- struct fsnotify_iter_info *iter_info)
+static int audit_tree_handle_event(struct fsnotify_mark *mark, u32 mask,
+ struct inode *inode, struct inode *dir,
+ const struct qstr *file_name)
{
return 0;
}
@@ -1068,7 +1066,7 @@
}
static const struct fsnotify_ops audit_tree_ops = {
- .handle_event = audit_tree_handle_event,
+ .handle_inode_event = audit_tree_handle_event,
.freeing_mark = audit_tree_freeing_mark,
.free_mark = audit_tree_destroy_watch,
};
diff --git a/kernel/audit_watch.c b/kernel/audit_watch.c
index e09c551..246e5ba 100644
--- a/kernel/audit_watch.c
+++ b/kernel/audit_watch.c
@@ -53,7 +53,7 @@
/* fsnotify events we care about. */
#define AUDIT_FS_WATCH (FS_MOVE | FS_CREATE | FS_DELETE | FS_DELETE_SELF |\
- FS_MOVE_SELF | FS_EVENT_ON_CHILD | FS_UNMOUNT)
+ FS_MOVE_SELF | FS_UNMOUNT)
static void audit_free_parent(struct audit_parent *parent)
{
@@ -464,20 +464,17 @@
}
/* Update watch data in audit rules based on fsnotify events. */
-static int audit_watch_handle_event(struct fsnotify_group *group,
- struct inode *to_tell,
- u32 mask, const void *data, int data_type,
- const struct qstr *dname, u32 cookie,
- struct fsnotify_iter_info *iter_info)
+static int audit_watch_handle_event(struct fsnotify_mark *inode_mark, u32 mask,
+ struct inode *inode, struct inode *dir,
+ const struct qstr *dname)
{
- struct fsnotify_mark *inode_mark = fsnotify_iter_inode_mark(iter_info);
- const struct inode *inode = fsnotify_data_inode(data, data_type);
struct audit_parent *parent;
parent = container_of(inode_mark, struct audit_parent, mark);
- BUG_ON(group != audit_watch_group);
- WARN_ON(!inode);
+ if (WARN_ON_ONCE(inode_mark->group != audit_watch_group) ||
+ WARN_ON_ONCE(!inode))
+ return 0;
if (mask & (FS_CREATE|FS_MOVED_TO) && inode)
audit_update_watch(parent, dname, inode->i_sb->s_dev, inode->i_ino, 0);
@@ -490,7 +487,7 @@
}
static const struct fsnotify_ops audit_watch_fsnotify_ops = {
- .handle_event = audit_watch_handle_event,
+ .handle_inode_event = audit_watch_handle_event,
.free_mark = audit_watch_free_mark,
};
diff --git a/kernel/kexec_file.c b/kernel/kexec_file.c
index 94661d2..78c0837 100644
--- a/kernel/kexec_file.c
+++ b/kernel/kexec_file.c
@@ -636,6 +636,19 @@
}
/**
+ * arch_kexec_locate_mem_hole - Find free memory to place the segments.
+ * @kbuf: Parameters for the memory search.
+ *
+ * On success, kbuf->mem will have the start address of the memory region found.
+ *
+ * Return: 0 on success, negative errno on error.
+ */
+int __weak arch_kexec_locate_mem_hole(struct kexec_buf *kbuf)
+{
+ return kexec_locate_mem_hole(kbuf);
+}
+
+/**
* kexec_add_buffer - place a buffer in a kexec segment
* @kbuf: Buffer contents and memory parameters.
*
@@ -647,7 +660,6 @@
*/
int kexec_add_buffer(struct kexec_buf *kbuf)
{
-
struct kexec_segment *ksegment;
int ret;
@@ -675,7 +687,7 @@
kbuf->buf_align = max(kbuf->buf_align, PAGE_SIZE);
/* Walk the RAM ranges and allocate a suitable range for the buffer */
- ret = kexec_locate_mem_hole(kbuf);
+ ret = arch_kexec_locate_mem_hole(kbuf);
if (ret)
return ret;
diff --git a/kernel/regset.c b/kernel/regset.c
new file mode 100644
index 0000000..5868237
--- /dev/null
+++ b/kernel/regset.c
@@ -0,0 +1,76 @@
+// SPDX-License-Identifier: GPL-2.0-only
+#include <linux/export.h>
+#include <linux/slab.h>
+#include <linux/regset.h>
+
+static int __regset_get(struct task_struct *target,
+ const struct user_regset *regset,
+ unsigned int size,
+ void **data)
+{
+ void *p = *data, *to_free = NULL;
+ int res;
+
+ if (!regset->regset_get)
+ return -EOPNOTSUPP;
+ if (size > regset->n * regset->size)
+ size = regset->n * regset->size;
+ if (!p) {
+ to_free = p = kzalloc(size, GFP_KERNEL);
+ if (!p)
+ return -ENOMEM;
+ }
+ res = regset->regset_get(target, regset,
+ (struct membuf){.p = p, .left = size});
+ if (res < 0) {
+ kfree(to_free);
+ return res;
+ }
+ *data = p;
+ return size - res;
+}
+
+int regset_get(struct task_struct *target,
+ const struct user_regset *regset,
+ unsigned int size,
+ void *data)
+{
+ return __regset_get(target, regset, size, &data);
+}
+EXPORT_SYMBOL(regset_get);
+
+int regset_get_alloc(struct task_struct *target,
+ const struct user_regset *regset,
+ unsigned int size,
+ void **data)
+{
+ *data = NULL;
+ return __regset_get(target, regset, size, data);
+}
+EXPORT_SYMBOL(regset_get_alloc);
+
+/**
+ * copy_regset_to_user - fetch a thread's user_regset data into user memory
+ * @target: thread to be examined
+ * @view: &struct user_regset_view describing user thread machine state
+ * @setno: index in @view->regsets
+ * @offset: offset into the regset data, in bytes
+ * @size: amount of data to copy, in bytes
+ * @data: user-mode pointer to copy into
+ */
+int copy_regset_to_user(struct task_struct *target,
+ const struct user_regset_view *view,
+ unsigned int setno,
+ unsigned int offset, unsigned int size,
+ void __user *data)
+{
+ const struct user_regset *regset = &view->regsets[setno];
+ void *buf;
+ int ret;
+
+ ret = regset_get_alloc(target, regset, size, &buf);
+ if (ret > 0)
+ ret = copy_to_user(data, buf, ret) ? -EFAULT : 0;
+ kfree(buf);
+ return ret;
+}
diff --git a/kernel/trace/trace.c b/kernel/trace/trace.c
index 848f67a..b0c338ec 100644
--- a/kernel/trace/trace.c
+++ b/kernel/trace/trace.c
@@ -1543,8 +1543,7 @@
{
struct trace_array *tr = container_of(work, struct trace_array,
fsnotify_work);
- fsnotify(tr->d_max_latency->d_inode, FS_MODIFY,
- tr->d_max_latency->d_inode, FSNOTIFY_EVENT_INODE, NULL, 0);
+ fsnotify_inode(tr->d_max_latency->d_inode, FS_MODIFY);
}
static void latency_fsnotify_workfn_irq(struct irq_work *iwork)
diff --git a/mm/ksm.c b/mm/ksm.c
index 5fb176d..217842a 100644
--- a/mm/ksm.c
+++ b/mm/ksm.c
@@ -2452,10 +2452,6 @@
if (vma_is_dax(vma))
return 0;
-#ifdef VM_SAO
- if (*vm_flags & VM_SAO)
- return 0;
-#endif
#ifdef VM_SPARC_ADI
if (*vm_flags & VM_SPARC_ADI)
return 0;
diff --git a/net/core/sock.c b/net/core/sock.c
index 9d8b15b..a2044b4 100644
--- a/net/core/sock.c
+++ b/net/core/sock.c
@@ -820,6 +820,14 @@
}
EXPORT_SYMBOL(sock_set_rcvbuf);
+void sock_set_mark(struct sock *sk, u32 val)
+{
+ lock_sock(sk);
+ sk->sk_mark = val;
+ release_sock(sk);
+}
+EXPORT_SYMBOL(sock_set_mark);
+
/*
* This is meant for all protocols to use and covers goings on
* at the socket level. Everything here is generic.
diff --git a/net/netlink/genetlink.c b/net/netlink/genetlink.c
index 9395ee8..1eb65a7 100644
--- a/net/netlink/genetlink.c
+++ b/net/netlink/genetlink.c
@@ -1192,7 +1192,7 @@
panic("GENL: Cannot register controller: %d\n", err);
}
-subsys_initcall(genl_init);
+core_initcall(genl_init);
static int genlmsg_mcast(struct sk_buff *skb, u32 portid, unsigned long group,
gfp_t flags)
diff --git a/tools/arch/powerpc/include/uapi/asm/kvm.h b/tools/arch/powerpc/include/uapi/asm/kvm.h
index 264e266..c3af3f3 100644
--- a/tools/arch/powerpc/include/uapi/asm/kvm.h
+++ b/tools/arch/powerpc/include/uapi/asm/kvm.h
@@ -640,6 +640,11 @@
#define KVM_REG_PPC_ONLINE (KVM_REG_PPC | KVM_REG_SIZE_U32 | 0xbf)
#define KVM_REG_PPC_PTCR (KVM_REG_PPC | KVM_REG_SIZE_U64 | 0xc0)
+/* POWER10 registers */
+#define KVM_REG_PPC_MMCR3 (KVM_REG_PPC | KVM_REG_SIZE_U64 | 0xc1)
+#define KVM_REG_PPC_SIER2 (KVM_REG_PPC | KVM_REG_SIZE_U64 | 0xc2)
+#define KVM_REG_PPC_SIER3 (KVM_REG_PPC | KVM_REG_SIZE_U64 | 0xc3)
+
/* Transactional Memory checkpointed state:
* This is all GPRs, all VSX regs and a subset of SPRs
*/
diff --git a/tools/arch/riscv/include/uapi/asm/unistd.h b/tools/arch/riscv/include/uapi/asm/unistd.h
index 0e2eeeb..f506cca 100644
--- a/tools/arch/riscv/include/uapi/asm/unistd.h
+++ b/tools/arch/riscv/include/uapi/asm/unistd.h
@@ -12,7 +12,7 @@
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
#ifdef __LP64__
diff --git a/tools/perf/arch/powerpc/util/book3s_hcalls.h b/tools/perf/arch/powerpc/util/book3s_hcalls.h
index 54cfa05..488f433 100644
--- a/tools/perf/arch/powerpc/util/book3s_hcalls.h
+++ b/tools/perf/arch/powerpc/util/book3s_hcalls.h
@@ -84,7 +84,7 @@
{0x1a4, "H_CREATE_RPT"}, \
{0x1a8, "H_REMOVE_RPT"}, \
{0x1ac, "H_REGISTER_RPAGES"}, \
- {0x1b0, "H_DISABLE_AND_GETC"}, \
+ {0x1b0, "H_DISABLE_AND_GET"}, \
{0x1b4, "H_ERROR_DATA"}, \
{0x1b8, "H_GET_HCA_INFO"}, \
{0x1bc, "H_GET_PERF_COUNT"}, \
diff --git a/tools/testing/selftests/powerpc/alignment/alignment_handler.c b/tools/testing/selftests/powerpc/alignment/alignment_handler.c
index 0453c50..55ef151 100644
--- a/tools/testing/selftests/powerpc/alignment/alignment_handler.c
+++ b/tools/testing/selftests/powerpc/alignment/alignment_handler.c
@@ -9,7 +9,17 @@
* This selftest exercises the powerpc alignment fault handler.
*
* We create two sets of source and destination buffers, one in regular memory,
- * the other cache-inhibited (we use /dev/fb0 for this).
+ * the other cache-inhibited (by default we use /dev/fb0 for this, but an
+ * alterative path for cache-inhibited memory may be provided).
+ *
+ * One way to get cache-inhibited memory is to use the "mem" kernel parameter
+ * to limit the kernel to less memory than actually exists. Addresses above
+ * the limit may still be accessed but will be treated as cache-inhibited. For
+ * example, if there is actually 4GB of memory and the parameter "mem=3GB" is
+ * used, memory from address 0xC0000000 onwards is treated as cache-inhibited.
+ * To access this region /dev/mem is used. The kernel should be configured
+ * without CONFIG_STRICT_DEVMEM. In this case use:
+ * ./alignment_handler /dev/mem 0xc0000000
*
* We initialise the source buffers, then use whichever set of load/store
* instructions is under test to copy bytes from the source buffers to the
@@ -48,11 +58,14 @@
#include <asm/cputable.h>
#include "utils.h"
+#include "instructions.h"
int bufsize;
int debug;
int testing;
volatile int gotsig;
+char *cipath = "/dev/fb0";
+long cioffset;
void sighandler(int sig, siginfo_t *info, void *ctx)
{
@@ -84,6 +97,17 @@
} \
rc |= do_test(#name, test_##name)
+#define TESTP(name, ld_op, st_op, ld_reg, st_reg) \
+ void test_##name(char *s, char *d) \
+ { \
+ asm volatile( \
+ ld_op(ld_reg, %0, 0, 0) \
+ st_op(st_reg, %1, 0, 0) \
+ :: "r"(s), "r"(d), "r"(0) \
+ : "memory", "vs0", "vs32", "r31"); \
+ } \
+ rc |= do_test(#name, test_##name)
+
#define LOAD_VSX_XFORM_TEST(op) TEST(op, op, stxvd2x, XFORM, 32, 32)
#define STORE_VSX_XFORM_TEST(op) TEST(op, lxvd2x, op, XFORM, 32, 32)
#define LOAD_VSX_DFORM_TEST(op) TEST(op, op, stxv, DFORM, 32, 32)
@@ -103,6 +127,17 @@
#define LOAD_FLOAT_XFORM_TEST(op) TEST(op, op, stfdx, XFORM, 0, 0)
#define STORE_FLOAT_XFORM_TEST(op) TEST(op, lfdx, op, XFORM, 0, 0)
+#define LOAD_MLS_PREFIX_TEST(op) TESTP(op, op, PSTD, 31, 31)
+#define STORE_MLS_PREFIX_TEST(op) TESTP(op, PLD, op, 31, 31)
+
+#define LOAD_8LS_PREFIX_TEST(op) TESTP(op, op, PSTD, 31, 31)
+#define STORE_8LS_PREFIX_TEST(op) TESTP(op, PLD, op, 31, 31)
+
+#define LOAD_FLOAT_MLS_PREFIX_TEST(op) TESTP(op, op, PSTFD, 0, 0)
+#define STORE_FLOAT_MLS_PREFIX_TEST(op) TESTP(op, PLFD, op, 0, 0)
+
+#define LOAD_VSX_8LS_PREFIX_TEST(op, tail) TESTP(op, op, PSTXV ## tail, 0, 32)
+#define STORE_VSX_8LS_PREFIX_TEST(op, tail) TESTP(op, PLXV ## tail, op, 32, 0)
/* FIXME: Unimplemented tests: */
// STORE_DFORM_TEST(stq) /* FIXME: need two registers for quad */
@@ -195,17 +230,18 @@
printf("\tDoing %s:\t", test_name);
- fd = open("/dev/fb0", O_RDWR);
+ fd = open(cipath, O_RDWR);
if (fd < 0) {
printf("\n");
- perror("Can't open /dev/fb0 now?");
+ perror("Can't open ci file now?");
return 1;
}
- ci0 = mmap(NULL, bufsize, PROT_WRITE, MAP_SHARED,
- fd, 0x0);
- ci1 = mmap(NULL, bufsize, PROT_WRITE, MAP_SHARED,
- fd, bufsize);
+ ci0 = mmap(NULL, bufsize, PROT_WRITE | PROT_READ, MAP_SHARED,
+ fd, cioffset);
+ ci1 = mmap(NULL, bufsize, PROT_WRITE | PROT_READ, MAP_SHARED,
+ fd, cioffset + bufsize);
+
if ((ci0 == MAP_FAILED) || (ci1 == MAP_FAILED)) {
printf("\n");
perror("mmap failed");
@@ -270,11 +306,11 @@
return rc;
}
-static bool can_open_fb0(void)
+static bool can_open_cifile(void)
{
int fd;
- fd = open("/dev/fb0", O_RDWR);
+ fd = open(cipath, O_RDWR);
if (fd < 0)
return false;
@@ -286,7 +322,7 @@
{
int rc = 0;
- SKIP_IF(!can_open_fb0());
+ SKIP_IF(!can_open_cifile());
SKIP_IF(!have_hwcap(PPC_FEATURE_ARCH_2_06));
printf("VSX: 2.06B\n");
@@ -304,7 +340,7 @@
{
int rc = 0;
- SKIP_IF(!can_open_fb0());
+ SKIP_IF(!can_open_cifile());
SKIP_IF(!have_hwcap2(PPC_FEATURE2_ARCH_2_07));
printf("VSX: 2.07B\n");
@@ -320,7 +356,7 @@
{
int rc = 0;
- SKIP_IF(!can_open_fb0());
+ SKIP_IF(!can_open_cifile());
SKIP_IF(!have_hwcap2(PPC_FEATURE2_ARCH_3_00));
printf("VSX: 3.00B\n");
@@ -348,11 +384,30 @@
return rc;
}
+int test_alignment_handler_vsx_prefix(void)
+{
+ int rc = 0;
+
+ SKIP_IF(!can_open_cifile());
+ SKIP_IF(!have_hwcap2(PPC_FEATURE2_ARCH_3_1));
+
+ printf("VSX: PREFIX\n");
+ LOAD_VSX_8LS_PREFIX_TEST(PLXSD, 0);
+ LOAD_VSX_8LS_PREFIX_TEST(PLXSSP, 0);
+ LOAD_VSX_8LS_PREFIX_TEST(PLXV0, 0);
+ LOAD_VSX_8LS_PREFIX_TEST(PLXV1, 1);
+ STORE_VSX_8LS_PREFIX_TEST(PSTXSD, 0);
+ STORE_VSX_8LS_PREFIX_TEST(PSTXSSP, 0);
+ STORE_VSX_8LS_PREFIX_TEST(PSTXV0, 0);
+ STORE_VSX_8LS_PREFIX_TEST(PSTXV1, 1);
+ return rc;
+}
+
int test_alignment_handler_integer(void)
{
int rc = 0;
- SKIP_IF(!can_open_fb0());
+ SKIP_IF(!can_open_cifile());
printf("Integer\n");
LOAD_DFORM_TEST(lbz);
@@ -408,7 +463,7 @@
{
int rc = 0;
- SKIP_IF(!can_open_fb0());
+ SKIP_IF(!can_open_cifile());
SKIP_IF(!have_hwcap(PPC_FEATURE_ARCH_2_06));
printf("Integer: 2.06\n");
@@ -419,11 +474,32 @@
return rc;
}
+int test_alignment_handler_integer_prefix(void)
+{
+ int rc = 0;
+
+ SKIP_IF(!can_open_cifile());
+ SKIP_IF(!have_hwcap2(PPC_FEATURE2_ARCH_3_1));
+
+ printf("Integer: PREFIX\n");
+ LOAD_MLS_PREFIX_TEST(PLBZ);
+ LOAD_MLS_PREFIX_TEST(PLHZ);
+ LOAD_MLS_PREFIX_TEST(PLHA);
+ LOAD_MLS_PREFIX_TEST(PLWZ);
+ LOAD_8LS_PREFIX_TEST(PLWA);
+ LOAD_8LS_PREFIX_TEST(PLD);
+ STORE_MLS_PREFIX_TEST(PSTB);
+ STORE_MLS_PREFIX_TEST(PSTH);
+ STORE_MLS_PREFIX_TEST(PSTW);
+ STORE_8LS_PREFIX_TEST(PSTD);
+ return rc;
+}
+
int test_alignment_handler_vmx(void)
{
int rc = 0;
- SKIP_IF(!can_open_fb0());
+ SKIP_IF(!can_open_cifile());
SKIP_IF(!have_hwcap(PPC_FEATURE_HAS_ALTIVEC));
printf("VMX\n");
@@ -451,7 +527,7 @@
{
int rc = 0;
- SKIP_IF(!can_open_fb0());
+ SKIP_IF(!can_open_cifile());
printf("Floating point\n");
LOAD_FLOAT_DFORM_TEST(lfd);
@@ -479,7 +555,7 @@
{
int rc = 0;
- SKIP_IF(!can_open_fb0());
+ SKIP_IF(!can_open_cifile());
SKIP_IF(!have_hwcap(PPC_FEATURE_ARCH_2_05));
printf("Floating point: 2.05\n");
@@ -497,7 +573,7 @@
{
int rc = 0;
- SKIP_IF(!can_open_fb0());
+ SKIP_IF(!can_open_cifile());
SKIP_IF(!have_hwcap(PPC_FEATURE_ARCH_2_06));
printf("Floating point: 2.06\n");
@@ -507,13 +583,32 @@
return rc;
}
+
+int test_alignment_handler_fp_prefix(void)
+{
+ int rc = 0;
+
+ SKIP_IF(!can_open_cifile());
+ SKIP_IF(!have_hwcap2(PPC_FEATURE2_ARCH_3_1));
+
+ printf("Floating point: PREFIX\n");
+ LOAD_FLOAT_DFORM_TEST(lfs);
+ LOAD_FLOAT_MLS_PREFIX_TEST(PLFS);
+ LOAD_FLOAT_MLS_PREFIX_TEST(PLFD);
+ STORE_FLOAT_MLS_PREFIX_TEST(PSTFS);
+ STORE_FLOAT_MLS_PREFIX_TEST(PSTFD);
+ return rc;
+}
+
void usage(char *prog)
{
- printf("Usage: %s [options]\n", prog);
+ printf("Usage: %s [options] [path [offset]]\n", prog);
printf(" -d Enable debug error output\n");
printf("\n");
- printf("This test requires a POWER8 or POWER9 CPU and a usable ");
- printf("framebuffer at /dev/fb0.\n");
+ printf("This test requires a POWER8, POWER9 or POWER10 CPU ");
+ printf("and either a usable framebuffer at /dev/fb0 or ");
+ printf("the path to usable cache inhibited memory and optional ");
+ printf("offset to be provided\n");
}
int main(int argc, char *argv[])
@@ -533,6 +628,13 @@
exit(1);
}
}
+ argc -= optind;
+ argv += optind;
+
+ if (argc > 0)
+ cipath = argv[0];
+ if (argc > 1)
+ cioffset = strtol(argv[1], 0, 0x10);
bufsize = getpagesize();
@@ -552,10 +654,14 @@
"test_alignment_handler_vsx_207");
rc |= test_harness(test_alignment_handler_vsx_300,
"test_alignment_handler_vsx_300");
+ rc |= test_harness(test_alignment_handler_vsx_prefix,
+ "test_alignment_handler_vsx_prefix");
rc |= test_harness(test_alignment_handler_integer,
"test_alignment_handler_integer");
rc |= test_harness(test_alignment_handler_integer_206,
"test_alignment_handler_integer_206");
+ rc |= test_harness(test_alignment_handler_integer_prefix,
+ "test_alignment_handler_integer_prefix");
rc |= test_harness(test_alignment_handler_vmx,
"test_alignment_handler_vmx");
rc |= test_harness(test_alignment_handler_fp,
@@ -564,5 +670,7 @@
"test_alignment_handler_fp_205");
rc |= test_harness(test_alignment_handler_fp_206,
"test_alignment_handler_fp_206");
+ rc |= test_harness(test_alignment_handler_fp_prefix,
+ "test_alignment_handler_fp_prefix");
return rc;
}
diff --git a/tools/testing/selftests/powerpc/benchmarks/context_switch.c b/tools/testing/selftests/powerpc/benchmarks/context_switch.c
index a2e8c9d..d50cc05 100644
--- a/tools/testing/selftests/powerpc/benchmarks/context_switch.c
+++ b/tools/testing/selftests/powerpc/benchmarks/context_switch.c
@@ -19,6 +19,7 @@
#include <limits.h>
#include <sys/time.h>
#include <sys/syscall.h>
+#include <sys/sysinfo.h>
#include <sys/types.h>
#include <sys/shm.h>
#include <linux/futex.h>
@@ -104,8 +105,9 @@
static void start_process_on(void *(*fn)(void *), void *arg, unsigned long cpu)
{
- int pid;
- cpu_set_t cpuset;
+ int pid, ncpus;
+ cpu_set_t *cpuset;
+ size_t size;
pid = fork();
if (pid == -1) {
@@ -116,14 +118,23 @@
if (pid)
return;
- CPU_ZERO(&cpuset);
- CPU_SET(cpu, &cpuset);
+ ncpus = get_nprocs();
+ size = CPU_ALLOC_SIZE(ncpus);
+ cpuset = CPU_ALLOC(ncpus);
+ if (!cpuset) {
+ perror("malloc");
+ exit(1);
+ }
+ CPU_ZERO_S(size, cpuset);
+ CPU_SET_S(cpu, size, cpuset);
- if (sched_setaffinity(0, sizeof(cpuset), &cpuset)) {
+ if (sched_setaffinity(0, size, cpuset)) {
perror("sched_setaffinity");
+ CPU_FREE(cpuset);
exit(1);
}
+ CPU_FREE(cpuset);
fn(arg);
exit(0);
diff --git a/tools/testing/selftests/powerpc/eeh/eeh-functions.sh b/tools/testing/selftests/powerpc/eeh/eeh-functions.sh
index f52ed92..00dc32c 100755
--- a/tools/testing/selftests/powerpc/eeh/eeh-functions.sh
+++ b/tools/testing/selftests/powerpc/eeh/eeh-functions.sh
@@ -5,12 +5,17 @@
local dev="$1"
local path="/sys/bus/pci/devices/$dev/eeh_pe_state"
- if ! [ -e "$path" ] ; then
+ # if a driver doesn't support the error handling callbacks then the
+ # device is recovered by removing and re-probing it. This causes the
+ # sysfs directory to disappear so read the PE state once and squash
+ # any potential error messages
+ local eeh_state="$(cat $path 2>/dev/null)"
+ if [ -z "$eeh_state" ]; then
return 1;
fi
- local fw_state="$(cut -d' ' -f1 < $path)"
- local sw_state="$(cut -d' ' -f2 < $path)"
+ local fw_state="$(echo $eeh_state | cut -d' ' -f1)"
+ local sw_state="$(echo $eeh_state | cut -d' ' -f2)"
# If EEH_PE_ISOLATED or EEH_PE_RECOVERING are set then the PE is in an
# error state or being recovered. Either way, not ok.
diff --git a/tools/testing/selftests/powerpc/include/instructions.h b/tools/testing/selftests/powerpc/include/instructions.h
index f36061e..4efa631 100644
--- a/tools/testing/selftests/powerpc/include/instructions.h
+++ b/tools/testing/selftests/powerpc/include/instructions.h
@@ -66,4 +66,81 @@
#define PPC_INST_PASTE __PASTE(0, 0, 0, 0)
#define PPC_INST_PASTE_LAST __PASTE(0, 0, 1, 1)
+/* This defines the prefixed load/store instructions */
+#ifdef __ASSEMBLY__
+# define stringify_in_c(...) __VA_ARGS__
+#else
+# define __stringify_in_c(...) #__VA_ARGS__
+# define stringify_in_c(...) __stringify_in_c(__VA_ARGS__) " "
+#endif
+
+#define __PPC_RA(a) (((a) & 0x1f) << 16)
+#define __PPC_RS(s) (((s) & 0x1f) << 21)
+#define __PPC_RT(t) __PPC_RS(t)
+#define __PPC_PREFIX_R(r) (((r) & 0x1) << 20)
+
+#define PPC_PREFIX_MLS 0x06000000
+#define PPC_PREFIX_8LS 0x04000000
+
+#define PPC_INST_LBZ 0x88000000
+#define PPC_INST_LHZ 0xa0000000
+#define PPC_INST_LHA 0xa8000000
+#define PPC_INST_LWZ 0x80000000
+#define PPC_INST_STB 0x98000000
+#define PPC_INST_STH 0xb0000000
+#define PPC_INST_STW 0x90000000
+#define PPC_INST_STD 0xf8000000
+#define PPC_INST_LFS 0xc0000000
+#define PPC_INST_LFD 0xc8000000
+#define PPC_INST_STFS 0xd0000000
+#define PPC_INST_STFD 0xd8000000
+
+#define PREFIX_MLS(instr, t, a, r, d) stringify_in_c(.balign 64, , 4;) \
+ stringify_in_c(.long PPC_PREFIX_MLS | \
+ __PPC_PREFIX_R(r) | \
+ (((d) >> 16) & 0x3ffff);) \
+ stringify_in_c(.long (instr) | \
+ __PPC_RT(t) | \
+ __PPC_RA(a) | \
+ ((d) & 0xffff);\n)
+
+#define PREFIX_8LS(instr, t, a, r, d) stringify_in_c(.balign 64, , 4;) \
+ stringify_in_c(.long PPC_PREFIX_8LS | \
+ __PPC_PREFIX_R(r) | \
+ (((d) >> 16) & 0x3ffff);) \
+ stringify_in_c(.long (instr) | \
+ __PPC_RT(t) | \
+ __PPC_RA(a) | \
+ ((d) & 0xffff);\n)
+
+/* Prefixed Integer Load/Store instructions */
+#define PLBZ(t, a, r, d) PREFIX_MLS(PPC_INST_LBZ, t, a, r, d)
+#define PLHZ(t, a, r, d) PREFIX_MLS(PPC_INST_LHZ, t, a, r, d)
+#define PLHA(t, a, r, d) PREFIX_MLS(PPC_INST_LHA, t, a, r, d)
+#define PLWZ(t, a, r, d) PREFIX_MLS(PPC_INST_LWZ, t, a, r, d)
+#define PLWA(t, a, r, d) PREFIX_8LS(0xa4000000, t, a, r, d)
+#define PLD(t, a, r, d) PREFIX_8LS(0xe4000000, t, a, r, d)
+#define PLQ(t, a, r, d) PREFIX_8LS(0xe0000000, t, a, r, d)
+#define PSTB(s, a, r, d) PREFIX_MLS(PPC_INST_STB, s, a, r, d)
+#define PSTH(s, a, r, d) PREFIX_MLS(PPC_INST_STH, s, a, r, d)
+#define PSTW(s, a, r, d) PREFIX_MLS(PPC_INST_STW, s, a, r, d)
+#define PSTD(s, a, r, d) PREFIX_8LS(0xf4000000, s, a, r, d)
+#define PSTQ(s, a, r, d) PREFIX_8LS(0xf0000000, s, a, r, d)
+
+/* Prefixed Floating-Point Load/Store Instructions */
+#define PLFS(frt, a, r, d) PREFIX_MLS(PPC_INST_LFS, frt, a, r, d)
+#define PLFD(frt, a, r, d) PREFIX_MLS(PPC_INST_LFD, frt, a, r, d)
+#define PSTFS(frs, a, r, d) PREFIX_MLS(PPC_INST_STFS, frs, a, r, d)
+#define PSTFD(frs, a, r, d) PREFIX_MLS(PPC_INST_STFD, frs, a, r, d)
+
+/* Prefixed VSX Load/Store Instructions */
+#define PLXSD(vrt, a, r, d) PREFIX_8LS(0xa8000000, vrt, a, r, d)
+#define PLXSSP(vrt, a, r, d) PREFIX_8LS(0xac000000, vrt, a, r, d)
+#define PLXV0(s, a, r, d) PREFIX_8LS(0xc8000000, s, a, r, d)
+#define PLXV1(s, a, r, d) PREFIX_8LS(0xcc000000, s, a, r, d)
+#define PSTXSD(vrs, a, r, d) PREFIX_8LS(0xb8000000, vrs, a, r, d)
+#define PSTXSSP(vrs, a, r, d) PREFIX_8LS(0xbc000000, vrs, a, r, d)
+#define PSTXV0(s, a, r, d) PREFIX_8LS(0xd8000000, s, a, r, d)
+#define PSTXV1(s, a, r, d) PREFIX_8LS(0xdc000000, s, a, r, d)
+
#endif /* _SELFTESTS_POWERPC_INSTRUCTIONS_H */
diff --git a/tools/testing/selftests/powerpc/include/pkeys.h b/tools/testing/selftests/powerpc/include/pkeys.h
new file mode 100644
index 0000000..3312cb1
--- /dev/null
+++ b/tools/testing/selftests/powerpc/include/pkeys.h
@@ -0,0 +1,136 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright 2020, Sandipan Das, IBM Corp.
+ */
+
+#ifndef _SELFTESTS_POWERPC_PKEYS_H
+#define _SELFTESTS_POWERPC_PKEYS_H
+
+#include <sys/mman.h>
+
+#include "reg.h"
+#include "utils.h"
+
+/*
+ * Older versions of libc use the Intel-specific access rights.
+ * Hence, override the definitions as they might be incorrect.
+ */
+#undef PKEY_DISABLE_ACCESS
+#define PKEY_DISABLE_ACCESS 0x3
+
+#undef PKEY_DISABLE_WRITE
+#define PKEY_DISABLE_WRITE 0x2
+
+#undef PKEY_DISABLE_EXECUTE
+#define PKEY_DISABLE_EXECUTE 0x4
+
+/* Older versions of libc do not not define this */
+#ifndef SEGV_PKUERR
+#define SEGV_PKUERR 4
+#endif
+
+#define SI_PKEY_OFFSET 0x20
+
+#define __NR_pkey_mprotect 386
+#define __NR_pkey_alloc 384
+#define __NR_pkey_free 385
+
+#define PKEY_BITS_PER_PKEY 2
+#define NR_PKEYS 32
+#define PKEY_BITS_MASK ((1UL << PKEY_BITS_PER_PKEY) - 1)
+
+inline unsigned long pkeyreg_get(void)
+{
+ return mfspr(SPRN_AMR);
+}
+
+inline void pkeyreg_set(unsigned long amr)
+{
+ set_amr(amr);
+}
+
+void pkey_set_rights(int pkey, unsigned long rights)
+{
+ unsigned long amr, shift;
+
+ shift = (NR_PKEYS - pkey - 1) * PKEY_BITS_PER_PKEY;
+ amr = pkeyreg_get();
+ amr &= ~(PKEY_BITS_MASK << shift);
+ amr |= (rights & PKEY_BITS_MASK) << shift;
+ pkeyreg_set(amr);
+}
+
+int sys_pkey_mprotect(void *addr, size_t len, int prot, int pkey)
+{
+ return syscall(__NR_pkey_mprotect, addr, len, prot, pkey);
+}
+
+int sys_pkey_alloc(unsigned long flags, unsigned long rights)
+{
+ return syscall(__NR_pkey_alloc, flags, rights);
+}
+
+int sys_pkey_free(int pkey)
+{
+ return syscall(__NR_pkey_free, pkey);
+}
+
+int pkeys_unsupported(void)
+{
+ bool hash_mmu = false;
+ int pkey;
+
+ /* Protection keys are currently supported on Hash MMU only */
+ FAIL_IF(using_hash_mmu(&hash_mmu));
+ SKIP_IF(!hash_mmu);
+
+ /* Check if the system call is supported */
+ pkey = sys_pkey_alloc(0, 0);
+ SKIP_IF(pkey < 0);
+ sys_pkey_free(pkey);
+
+ return 0;
+}
+
+int siginfo_pkey(siginfo_t *si)
+{
+ /*
+ * In older versions of libc, siginfo_t does not have si_pkey as
+ * a member.
+ */
+#ifdef si_pkey
+ return si->si_pkey;
+#else
+ return *((int *)(((char *) si) + SI_PKEY_OFFSET));
+#endif
+}
+
+#define pkey_rights(r) ({ \
+ static char buf[4] = "rwx"; \
+ unsigned int amr_bits; \
+ if ((r) & PKEY_DISABLE_EXECUTE) \
+ buf[2] = '-'; \
+ amr_bits = (r) & PKEY_BITS_MASK; \
+ if (amr_bits & PKEY_DISABLE_WRITE) \
+ buf[1] = '-'; \
+ if (amr_bits & PKEY_DISABLE_ACCESS & ~PKEY_DISABLE_WRITE) \
+ buf[0] = '-'; \
+ buf; \
+})
+
+unsigned long next_pkey_rights(unsigned long rights)
+{
+ if (rights == PKEY_DISABLE_ACCESS)
+ return PKEY_DISABLE_EXECUTE;
+ else if (rights == (PKEY_DISABLE_ACCESS | PKEY_DISABLE_EXECUTE))
+ return 0;
+
+ if ((rights & PKEY_BITS_MASK) == 0)
+ rights |= PKEY_DISABLE_WRITE;
+ else if ((rights & PKEY_BITS_MASK) == PKEY_DISABLE_WRITE)
+ rights |= PKEY_DISABLE_ACCESS;
+
+ return rights;
+}
+
+#endif /* _SELFTESTS_POWERPC_PKEYS_H */
diff --git a/tools/testing/selftests/powerpc/include/reg.h b/tools/testing/selftests/powerpc/include/reg.h
index 022c507..c0f2742 100644
--- a/tools/testing/selftests/powerpc/include/reg.h
+++ b/tools/testing/selftests/powerpc/include/reg.h
@@ -57,6 +57,12 @@
#define SPRN_PPR 896 /* Program Priority Register */
#define SPRN_AMR 13 /* Authority Mask Register - problem state */
+#define set_amr(v) asm volatile("isync;" \
+ "mtspr " __stringify(SPRN_AMR) ",%0;" \
+ "isync" : \
+ : "r" ((unsigned long)(v)) \
+ : "memory")
+
/* TEXASR register bits */
#define TEXASR_FC 0xFE00000000000000
#define TEXASR_FP 0x0100000000000000
diff --git a/tools/testing/selftests/powerpc/include/utils.h b/tools/testing/selftests/powerpc/include/utils.h
index e089a0c..71d2924f 100644
--- a/tools/testing/selftests/powerpc/include/utils.h
+++ b/tools/testing/selftests/powerpc/include/utils.h
@@ -42,6 +42,16 @@
int perf_event_disable(int fd);
int perf_event_reset(int fd);
+#if !defined(__GLIBC_PREREQ) || !__GLIBC_PREREQ(2, 30)
+#include <unistd.h>
+#include <sys/syscall.h>
+
+static inline pid_t gettid(void)
+{
+ return syscall(SYS_gettid);
+}
+#endif
+
static inline bool have_hwcap(unsigned long ftr)
{
return ((unsigned long)get_auxv_entry(AT_HWCAP) & ftr) == ftr;
@@ -60,6 +70,7 @@
#endif
bool is_ppc64le(void);
+int using_hash_mmu(bool *using_hash);
/* Yes, this is evil */
#define FAIL_IF(x) \
@@ -71,6 +82,15 @@
} \
} while (0)
+#define FAIL_IF_EXIT(x) \
+do { \
+ if ((x)) { \
+ fprintf(stderr, \
+ "[FAIL] Test FAILED on line %d\n", __LINE__); \
+ _exit(1); \
+ } \
+} while (0)
+
/* The test harness uses this, yes it's gross */
#define MAGIC_SKIP_RETURN_VALUE 99
@@ -96,11 +116,20 @@
#define _str(s) #s
#define str(s) _str(s)
+#define sigsafe_err(msg) ({ \
+ ssize_t nbytes __attribute__((unused)); \
+ nbytes = write(STDERR_FILENO, msg, strlen(msg)); })
+
/* POWER9 feature */
#ifndef PPC_FEATURE2_ARCH_3_00
#define PPC_FEATURE2_ARCH_3_00 0x00800000
#endif
+/* POWER10 feature */
+#ifndef PPC_FEATURE2_ARCH_3_1
+#define PPC_FEATURE2_ARCH_3_1 0x00040000
+#endif
+
#if defined(__powerpc64__)
#define UCONTEXT_NIA(UC) (UC)->uc_mcontext.gp_regs[PT_NIP]
#define UCONTEXT_MSR(UC) (UC)->uc_mcontext.gp_regs[PT_MSR]
diff --git a/tools/testing/selftests/powerpc/math/.gitignore b/tools/testing/selftests/powerpc/math/.gitignore
index e31ca6f..d0c23b2 100644
--- a/tools/testing/selftests/powerpc/math/.gitignore
+++ b/tools/testing/selftests/powerpc/math/.gitignore
@@ -6,3 +6,4 @@
fpu_signal
vmx_signal
vsx_preempt
+fpu_denormal
diff --git a/tools/testing/selftests/powerpc/math/Makefile b/tools/testing/selftests/powerpc/math/Makefile
index 11a10d7..fcc91c2 100644
--- a/tools/testing/selftests/powerpc/math/Makefile
+++ b/tools/testing/selftests/powerpc/math/Makefile
@@ -1,5 +1,5 @@
# SPDX-License-Identifier: GPL-2.0
-TEST_GEN_PROGS := fpu_syscall fpu_preempt fpu_signal vmx_syscall vmx_preempt vmx_signal vsx_preempt
+TEST_GEN_PROGS := fpu_syscall fpu_preempt fpu_signal fpu_denormal vmx_syscall vmx_preempt vmx_signal vsx_preempt
top_srcdir = ../../../../..
include ../../lib.mk
@@ -11,9 +11,9 @@
$(OUTPUT)/fpu_preempt: fpu_asm.S
$(OUTPUT)/fpu_signal: fpu_asm.S
-$(OUTPUT)/vmx_syscall: vmx_asm.S
-$(OUTPUT)/vmx_preempt: vmx_asm.S
-$(OUTPUT)/vmx_signal: vmx_asm.S
+$(OUTPUT)/vmx_syscall: vmx_asm.S ../utils.c
+$(OUTPUT)/vmx_preempt: vmx_asm.S ../utils.c
+$(OUTPUT)/vmx_signal: vmx_asm.S ../utils.c
$(OUTPUT)/vsx_preempt: CFLAGS += -mvsx
-$(OUTPUT)/vsx_preempt: vsx_asm.S
+$(OUTPUT)/vsx_preempt: vsx_asm.S ../utils.c
diff --git a/tools/testing/selftests/powerpc/math/fpu_denormal.c b/tools/testing/selftests/powerpc/math/fpu_denormal.c
new file mode 100644
index 0000000..5f96682
--- /dev/null
+++ b/tools/testing/selftests/powerpc/math/fpu_denormal.c
@@ -0,0 +1,38 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ * Copyright IBM Corp. 2020
+ *
+ * This test attempts to cause a FP denormal exception on POWER8 CPUs. Unfortunately
+ * if the denormal handler is not configured or working properly, this can cause a bad
+ * crash in kernel mode when the kernel tries to save FP registers when the process
+ * exits.
+ */
+
+#include <stdio.h>
+#include <string.h>
+
+#include "utils.h"
+
+static int test_denormal_fpu(void)
+{
+ unsigned int m32;
+ unsigned long m64;
+ volatile float f;
+ volatile double d;
+
+ /* try to induce lfs <denormal> ; stfd */
+
+ m32 = 0x00715fcf; /* random denormal */
+ memcpy((float *)&f, &m32, sizeof(f));
+ d = f;
+ memcpy(&m64, (double *)&d, sizeof(d));
+
+ FAIL_IF((long)(m64 != 0x380c57f3c0000000)); /* renormalised value */
+
+ return 0;
+}
+
+int main(int argc, char *argv[])
+{
+ return test_harness(test_denormal_fpu, "fpu_denormal");
+}
diff --git a/tools/testing/selftests/powerpc/math/vmx_preempt.c b/tools/testing/selftests/powerpc/math/vmx_preempt.c
index 2e059f1..6761d6c 100644
--- a/tools/testing/selftests/powerpc/math/vmx_preempt.c
+++ b/tools/testing/selftests/powerpc/math/vmx_preempt.c
@@ -57,6 +57,9 @@
int i, rc, threads;
pthread_t *tids;
+ // vcmpequd used in vmx_asm.S is v2.07
+ SKIP_IF(!have_hwcap2(PPC_FEATURE2_ARCH_2_07));
+
threads = sysconf(_SC_NPROCESSORS_ONLN) * THREAD_FACTOR;
tids = malloc(threads * sizeof(pthread_t));
FAIL_IF(!tids);
diff --git a/tools/testing/selftests/powerpc/math/vmx_signal.c b/tools/testing/selftests/powerpc/math/vmx_signal.c
index 785a48e..b340a5c 100644
--- a/tools/testing/selftests/powerpc/math/vmx_signal.c
+++ b/tools/testing/selftests/powerpc/math/vmx_signal.c
@@ -96,6 +96,9 @@
void *rc_p;
pthread_t *tids;
+ // vcmpequd used in vmx_asm.S is v2.07
+ SKIP_IF(!have_hwcap2(PPC_FEATURE2_ARCH_2_07));
+
threads = sysconf(_SC_NPROCESSORS_ONLN) * THREAD_FACTOR;
tids = malloc(threads * sizeof(pthread_t));
FAIL_IF(!tids);
diff --git a/tools/testing/selftests/powerpc/math/vmx_syscall.c b/tools/testing/selftests/powerpc/math/vmx_syscall.c
index 9ee293c..03c78df 100644
--- a/tools/testing/selftests/powerpc/math/vmx_syscall.c
+++ b/tools/testing/selftests/powerpc/math/vmx_syscall.c
@@ -49,9 +49,14 @@
* Setup an environment with much context switching
*/
pid_t pid2;
- pid_t pid = fork();
+ pid_t pid;
int ret;
int child_ret;
+
+ // vcmpequd used in vmx_asm.S is v2.07
+ SKIP_IF(!have_hwcap2(PPC_FEATURE2_ARCH_2_07));
+
+ pid = fork();
FAIL_IF(pid == -1);
pid2 = fork();
diff --git a/tools/testing/selftests/powerpc/math/vsx_preempt.c b/tools/testing/selftests/powerpc/math/vsx_preempt.c
index 63de9c6..d1601bb 100644
--- a/tools/testing/selftests/powerpc/math/vsx_preempt.c
+++ b/tools/testing/selftests/powerpc/math/vsx_preempt.c
@@ -92,6 +92,8 @@
int i, rc, threads;
pthread_t *tids;
+ SKIP_IF(!have_hwcap(PPC_FEATURE_HAS_VSX));
+
threads = sysconf(_SC_NPROCESSORS_ONLN) * THREAD_FACTOR;
tids = malloc(threads * sizeof(pthread_t));
FAIL_IF(!tids);
diff --git a/tools/testing/selftests/powerpc/mm/.gitignore b/tools/testing/selftests/powerpc/mm/.gitignore
index 2ca5232..91c775c 100644
--- a/tools/testing/selftests/powerpc/mm/.gitignore
+++ b/tools/testing/selftests/powerpc/mm/.gitignore
@@ -2,9 +2,12 @@
hugetlb_vs_thp_test
subpage_prot
tempfile
-prot_sao
segv_errors
wild_bctr
large_vm_fork_separation
bad_accesses
tlbie_test
+pkey_exec_prot
+pkey_siginfo
+stack_expansion_ldst
+stack_expansion_signal
diff --git a/tools/testing/selftests/powerpc/mm/Makefile b/tools/testing/selftests/powerpc/mm/Makefile
index b9103c4..250ce17 100644
--- a/tools/testing/selftests/powerpc/mm/Makefile
+++ b/tools/testing/selftests/powerpc/mm/Makefile
@@ -2,23 +2,31 @@
noarg:
$(MAKE) -C ../
-TEST_GEN_PROGS := hugetlb_vs_thp_test subpage_prot prot_sao segv_errors wild_bctr \
- large_vm_fork_separation bad_accesses
+TEST_GEN_PROGS := hugetlb_vs_thp_test subpage_prot segv_errors wild_bctr \
+ large_vm_fork_separation bad_accesses pkey_exec_prot \
+ pkey_siginfo stack_expansion_signal stack_expansion_ldst
+
TEST_GEN_PROGS_EXTENDED := tlbie_test
TEST_GEN_FILES := tempfile
top_srcdir = ../../../../..
include ../../lib.mk
-$(TEST_GEN_PROGS): ../harness.c
-
-$(OUTPUT)/prot_sao: ../utils.c
+$(TEST_GEN_PROGS): ../harness.c ../utils.c
$(OUTPUT)/wild_bctr: CFLAGS += -m64
$(OUTPUT)/large_vm_fork_separation: CFLAGS += -m64
$(OUTPUT)/bad_accesses: CFLAGS += -m64
+$(OUTPUT)/pkey_exec_prot: CFLAGS += -m64
+$(OUTPUT)/pkey_siginfo: CFLAGS += -m64
+
+$(OUTPUT)/stack_expansion_signal: ../utils.c ../pmu/lib.c
+
+$(OUTPUT)/stack_expansion_ldst: CFLAGS += -fno-stack-protector
+$(OUTPUT)/stack_expansion_ldst: ../utils.c
$(OUTPUT)/tempfile:
dd if=/dev/zero of=$@ bs=64k count=1
$(OUTPUT)/tlbie_test: LDLIBS += -lpthread
+$(OUTPUT)/pkey_siginfo: LDLIBS += -lpthread
diff --git a/tools/testing/selftests/powerpc/mm/bad_accesses.c b/tools/testing/selftests/powerpc/mm/bad_accesses.c
index adc465f..a864ed7 100644
--- a/tools/testing/selftests/powerpc/mm/bad_accesses.c
+++ b/tools/testing/selftests/powerpc/mm/bad_accesses.c
@@ -64,34 +64,6 @@
return 0;
}
-static int using_hash_mmu(bool *using_hash)
-{
- char line[128];
- FILE *f;
- int rc;
-
- f = fopen("/proc/cpuinfo", "r");
- FAIL_IF(!f);
-
- rc = 0;
- while (fgets(line, sizeof(line), f) != NULL) {
- if (strcmp(line, "MMU : Hash\n") == 0) {
- *using_hash = true;
- goto out;
- }
-
- if (strcmp(line, "MMU : Radix\n") == 0) {
- *using_hash = false;
- goto out;
- }
- }
-
- rc = -1;
-out:
- fclose(f);
- return rc;
-}
-
static int test(void)
{
unsigned long i, j, addr, region_shift, page_shift, page_size;
diff --git a/tools/testing/selftests/powerpc/mm/pkey_exec_prot.c b/tools/testing/selftests/powerpc/mm/pkey_exec_prot.c
new file mode 100644
index 0000000..9e5c7f3
--- /dev/null
+++ b/tools/testing/selftests/powerpc/mm/pkey_exec_prot.c
@@ -0,0 +1,294 @@
+// SPDX-License-Identifier: GPL-2.0+
+
+/*
+ * Copyright 2020, Sandipan Das, IBM Corp.
+ *
+ * Test if applying execute protection on pages using memory
+ * protection keys works as expected.
+ */
+
+#define _GNU_SOURCE
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <signal.h>
+
+#include <unistd.h>
+
+#include "pkeys.h"
+
+#define PPC_INST_NOP 0x60000000
+#define PPC_INST_TRAP 0x7fe00008
+#define PPC_INST_BLR 0x4e800020
+
+static volatile sig_atomic_t fault_pkey, fault_code, fault_type;
+static volatile sig_atomic_t remaining_faults;
+static volatile unsigned int *fault_addr;
+static unsigned long pgsize, numinsns;
+static unsigned int *insns;
+
+static void trap_handler(int signum, siginfo_t *sinfo, void *ctx)
+{
+ /* Check if this fault originated from the expected address */
+ if (sinfo->si_addr != (void *) fault_addr)
+ sigsafe_err("got a fault for an unexpected address\n");
+
+ _exit(1);
+}
+
+static void segv_handler(int signum, siginfo_t *sinfo, void *ctx)
+{
+ int signal_pkey;
+
+ signal_pkey = siginfo_pkey(sinfo);
+ fault_code = sinfo->si_code;
+
+ /* Check if this fault originated from the expected address */
+ if (sinfo->si_addr != (void *) fault_addr) {
+ sigsafe_err("got a fault for an unexpected address\n");
+ _exit(1);
+ }
+
+ /* Check if too many faults have occurred for a single test case */
+ if (!remaining_faults) {
+ sigsafe_err("got too many faults for the same address\n");
+ _exit(1);
+ }
+
+
+ /* Restore permissions in order to continue */
+ switch (fault_code) {
+ case SEGV_ACCERR:
+ if (mprotect(insns, pgsize, PROT_READ | PROT_WRITE)) {
+ sigsafe_err("failed to set access permissions\n");
+ _exit(1);
+ }
+ break;
+ case SEGV_PKUERR:
+ if (signal_pkey != fault_pkey) {
+ sigsafe_err("got a fault for an unexpected pkey\n");
+ _exit(1);
+ }
+
+ switch (fault_type) {
+ case PKEY_DISABLE_ACCESS:
+ pkey_set_rights(fault_pkey, 0);
+ break;
+ case PKEY_DISABLE_EXECUTE:
+ /*
+ * Reassociate the exec-only pkey with the region
+ * to be able to continue. Unlike AMR, we cannot
+ * set IAMR directly from userspace to restore the
+ * permissions.
+ */
+ if (mprotect(insns, pgsize, PROT_EXEC)) {
+ sigsafe_err("failed to set execute permissions\n");
+ _exit(1);
+ }
+ break;
+ default:
+ sigsafe_err("got a fault with an unexpected type\n");
+ _exit(1);
+ }
+ break;
+ default:
+ sigsafe_err("got a fault with an unexpected code\n");
+ _exit(1);
+ }
+
+ remaining_faults--;
+}
+
+static int test(void)
+{
+ struct sigaction segv_act, trap_act;
+ unsigned long rights;
+ int pkey, ret, i;
+
+ ret = pkeys_unsupported();
+ if (ret)
+ return ret;
+
+ /* Setup SIGSEGV handler */
+ segv_act.sa_handler = 0;
+ segv_act.sa_sigaction = segv_handler;
+ FAIL_IF(sigprocmask(SIG_SETMASK, 0, &segv_act.sa_mask) != 0);
+ segv_act.sa_flags = SA_SIGINFO;
+ segv_act.sa_restorer = 0;
+ FAIL_IF(sigaction(SIGSEGV, &segv_act, NULL) != 0);
+
+ /* Setup SIGTRAP handler */
+ trap_act.sa_handler = 0;
+ trap_act.sa_sigaction = trap_handler;
+ FAIL_IF(sigprocmask(SIG_SETMASK, 0, &trap_act.sa_mask) != 0);
+ trap_act.sa_flags = SA_SIGINFO;
+ trap_act.sa_restorer = 0;
+ FAIL_IF(sigaction(SIGTRAP, &trap_act, NULL) != 0);
+
+ /* Setup executable region */
+ pgsize = getpagesize();
+ numinsns = pgsize / sizeof(unsigned int);
+ insns = (unsigned int *) mmap(NULL, pgsize, PROT_READ | PROT_WRITE,
+ MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
+ FAIL_IF(insns == MAP_FAILED);
+
+ /* Write the instruction words */
+ for (i = 1; i < numinsns - 1; i++)
+ insns[i] = PPC_INST_NOP;
+
+ /*
+ * Set the first instruction as an unconditional trap. If
+ * the last write to this address succeeds, this should
+ * get overwritten by a no-op.
+ */
+ insns[0] = PPC_INST_TRAP;
+
+ /*
+ * Later, to jump to the executable region, we use a branch
+ * and link instruction (bctrl) which sets the return address
+ * automatically in LR. Use that to return back.
+ */
+ insns[numinsns - 1] = PPC_INST_BLR;
+
+ /* Allocate a pkey that restricts execution */
+ rights = PKEY_DISABLE_EXECUTE;
+ pkey = sys_pkey_alloc(0, rights);
+ FAIL_IF(pkey < 0);
+
+ /*
+ * Pick the first instruction's address from the executable
+ * region.
+ */
+ fault_addr = insns;
+
+ /* The following two cases will avoid SEGV_PKUERR */
+ fault_type = -1;
+ fault_pkey = -1;
+
+ /*
+ * Read an instruction word from the address when AMR bits
+ * are not set i.e. the pkey permits both read and write
+ * access.
+ *
+ * This should not generate a fault as having PROT_EXEC
+ * implies PROT_READ on GNU systems. The pkey currently
+ * restricts execution only based on the IAMR bits. The
+ * AMR bits are cleared.
+ */
+ remaining_faults = 0;
+ FAIL_IF(sys_pkey_mprotect(insns, pgsize, PROT_EXEC, pkey) != 0);
+ printf("read from %p, pkey permissions are %s\n", fault_addr,
+ pkey_rights(rights));
+ i = *fault_addr;
+ FAIL_IF(remaining_faults != 0);
+
+ /*
+ * Write an instruction word to the address when AMR bits
+ * are not set i.e. the pkey permits both read and write
+ * access.
+ *
+ * This should generate an access fault as having just
+ * PROT_EXEC also restricts writes. The pkey currently
+ * restricts execution only based on the IAMR bits. The
+ * AMR bits are cleared.
+ */
+ remaining_faults = 1;
+ FAIL_IF(sys_pkey_mprotect(insns, pgsize, PROT_EXEC, pkey) != 0);
+ printf("write to %p, pkey permissions are %s\n", fault_addr,
+ pkey_rights(rights));
+ *fault_addr = PPC_INST_TRAP;
+ FAIL_IF(remaining_faults != 0 || fault_code != SEGV_ACCERR);
+
+ /* The following three cases will generate SEGV_PKUERR */
+ rights |= PKEY_DISABLE_ACCESS;
+ fault_type = PKEY_DISABLE_ACCESS;
+ fault_pkey = pkey;
+
+ /*
+ * Read an instruction word from the address when AMR bits
+ * are set i.e. the pkey permits neither read nor write
+ * access.
+ *
+ * This should generate a pkey fault based on AMR bits only
+ * as having PROT_EXEC implicitly allows reads.
+ */
+ remaining_faults = 1;
+ FAIL_IF(sys_pkey_mprotect(insns, pgsize, PROT_EXEC, pkey) != 0);
+ pkey_set_rights(pkey, rights);
+ printf("read from %p, pkey permissions are %s\n", fault_addr,
+ pkey_rights(rights));
+ i = *fault_addr;
+ FAIL_IF(remaining_faults != 0 || fault_code != SEGV_PKUERR);
+
+ /*
+ * Write an instruction word to the address when AMR bits
+ * are set i.e. the pkey permits neither read nor write
+ * access.
+ *
+ * This should generate two faults. First, a pkey fault
+ * based on AMR bits and then an access fault since
+ * PROT_EXEC does not allow writes.
+ */
+ remaining_faults = 2;
+ FAIL_IF(sys_pkey_mprotect(insns, pgsize, PROT_EXEC, pkey) != 0);
+ pkey_set_rights(pkey, rights);
+ printf("write to %p, pkey permissions are %s\n", fault_addr,
+ pkey_rights(rights));
+ *fault_addr = PPC_INST_NOP;
+ FAIL_IF(remaining_faults != 0 || fault_code != SEGV_ACCERR);
+
+ /* Free the current pkey */
+ sys_pkey_free(pkey);
+
+ rights = 0;
+ do {
+ /*
+ * Allocate pkeys with all valid combinations of read,
+ * write and execute restrictions.
+ */
+ pkey = sys_pkey_alloc(0, rights);
+ FAIL_IF(pkey < 0);
+
+ /*
+ * Jump to the executable region. AMR bits may or may not
+ * be set but they should not affect execution.
+ *
+ * This should generate pkey faults based on IAMR bits which
+ * may be set to restrict execution.
+ *
+ * The first iteration also checks if the overwrite of the
+ * first instruction word from a trap to a no-op succeeded.
+ */
+ fault_pkey = pkey;
+ fault_type = -1;
+ remaining_faults = 0;
+ if (rights & PKEY_DISABLE_EXECUTE) {
+ fault_type = PKEY_DISABLE_EXECUTE;
+ remaining_faults = 1;
+ }
+
+ FAIL_IF(sys_pkey_mprotect(insns, pgsize, PROT_EXEC, pkey) != 0);
+ printf("execute at %p, pkey permissions are %s\n", fault_addr,
+ pkey_rights(rights));
+ asm volatile("mtctr %0; bctrl" : : "r"(insns));
+ FAIL_IF(remaining_faults != 0);
+ if (rights & PKEY_DISABLE_EXECUTE)
+ FAIL_IF(fault_code != SEGV_PKUERR);
+
+ /* Free the current pkey */
+ sys_pkey_free(pkey);
+
+ /* Find next valid combination of pkey rights */
+ rights = next_pkey_rights(rights);
+ } while (rights);
+
+ /* Cleanup */
+ munmap((void *) insns, pgsize);
+
+ return 0;
+}
+
+int main(void)
+{
+ test_harness(test, "pkey_exec_prot");
+}
diff --git a/tools/testing/selftests/powerpc/mm/pkey_siginfo.c b/tools/testing/selftests/powerpc/mm/pkey_siginfo.c
new file mode 100644
index 0000000..4f815d7
--- /dev/null
+++ b/tools/testing/selftests/powerpc/mm/pkey_siginfo.c
@@ -0,0 +1,333 @@
+// SPDX-License-Identifier: GPL-2.0
+
+/*
+ * Copyright 2020, Sandipan Das, IBM Corp.
+ *
+ * Test if the signal information reports the correct memory protection
+ * key upon getting a key access violation fault for a page that was
+ * attempted to be protected by two different keys from two competing
+ * threads at the same time.
+ */
+
+#define _GNU_SOURCE
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <signal.h>
+
+#include <unistd.h>
+#include <pthread.h>
+#include <sys/mman.h>
+
+#include "pkeys.h"
+
+#define PPC_INST_NOP 0x60000000
+#define PPC_INST_BLR 0x4e800020
+#define PROT_RWX (PROT_READ | PROT_WRITE | PROT_EXEC)
+
+#define NUM_ITERATIONS 1000000
+
+static volatile sig_atomic_t perm_pkey, rest_pkey;
+static volatile sig_atomic_t rights, fault_count;
+static volatile unsigned int *volatile fault_addr;
+static pthread_barrier_t iteration_barrier;
+
+static void segv_handler(int signum, siginfo_t *sinfo, void *ctx)
+{
+ void *pgstart;
+ size_t pgsize;
+ int pkey;
+
+ pkey = siginfo_pkey(sinfo);
+
+ /* Check if this fault originated from a pkey access violation */
+ if (sinfo->si_code != SEGV_PKUERR) {
+ sigsafe_err("got a fault for an unexpected reason\n");
+ _exit(1);
+ }
+
+ /* Check if this fault originated from the expected address */
+ if (sinfo->si_addr != (void *) fault_addr) {
+ sigsafe_err("got a fault for an unexpected address\n");
+ _exit(1);
+ }
+
+ /* Check if this fault originated from the restrictive pkey */
+ if (pkey != rest_pkey) {
+ sigsafe_err("got a fault for an unexpected pkey\n");
+ _exit(1);
+ }
+
+ /* Check if too many faults have occurred for the same iteration */
+ if (fault_count > 0) {
+ sigsafe_err("got too many faults for the same address\n");
+ _exit(1);
+ }
+
+ pgsize = getpagesize();
+ pgstart = (void *) ((unsigned long) fault_addr & ~(pgsize - 1));
+
+ /*
+ * If the current fault occurred due to lack of execute rights,
+ * reassociate the page with the exec-only pkey since execute
+ * rights cannot be changed directly for the faulting pkey as
+ * IAMR is inaccessible from userspace.
+ *
+ * Otherwise, if the current fault occurred due to lack of
+ * read-write rights, change the AMR permission bits for the
+ * pkey.
+ *
+ * This will let the test continue.
+ */
+ if (rights == PKEY_DISABLE_EXECUTE &&
+ mprotect(pgstart, pgsize, PROT_EXEC))
+ _exit(1);
+ else
+ pkey_set_rights(pkey, 0);
+
+ fault_count++;
+}
+
+struct region {
+ unsigned long rights;
+ unsigned int *base;
+ size_t size;
+};
+
+static void *protect(void *p)
+{
+ unsigned long rights;
+ unsigned int *base;
+ size_t size;
+ int tid, i;
+
+ tid = gettid();
+ base = ((struct region *) p)->base;
+ size = ((struct region *) p)->size;
+ FAIL_IF_EXIT(!base);
+
+ /* No read, write and execute restrictions */
+ rights = 0;
+
+ printf("tid %d, pkey permissions are %s\n", tid, pkey_rights(rights));
+
+ /* Allocate the permissive pkey */
+ perm_pkey = sys_pkey_alloc(0, rights);
+ FAIL_IF_EXIT(perm_pkey < 0);
+
+ /*
+ * Repeatedly try to protect the common region with a permissive
+ * pkey
+ */
+ for (i = 0; i < NUM_ITERATIONS; i++) {
+ /*
+ * Wait until the other thread has finished allocating the
+ * restrictive pkey or until the next iteration has begun
+ */
+ pthread_barrier_wait(&iteration_barrier);
+
+ /* Try to associate the permissive pkey with the region */
+ FAIL_IF_EXIT(sys_pkey_mprotect(base, size, PROT_RWX,
+ perm_pkey));
+ }
+
+ /* Free the permissive pkey */
+ sys_pkey_free(perm_pkey);
+
+ return NULL;
+}
+
+static void *protect_access(void *p)
+{
+ size_t size, numinsns;
+ unsigned int *base;
+ int tid, i;
+
+ tid = gettid();
+ base = ((struct region *) p)->base;
+ size = ((struct region *) p)->size;
+ rights = ((struct region *) p)->rights;
+ numinsns = size / sizeof(base[0]);
+ FAIL_IF_EXIT(!base);
+
+ /* Allocate the restrictive pkey */
+ rest_pkey = sys_pkey_alloc(0, rights);
+ FAIL_IF_EXIT(rest_pkey < 0);
+
+ printf("tid %d, pkey permissions are %s\n", tid, pkey_rights(rights));
+ printf("tid %d, %s randomly in range [%p, %p]\n", tid,
+ (rights == PKEY_DISABLE_EXECUTE) ? "execute" :
+ (rights == PKEY_DISABLE_WRITE) ? "write" : "read",
+ base, base + numinsns);
+
+ /*
+ * Repeatedly try to protect the common region with a restrictive
+ * pkey and read, write or execute from it
+ */
+ for (i = 0; i < NUM_ITERATIONS; i++) {
+ /*
+ * Wait until the other thread has finished allocating the
+ * permissive pkey or until the next iteration has begun
+ */
+ pthread_barrier_wait(&iteration_barrier);
+
+ /* Try to associate the restrictive pkey with the region */
+ FAIL_IF_EXIT(sys_pkey_mprotect(base, size, PROT_RWX,
+ rest_pkey));
+
+ /* Choose a random instruction word address from the region */
+ fault_addr = base + (rand() % numinsns);
+ fault_count = 0;
+
+ switch (rights) {
+ /* Read protection test */
+ case PKEY_DISABLE_ACCESS:
+ /*
+ * Read an instruction word from the region and
+ * verify if it has not been overwritten to
+ * something unexpected
+ */
+ FAIL_IF_EXIT(*fault_addr != PPC_INST_NOP &&
+ *fault_addr != PPC_INST_BLR);
+ break;
+
+ /* Write protection test */
+ case PKEY_DISABLE_WRITE:
+ /*
+ * Write an instruction word to the region and
+ * verify if the overwrite has succeeded
+ */
+ *fault_addr = PPC_INST_BLR;
+ FAIL_IF_EXIT(*fault_addr != PPC_INST_BLR);
+ break;
+
+ /* Execute protection test */
+ case PKEY_DISABLE_EXECUTE:
+ /* Jump to the region and execute instructions */
+ asm volatile(
+ "mtctr %0; bctrl"
+ : : "r"(fault_addr) : "ctr", "lr");
+ break;
+ }
+
+ /*
+ * Restore the restrictions originally imposed by the
+ * restrictive pkey as the signal handler would have
+ * cleared out the corresponding AMR bits
+ */
+ pkey_set_rights(rest_pkey, rights);
+ }
+
+ /* Free restrictive pkey */
+ sys_pkey_free(rest_pkey);
+
+ return NULL;
+}
+
+static void reset_pkeys(unsigned long rights)
+{
+ int pkeys[NR_PKEYS], i;
+
+ /* Exhaustively allocate all available pkeys */
+ for (i = 0; i < NR_PKEYS; i++)
+ pkeys[i] = sys_pkey_alloc(0, rights);
+
+ /* Free all allocated pkeys */
+ for (i = 0; i < NR_PKEYS; i++)
+ sys_pkey_free(pkeys[i]);
+}
+
+static int test(void)
+{
+ pthread_t prot_thread, pacc_thread;
+ struct sigaction act;
+ pthread_attr_t attr;
+ size_t numinsns;
+ struct region r;
+ int ret, i;
+
+ srand(time(NULL));
+ ret = pkeys_unsupported();
+ if (ret)
+ return ret;
+
+ /* Allocate the region */
+ r.size = getpagesize();
+ r.base = mmap(NULL, r.size, PROT_RWX,
+ MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
+ FAIL_IF(r.base == MAP_FAILED);
+
+ /*
+ * Fill the region with no-ops with a branch at the end
+ * for returning to the caller
+ */
+ numinsns = r.size / sizeof(r.base[0]);
+ for (i = 0; i < numinsns - 1; i++)
+ r.base[i] = PPC_INST_NOP;
+ r.base[i] = PPC_INST_BLR;
+
+ /* Setup SIGSEGV handler */
+ act.sa_handler = 0;
+ act.sa_sigaction = segv_handler;
+ FAIL_IF(sigprocmask(SIG_SETMASK, 0, &act.sa_mask) != 0);
+ act.sa_flags = SA_SIGINFO;
+ act.sa_restorer = 0;
+ FAIL_IF(sigaction(SIGSEGV, &act, NULL) != 0);
+
+ /*
+ * For these tests, the parent process should clear all bits of
+ * AMR and IAMR, i.e. impose no restrictions, for all available
+ * pkeys. This will be the base for the initial AMR and IAMR
+ * values for all the test thread pairs.
+ *
+ * If the AMR and IAMR bits of all available pkeys are cleared
+ * before running the tests and a fault is generated when
+ * attempting to read, write or execute instructions from a
+ * pkey protected region, the pkey responsible for this must be
+ * the one from the protect-and-access thread since the other
+ * one is fully permissive. Despite that, if the pkey reported
+ * by siginfo is not the restrictive pkey, then there must be a
+ * kernel bug.
+ */
+ reset_pkeys(0);
+
+ /* Setup barrier for protect and protect-and-access threads */
+ FAIL_IF(pthread_attr_init(&attr) != 0);
+ FAIL_IF(pthread_barrier_init(&iteration_barrier, NULL, 2) != 0);
+
+ /* Setup and start protect and protect-and-read threads */
+ puts("starting thread pair (protect, protect-and-read)");
+ r.rights = PKEY_DISABLE_ACCESS;
+ FAIL_IF(pthread_create(&prot_thread, &attr, &protect, &r) != 0);
+ FAIL_IF(pthread_create(&pacc_thread, &attr, &protect_access, &r) != 0);
+ FAIL_IF(pthread_join(prot_thread, NULL) != 0);
+ FAIL_IF(pthread_join(pacc_thread, NULL) != 0);
+
+ /* Setup and start protect and protect-and-write threads */
+ puts("starting thread pair (protect, protect-and-write)");
+ r.rights = PKEY_DISABLE_WRITE;
+ FAIL_IF(pthread_create(&prot_thread, &attr, &protect, &r) != 0);
+ FAIL_IF(pthread_create(&pacc_thread, &attr, &protect_access, &r) != 0);
+ FAIL_IF(pthread_join(prot_thread, NULL) != 0);
+ FAIL_IF(pthread_join(pacc_thread, NULL) != 0);
+
+ /* Setup and start protect and protect-and-execute threads */
+ puts("starting thread pair (protect, protect-and-execute)");
+ r.rights = PKEY_DISABLE_EXECUTE;
+ FAIL_IF(pthread_create(&prot_thread, &attr, &protect, &r) != 0);
+ FAIL_IF(pthread_create(&pacc_thread, &attr, &protect_access, &r) != 0);
+ FAIL_IF(pthread_join(prot_thread, NULL) != 0);
+ FAIL_IF(pthread_join(pacc_thread, NULL) != 0);
+
+ /* Cleanup */
+ FAIL_IF(pthread_attr_destroy(&attr) != 0);
+ FAIL_IF(pthread_barrier_destroy(&iteration_barrier) != 0);
+ munmap(r.base, r.size);
+
+ return 0;
+}
+
+int main(void)
+{
+ test_harness(test, "pkey_siginfo");
+}
diff --git a/tools/testing/selftests/powerpc/mm/prot_sao.c b/tools/testing/selftests/powerpc/mm/prot_sao.c
deleted file mode 100644
index e2eed65..0000000
--- a/tools/testing/selftests/powerpc/mm/prot_sao.c
+++ /dev/null
@@ -1,42 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-only
-/*
- * Copyright 2016, Michael Ellerman, IBM Corp.
- */
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-#include <sys/mman.h>
-
-#include <asm/cputable.h>
-
-#include "utils.h"
-
-#define SIZE (64 * 1024)
-
-int test_prot_sao(void)
-{
- char *p;
-
- /* 2.06 or later should support SAO */
- SKIP_IF(!have_hwcap(PPC_FEATURE_ARCH_2_06));
-
- /*
- * Ensure we can ask for PROT_SAO.
- * We can't really verify that it does the right thing, but at least we
- * confirm the kernel will accept it.
- */
- p = mmap(NULL, SIZE, PROT_READ | PROT_WRITE | PROT_SAO,
- MAP_ANONYMOUS | MAP_PRIVATE, -1, 0);
- FAIL_IF(p == MAP_FAILED);
-
- /* Write to the mapping, to at least cause a fault */
- memset(p, 0xaa, SIZE);
-
- return 0;
-}
-
-int main(void)
-{
- return test_harness(test_prot_sao, "prot-sao");
-}
diff --git a/tools/testing/selftests/powerpc/mm/stack_expansion_ldst.c b/tools/testing/selftests/powerpc/mm/stack_expansion_ldst.c
new file mode 100644
index 0000000..ed91439
--- /dev/null
+++ b/tools/testing/selftests/powerpc/mm/stack_expansion_ldst.c
@@ -0,0 +1,202 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Test that loads/stores expand the stack segment, or trigger a SEGV, in
+ * various conditions.
+ *
+ * Based on test code by Tom Lane.
+ */
+
+#undef NDEBUG
+#include <assert.h>
+
+#include <err.h>
+#include <errno.h>
+#include <stdio.h>
+#include <signal.h>
+#include <stdlib.h>
+#include <string.h>
+#include <sys/resource.h>
+#include <sys/time.h>
+#include <sys/types.h>
+#include <sys/wait.h>
+#include <unistd.h>
+
+#define _KB (1024)
+#define _MB (1024 * 1024)
+
+volatile char *stack_top_ptr;
+volatile unsigned long stack_top_sp;
+volatile char c;
+
+enum access_type {
+ LOAD,
+ STORE,
+};
+
+/*
+ * Consume stack until the stack pointer is below @target_sp, then do an access
+ * (load or store) at offset @delta from either the base of the stack or the
+ * current stack pointer.
+ */
+__attribute__ ((noinline))
+int consume_stack(unsigned long target_sp, unsigned long stack_high, int delta, enum access_type type)
+{
+ unsigned long target;
+ char stack_cur;
+
+ if ((unsigned long)&stack_cur > target_sp)
+ return consume_stack(target_sp, stack_high, delta, type);
+ else {
+ // We don't really need this, but without it GCC might not
+ // generate a recursive call above.
+ stack_top_ptr = &stack_cur;
+
+#ifdef __powerpc__
+ asm volatile ("mr %[sp], %%r1" : [sp] "=r" (stack_top_sp));
+#else
+ asm volatile ("mov %%rsp, %[sp]" : [sp] "=r" (stack_top_sp));
+#endif
+ target = stack_high - delta + 1;
+ volatile char *p = (char *)target;
+
+ if (type == STORE)
+ *p = c;
+ else
+ c = *p;
+
+ // Do something to prevent the stack frame being popped prior to
+ // our access above.
+ getpid();
+ }
+
+ return 0;
+}
+
+static int search_proc_maps(char *needle, unsigned long *low, unsigned long *high)
+{
+ unsigned long start, end;
+ static char buf[4096];
+ char name[128];
+ FILE *f;
+ int rc;
+
+ f = fopen("/proc/self/maps", "r");
+ if (!f) {
+ perror("fopen");
+ return -1;
+ }
+
+ while (fgets(buf, sizeof(buf), f)) {
+ rc = sscanf(buf, "%lx-%lx %*c%*c%*c%*c %*x %*d:%*d %*d %127s\n",
+ &start, &end, name);
+ if (rc == 2)
+ continue;
+
+ if (rc != 3) {
+ printf("sscanf errored\n");
+ rc = -1;
+ break;
+ }
+
+ if (strstr(name, needle)) {
+ *low = start;
+ *high = end - 1;
+ rc = 0;
+ break;
+ }
+ }
+
+ fclose(f);
+
+ return rc;
+}
+
+int child(unsigned int stack_used, int delta, enum access_type type)
+{
+ unsigned long low, stack_high;
+
+ assert(search_proc_maps("[stack]", &low, &stack_high) == 0);
+
+ assert(consume_stack(stack_high - stack_used, stack_high, delta, type) == 0);
+
+ printf("Access OK: %s delta %-7d used size 0x%06x stack high 0x%lx top_ptr %p top sp 0x%lx actual used 0x%lx\n",
+ type == LOAD ? "load" : "store", delta, stack_used, stack_high,
+ stack_top_ptr, stack_top_sp, stack_high - stack_top_sp + 1);
+
+ return 0;
+}
+
+static int test_one(unsigned int stack_used, int delta, enum access_type type)
+{
+ pid_t pid;
+ int rc;
+
+ pid = fork();
+ if (pid == 0)
+ exit(child(stack_used, delta, type));
+
+ assert(waitpid(pid, &rc, 0) != -1);
+
+ if (WIFEXITED(rc) && WEXITSTATUS(rc) == 0)
+ return 0;
+
+ // We don't expect a non-zero exit that's not a signal
+ assert(!WIFEXITED(rc));
+
+ printf("Faulted: %s delta %-7d used size 0x%06x signal %d\n",
+ type == LOAD ? "load" : "store", delta, stack_used,
+ WTERMSIG(rc));
+
+ return 1;
+}
+
+// This is fairly arbitrary but is well below any of the targets below,
+// so that the delta between the stack pointer and the target is large.
+#define DEFAULT_SIZE (32 * _KB)
+
+static void test_one_type(enum access_type type, unsigned long page_size, unsigned long rlim_cur)
+{
+ unsigned long delta;
+
+ // We should be able to access anywhere within the rlimit
+ for (delta = page_size; delta <= rlim_cur; delta += page_size)
+ assert(test_one(DEFAULT_SIZE, delta, type) == 0);
+
+ assert(test_one(DEFAULT_SIZE, rlim_cur, type) == 0);
+
+ // But if we go past the rlimit it should fail
+ assert(test_one(DEFAULT_SIZE, rlim_cur + 1, type) != 0);
+}
+
+static int test(void)
+{
+ unsigned long page_size;
+ struct rlimit rlimit;
+
+ page_size = getpagesize();
+ getrlimit(RLIMIT_STACK, &rlimit);
+ printf("Stack rlimit is 0x%lx\n", rlimit.rlim_cur);
+
+ printf("Testing loads ...\n");
+ test_one_type(LOAD, page_size, rlimit.rlim_cur);
+ printf("Testing stores ...\n");
+ test_one_type(STORE, page_size, rlimit.rlim_cur);
+
+ printf("All OK\n");
+
+ return 0;
+}
+
+#ifdef __powerpc__
+#include "utils.h"
+
+int main(void)
+{
+ return test_harness(test, "stack_expansion_ldst");
+}
+#else
+int main(void)
+{
+ return test();
+}
+#endif
diff --git a/tools/testing/selftests/powerpc/mm/stack_expansion_signal.c b/tools/testing/selftests/powerpc/mm/stack_expansion_signal.c
new file mode 100644
index 0000000..c8b32a2
--- /dev/null
+++ b/tools/testing/selftests/powerpc/mm/stack_expansion_signal.c
@@ -0,0 +1,118 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Test that signal delivery is able to expand the stack segment without
+ * triggering a SEGV.
+ *
+ * Based on test code by Tom Lane.
+ */
+
+#include <err.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <signal.h>
+#include <sys/types.h>
+#include <unistd.h>
+
+#include "../pmu/lib.h"
+#include "utils.h"
+
+#define _KB (1024)
+#define _MB (1024 * 1024)
+
+static char *stack_base_ptr;
+static char *stack_top_ptr;
+
+static volatile sig_atomic_t sig_occurred = 0;
+
+static void sigusr1_handler(int signal_arg)
+{
+ sig_occurred = 1;
+}
+
+static int consume_stack(unsigned int stack_size, union pipe write_pipe)
+{
+ char stack_cur;
+
+ if ((stack_base_ptr - &stack_cur) < stack_size)
+ return consume_stack(stack_size, write_pipe);
+ else {
+ stack_top_ptr = &stack_cur;
+
+ FAIL_IF(notify_parent(write_pipe));
+
+ while (!sig_occurred)
+ barrier();
+ }
+
+ return 0;
+}
+
+static int child(unsigned int stack_size, union pipe write_pipe)
+{
+ struct sigaction act;
+ char stack_base;
+
+ act.sa_handler = sigusr1_handler;
+ sigemptyset(&act.sa_mask);
+ act.sa_flags = 0;
+ if (sigaction(SIGUSR1, &act, NULL) < 0)
+ err(1, "sigaction");
+
+ stack_base_ptr = (char *) (((size_t) &stack_base + 65535) & ~65535UL);
+
+ FAIL_IF(consume_stack(stack_size, write_pipe));
+
+ printf("size 0x%06x: OK, stack base %p top %p (%zx used)\n",
+ stack_size, stack_base_ptr, stack_top_ptr,
+ stack_base_ptr - stack_top_ptr);
+
+ return 0;
+}
+
+static int test_one_size(unsigned int stack_size)
+{
+ union pipe read_pipe, write_pipe;
+ pid_t pid;
+
+ FAIL_IF(pipe(read_pipe.fds) == -1);
+ FAIL_IF(pipe(write_pipe.fds) == -1);
+
+ pid = fork();
+ if (pid == 0) {
+ close(read_pipe.read_fd);
+ close(write_pipe.write_fd);
+ exit(child(stack_size, read_pipe));
+ }
+
+ close(read_pipe.write_fd);
+ close(write_pipe.read_fd);
+ FAIL_IF(sync_with_child(read_pipe, write_pipe));
+
+ kill(pid, SIGUSR1);
+
+ FAIL_IF(wait_for_child(pid));
+
+ close(read_pipe.read_fd);
+ close(write_pipe.write_fd);
+
+ return 0;
+}
+
+int test(void)
+{
+ unsigned int i, size;
+
+ // Test with used stack from 1MB - 64K to 1MB + 64K
+ // Increment by 64 to get more coverage of odd sizes
+ for (i = 0; i < (128 * _KB); i += 64) {
+ size = i + (1 * _MB) - (64 * _KB);
+ FAIL_IF(test_one_size(size));
+ }
+
+ return 0;
+}
+
+int main(void)
+{
+ return test_harness(test, "stack_expansion_signal");
+}
diff --git a/tools/testing/selftests/powerpc/pmu/count_stcx_fail.c b/tools/testing/selftests/powerpc/pmu/count_stcx_fail.c
index 7b4ac45..2980abc 100644
--- a/tools/testing/selftests/powerpc/pmu/count_stcx_fail.c
+++ b/tools/testing/selftests/powerpc/pmu/count_stcx_fail.c
@@ -9,6 +9,7 @@
#include <stdbool.h>
#include <string.h>
#include <sys/prctl.h>
+#include <asm/cputable.h>
#include "event.h"
#include "utils.h"
@@ -104,6 +105,9 @@
struct event events[3];
u64 overhead;
+ // The STCX_FAIL event we use works on Power8 or later
+ SKIP_IF(!have_hwcap2(PPC_FEATURE2_ARCH_2_07));
+
setup_event(&events[0], PERF_COUNT_HW_INSTRUCTIONS, PERF_TYPE_HARDWARE, "instructions");
setup_event(&events[1], PERF_COUNT_HW_CPU_CYCLES, PERF_TYPE_HARDWARE, "cycles");
setup_event(&events[2], PM_STCX_FAIL, PERF_TYPE_RAW, "stcx_fail");
diff --git a/tools/testing/selftests/powerpc/pmu/ebb/back_to_back_ebbs_test.c b/tools/testing/selftests/powerpc/pmu/ebb/back_to_back_ebbs_test.c
index a2d7b0e..a26ac12 100644
--- a/tools/testing/selftests/powerpc/pmu/ebb/back_to_back_ebbs_test.c
+++ b/tools/testing/selftests/powerpc/pmu/ebb/back_to_back_ebbs_test.c
@@ -91,8 +91,6 @@
ebb_global_disable();
ebb_freeze_pmcs();
- count_pmc(1, sample_period);
-
dump_ebb_state();
event_close(&event);
diff --git a/tools/testing/selftests/powerpc/pmu/ebb/cycles_test.c b/tools/testing/selftests/powerpc/pmu/ebb/cycles_test.c
index bc89381..bb9f587 100644
--- a/tools/testing/selftests/powerpc/pmu/ebb/cycles_test.c
+++ b/tools/testing/selftests/powerpc/pmu/ebb/cycles_test.c
@@ -42,8 +42,6 @@
ebb_global_disable();
ebb_freeze_pmcs();
- count_pmc(1, sample_period);
-
dump_ebb_state();
event_close(&event);
diff --git a/tools/testing/selftests/powerpc/pmu/ebb/cycles_with_freeze_test.c b/tools/testing/selftests/powerpc/pmu/ebb/cycles_with_freeze_test.c
index dcd351d..9ae795c 100644
--- a/tools/testing/selftests/powerpc/pmu/ebb/cycles_with_freeze_test.c
+++ b/tools/testing/selftests/powerpc/pmu/ebb/cycles_with_freeze_test.c
@@ -99,8 +99,6 @@
ebb_global_disable();
ebb_freeze_pmcs();
- count_pmc(1, sample_period);
-
dump_ebb_state();
printf("EBBs while frozen %d\n", ebbs_while_frozen);
diff --git a/tools/testing/selftests/powerpc/pmu/ebb/cycles_with_mmcr2_test.c b/tools/testing/selftests/powerpc/pmu/ebb/cycles_with_mmcr2_test.c
index 94c99c12..4b45a2e 100644
--- a/tools/testing/selftests/powerpc/pmu/ebb/cycles_with_mmcr2_test.c
+++ b/tools/testing/selftests/powerpc/pmu/ebb/cycles_with_mmcr2_test.c
@@ -71,8 +71,6 @@
ebb_global_disable();
ebb_freeze_pmcs();
- count_pmc(1, sample_period);
-
dump_ebb_state();
event_close(&event);
diff --git a/tools/testing/selftests/powerpc/pmu/ebb/ebb.c b/tools/testing/selftests/powerpc/pmu/ebb/ebb.c
index dfbc5c3..21537d6 100644
--- a/tools/testing/selftests/powerpc/pmu/ebb/ebb.c
+++ b/tools/testing/selftests/powerpc/pmu/ebb/ebb.c
@@ -396,8 +396,6 @@
ebb_global_disable();
ebb_freeze_pmcs();
- count_pmc(1, sample_period);
-
dump_ebb_state();
event_close(&event);
diff --git a/tools/testing/selftests/powerpc/pmu/ebb/ebb_on_willing_child_test.c b/tools/testing/selftests/powerpc/pmu/ebb/ebb_on_willing_child_test.c
index ca2f7d72..b208bf6 100644
--- a/tools/testing/selftests/powerpc/pmu/ebb/ebb_on_willing_child_test.c
+++ b/tools/testing/selftests/powerpc/pmu/ebb/ebb_on_willing_child_test.c
@@ -38,8 +38,6 @@
ebb_global_disable();
ebb_freeze_pmcs();
- count_pmc(1, sample_period);
-
dump_ebb_state();
FAIL_IF(ebb_state.stats.ebb_count == 0);
diff --git a/tools/testing/selftests/powerpc/pmu/ebb/lost_exception_test.c b/tools/testing/selftests/powerpc/pmu/ebb/lost_exception_test.c
index ac3e6e1..ba2681a 100644
--- a/tools/testing/selftests/powerpc/pmu/ebb/lost_exception_test.c
+++ b/tools/testing/selftests/powerpc/pmu/ebb/lost_exception_test.c
@@ -75,7 +75,6 @@
ebb_freeze_pmcs();
ebb_global_disable();
- count_pmc(4, sample_period);
mtspr(SPRN_PMC4, 0xdead);
dump_summary_ebb_state();
diff --git a/tools/testing/selftests/powerpc/pmu/ebb/multi_counter_test.c b/tools/testing/selftests/powerpc/pmu/ebb/multi_counter_test.c
index b8242e9..791d37b 100644
--- a/tools/testing/selftests/powerpc/pmu/ebb/multi_counter_test.c
+++ b/tools/testing/selftests/powerpc/pmu/ebb/multi_counter_test.c
@@ -70,13 +70,6 @@
ebb_global_disable();
ebb_freeze_pmcs();
- count_pmc(1, sample_period);
- count_pmc(2, sample_period);
- count_pmc(3, sample_period);
- count_pmc(4, sample_period);
- count_pmc(5, sample_period);
- count_pmc(6, sample_period);
-
dump_ebb_state();
for (i = 0; i < 6; i++)
diff --git a/tools/testing/selftests/powerpc/pmu/ebb/multi_ebb_procs_test.c b/tools/testing/selftests/powerpc/pmu/ebb/multi_ebb_procs_test.c
index a05c0e1..9b0f70d 100644
--- a/tools/testing/selftests/powerpc/pmu/ebb/multi_ebb_procs_test.c
+++ b/tools/testing/selftests/powerpc/pmu/ebb/multi_ebb_procs_test.c
@@ -61,8 +61,6 @@
ebb_global_disable();
ebb_freeze_pmcs();
- count_pmc(1, sample_period);
-
dump_summary_ebb_state();
event_close(&event);
diff --git a/tools/testing/selftests/powerpc/pmu/ebb/pmae_handling_test.c b/tools/testing/selftests/powerpc/pmu/ebb/pmae_handling_test.c
index 153ebc9..2904c74 100644
--- a/tools/testing/selftests/powerpc/pmu/ebb/pmae_handling_test.c
+++ b/tools/testing/selftests/powerpc/pmu/ebb/pmae_handling_test.c
@@ -82,8 +82,6 @@
ebb_global_disable();
ebb_freeze_pmcs();
- count_pmc(1, sample_period);
-
dump_ebb_state();
if (mmcr0_mismatch)
diff --git a/tools/testing/selftests/powerpc/pmu/ebb/pmc56_overflow_test.c b/tools/testing/selftests/powerpc/pmu/ebb/pmc56_overflow_test.c
index eadad75..b29f8ba 100644
--- a/tools/testing/selftests/powerpc/pmu/ebb/pmc56_overflow_test.c
+++ b/tools/testing/selftests/powerpc/pmu/ebb/pmc56_overflow_test.c
@@ -76,8 +76,6 @@
ebb_global_disable();
ebb_freeze_pmcs();
- count_pmc(2, sample_period);
-
dump_ebb_state();
printf("PMC5/6 overflow %d\n", pmc56_overflowed);
diff --git a/tools/testing/selftests/powerpc/pmu/lib.h b/tools/testing/selftests/powerpc/pmu/lib.h
index fa12e7d..bf1bec0 100644
--- a/tools/testing/selftests/powerpc/pmu/lib.h
+++ b/tools/testing/selftests/powerpc/pmu/lib.h
@@ -6,6 +6,7 @@
#ifndef __SELFTESTS_POWERPC_PMU_LIB_H
#define __SELFTESTS_POWERPC_PMU_LIB_H
+#include <stdbool.h>
#include <stdio.h>
#include <stdint.h>
#include <string.h>
diff --git a/tools/testing/selftests/powerpc/pmu/per_event_excludes.c b/tools/testing/selftests/powerpc/pmu/per_event_excludes.c
index 2756fe2..2d37942 100644
--- a/tools/testing/selftests/powerpc/pmu/per_event_excludes.c
+++ b/tools/testing/selftests/powerpc/pmu/per_event_excludes.c
@@ -12,6 +12,8 @@
#include <string.h>
#include <sys/prctl.h>
+#include <asm/cputable.h>
+
#include "event.h"
#include "lib.h"
#include "utils.h"
@@ -23,12 +25,9 @@
static int per_event_excludes(void)
{
struct event *e, events[4];
- char *platform;
int i;
- platform = (char *)get_auxv_entry(AT_BASE_PLATFORM);
- FAIL_IF(!platform);
- SKIP_IF(strcmp(platform, "power8") != 0);
+ SKIP_IF(!have_hwcap2(PPC_FEATURE2_ARCH_2_07));
/*
* We need to create the events disabled, otherwise the running/enabled
diff --git a/tools/testing/selftests/powerpc/ptrace/core-pkey.c b/tools/testing/selftests/powerpc/ptrace/core-pkey.c
index d5c64fe..bbc05ff 100644
--- a/tools/testing/selftests/powerpc/ptrace/core-pkey.c
+++ b/tools/testing/selftests/powerpc/ptrace/core-pkey.c
@@ -150,7 +150,7 @@
printf("%-30s AMR: %016lx pkey1: %d pkey2: %d pkey3: %d\n",
user_write, info->amr, pkey1, pkey2, pkey3);
- mtspr(SPRN_AMR, info->amr);
+ set_amr(info->amr);
/*
* We won't use pkey3. This tests whether the kernel restores the UAMOR
diff --git a/tools/testing/selftests/powerpc/ptrace/ptrace-pkey.c b/tools/testing/selftests/powerpc/ptrace/ptrace-pkey.c
index bdbbbe8..bc454f8 100644
--- a/tools/testing/selftests/powerpc/ptrace/ptrace-pkey.c
+++ b/tools/testing/selftests/powerpc/ptrace/ptrace-pkey.c
@@ -44,7 +44,7 @@
unsigned long amr2;
/* AMR value that ptrace should refuse to write to the child. */
- unsigned long amr3;
+ unsigned long invalid_amr;
/* IAMR value the parent expects to read from the child. */
unsigned long expected_iamr;
@@ -57,8 +57,8 @@
* (even though they're valid ones) because userspace doesn't have
* access to those registers.
*/
- unsigned long new_iamr;
- unsigned long new_uamor;
+ unsigned long invalid_iamr;
+ unsigned long invalid_uamor;
};
static int sys_pkey_alloc(unsigned long flags, unsigned long init_access_rights)
@@ -66,11 +66,6 @@
return syscall(__NR_pkey_alloc, flags, init_access_rights);
}
-static int sys_pkey_free(int pkey)
-{
- return syscall(__NR_pkey_free, pkey);
-}
-
static int child(struct shared_info *info)
{
unsigned long reg;
@@ -100,33 +95,37 @@
info->amr1 |= 3ul << pkeyshift(pkey1);
info->amr2 |= 3ul << pkeyshift(pkey2);
- info->amr3 |= info->amr2 | 3ul << pkeyshift(pkey3);
+ /*
+ * invalid amr value where we try to force write
+ * things which are deined by a uamor setting.
+ */
+ info->invalid_amr = info->amr2 | (~0x0UL & ~info->expected_uamor);
+ /*
+ * if PKEY_DISABLE_EXECUTE succeeded we should update the expected_iamr
+ */
if (disable_execute)
info->expected_iamr |= 1ul << pkeyshift(pkey1);
else
info->expected_iamr &= ~(1ul << pkeyshift(pkey1));
- info->expected_iamr &= ~(1ul << pkeyshift(pkey2) | 1ul << pkeyshift(pkey3));
-
- info->expected_uamor |= 3ul << pkeyshift(pkey1) |
- 3ul << pkeyshift(pkey2);
- info->new_iamr |= 1ul << pkeyshift(pkey1) | 1ul << pkeyshift(pkey2);
- info->new_uamor |= 3ul << pkeyshift(pkey1);
+ /*
+ * We allocated pkey2 and pkey 3 above. Clear the IAMR bits.
+ */
+ info->expected_iamr &= ~(1ul << pkeyshift(pkey2));
+ info->expected_iamr &= ~(1ul << pkeyshift(pkey3));
/*
- * We won't use pkey3. We just want a plausible but invalid key to test
- * whether ptrace will let us write to AMR bits we are not supposed to.
- *
- * This also tests whether the kernel restores the UAMOR permissions
- * after a key is freed.
+ * Create an IAMR value different from expected value.
+ * Kernel will reject an IAMR and UAMOR change.
*/
- sys_pkey_free(pkey3);
+ info->invalid_iamr = info->expected_iamr | (1ul << pkeyshift(pkey1) | 1ul << pkeyshift(pkey2));
+ info->invalid_uamor = info->expected_uamor & ~(0x3ul << pkeyshift(pkey1));
printf("%-30s AMR: %016lx pkey1: %d pkey2: %d pkey3: %d\n",
user_write, info->amr1, pkey1, pkey2, pkey3);
- mtspr(SPRN_AMR, info->amr1);
+ set_amr(info->amr1);
/* Wait for parent to read our AMR value and write a new one. */
ret = prod_parent(&info->child_sync);
@@ -196,9 +195,9 @@
PARENT_SKIP_IF_UNSUPPORTED(ret, &info->child_sync);
PARENT_FAIL_IF(ret, &info->child_sync);
- info->amr1 = info->amr2 = info->amr3 = regs[0];
- info->expected_iamr = info->new_iamr = regs[1];
- info->expected_uamor = info->new_uamor = regs[2];
+ info->amr1 = info->amr2 = regs[0];
+ info->expected_iamr = regs[1];
+ info->expected_uamor = regs[2];
/* Wake up child so that it can set itself up. */
ret = prod_child(&info->child_sync);
@@ -234,10 +233,10 @@
return ret;
/* Write invalid AMR value in child. */
- ret = ptrace_write_regs(pid, NT_PPC_PKEY, &info->amr3, 1);
+ ret = ptrace_write_regs(pid, NT_PPC_PKEY, &info->invalid_amr, 1);
PARENT_FAIL_IF(ret, &info->child_sync);
- printf("%-30s AMR: %016lx\n", ptrace_write_running, info->amr3);
+ printf("%-30s AMR: %016lx\n", ptrace_write_running, info->invalid_amr);
/* Wake up child so that it can verify it didn't change. */
ret = prod_child(&info->child_sync);
@@ -249,7 +248,7 @@
/* Try to write to IAMR. */
regs[0] = info->amr1;
- regs[1] = info->new_iamr;
+ regs[1] = info->invalid_iamr;
ret = ptrace_write_regs(pid, NT_PPC_PKEY, regs, 2);
PARENT_FAIL_IF(!ret, &info->child_sync);
@@ -257,7 +256,7 @@
ptrace_write_running, regs[0], regs[1]);
/* Try to write to IAMR and UAMOR. */
- regs[2] = info->new_uamor;
+ regs[2] = info->invalid_uamor;
ret = ptrace_write_regs(pid, NT_PPC_PKEY, regs, 3);
PARENT_FAIL_IF(!ret, &info->child_sync);
diff --git a/tools/testing/selftests/powerpc/ptrace/ptrace-tar.c b/tools/testing/selftests/powerpc/ptrace/ptrace-tar.c
index 58cb1a8..4436ca9 100644
--- a/tools/testing/selftests/powerpc/ptrace/ptrace-tar.c
+++ b/tools/testing/selftests/powerpc/ptrace/ptrace-tar.c
@@ -78,6 +78,9 @@
pid_t pid;
int ret, status;
+ // TAR was added in v2.07
+ SKIP_IF(!have_hwcap2(PPC_FEATURE2_ARCH_2_07));
+
shm_id = shmget(IPC_PRIVATE, sizeof(int) * 3, 0777|IPC_CREAT);
pid = fork();
if (pid < 0) {
diff --git a/tools/testing/selftests/powerpc/ptrace/ptrace-vsx.c b/tools/testing/selftests/powerpc/ptrace/ptrace-vsx.c
index c4fe0e8..cb9875f 100644
--- a/tools/testing/selftests/powerpc/ptrace/ptrace-vsx.c
+++ b/tools/testing/selftests/powerpc/ptrace/ptrace-vsx.c
@@ -61,6 +61,8 @@
pid_t pid;
int ret, status, i;
+ SKIP_IF(!have_hwcap(PPC_FEATURE_HAS_VSX));
+
shm_id = shmget(IPC_PRIVATE, sizeof(int) * 2, 0777|IPC_CREAT);
for (i = 0; i < VEC_MAX; i++)
diff --git a/tools/testing/selftests/powerpc/security/spectre_v2.c b/tools/testing/selftests/powerpc/security/spectre_v2.c
index 8c6b982..c8d82b7 100644
--- a/tools/testing/selftests/powerpc/security/spectre_v2.c
+++ b/tools/testing/selftests/powerpc/security/spectre_v2.c
@@ -183,6 +183,16 @@
if (miss_percent > 15) {
printf("Branch misses > 15%% unexpected in this configuration!\n");
printf("Possible mis-match between reported & actual mitigation\n");
+ /*
+ * Such a mismatch may be caused by a guest system
+ * reporting as vulnerable when the host is mitigated.
+ * Return skip code to avoid detecting this as an error.
+ * We are not vulnerable and reporting otherwise, so
+ * missing such a mismatch is safe.
+ */
+ if (state == VULNERABLE)
+ return 4;
+
return 1;
}
break;
diff --git a/tools/testing/selftests/powerpc/stringloops/Makefile b/tools/testing/selftests/powerpc/stringloops/Makefile
index 7fc0623..9c39f55 100644
--- a/tools/testing/selftests/powerpc/stringloops/Makefile
+++ b/tools/testing/selftests/powerpc/stringloops/Makefile
@@ -8,7 +8,7 @@
TEST_GEN_PROGS := memcmp_64 strlen
-$(OUTPUT)/memcmp_64: memcmp.c
+$(OUTPUT)/memcmp_64: memcmp.c ../utils.c
$(OUTPUT)/memcmp_64: CFLAGS += -m64 -maltivec
ifeq ($(build_32bit),1)
diff --git a/tools/testing/selftests/powerpc/stringloops/memcmp.c b/tools/testing/selftests/powerpc/stringloops/memcmp.c
index b1fa754..979df3d 100644
--- a/tools/testing/selftests/powerpc/stringloops/memcmp.c
+++ b/tools/testing/selftests/powerpc/stringloops/memcmp.c
@@ -2,7 +2,9 @@
#include <malloc.h>
#include <stdlib.h>
#include <string.h>
+#include <sys/mman.h>
#include <time.h>
+#include <asm/cputable.h>
#include "utils.h"
#define SIZE 256
@@ -13,6 +15,9 @@
#define LARGE_MAX_OFFSET 32
#define LARGE_SIZE_START 4096
+/* This is big enough to fit LARGE_SIZE and works on 4K & 64K kernels */
+#define MAP_SIZE (64 * 1024)
+
#define MAX_OFFSET_DIFF_S1_S2 48
int vmx_count;
@@ -68,25 +73,25 @@
static int testcase(bool islarge)
{
- char *s1;
- char *s2;
- unsigned long i;
+ unsigned long i, comp_size, alloc_size;
+ char *p, *s1, *s2;
+ int iterations;
- unsigned long comp_size = (islarge ? LARGE_SIZE : SIZE);
- unsigned long alloc_size = comp_size + MAX_OFFSET_DIFF_S1_S2;
- int iterations = islarge ? LARGE_ITERATIONS : ITERATIONS;
+ comp_size = (islarge ? LARGE_SIZE : SIZE);
+ alloc_size = comp_size + MAX_OFFSET_DIFF_S1_S2;
+ iterations = islarge ? LARGE_ITERATIONS : ITERATIONS;
- s1 = memalign(128, alloc_size);
- if (!s1) {
- perror("memalign");
- exit(1);
- }
+ p = mmap(NULL, 4 * MAP_SIZE, PROT_READ | PROT_WRITE,
+ MAP_ANONYMOUS | MAP_PRIVATE, -1, 0);
+ FAIL_IF(p == MAP_FAILED);
- s2 = memalign(128, alloc_size);
- if (!s2) {
- perror("memalign");
- exit(1);
- }
+ /* Put s1/s2 at the end of a page */
+ s1 = p + MAP_SIZE - alloc_size;
+ s2 = p + 3 * MAP_SIZE - alloc_size;
+
+ /* And unmap the subsequent page to force a fault if we overread */
+ munmap(p + MAP_SIZE, MAP_SIZE);
+ munmap(p + 3 * MAP_SIZE, MAP_SIZE);
srandom(time(0));
@@ -147,6 +152,11 @@
static int testcases(void)
{
+#ifdef __powerpc64__
+ // vcmpequd used in memcmp_64.S is v2.07
+ SKIP_IF(!have_hwcap2(PPC_FEATURE2_ARCH_2_07));
+#endif
+
testcase(0);
testcase(1);
return 0;
diff --git a/tools/testing/selftests/powerpc/utils.c b/tools/testing/selftests/powerpc/utils.c
index 5ee0e98..18b6a77 100644
--- a/tools/testing/selftests/powerpc/utils.c
+++ b/tools/testing/selftests/powerpc/utils.c
@@ -16,6 +16,7 @@
#include <string.h>
#include <sys/ioctl.h>
#include <sys/stat.h>
+#include <sys/sysinfo.h>
#include <sys/types.h>
#include <sys/utsname.h>
#include <unistd.h>
@@ -88,28 +89,40 @@
int pick_online_cpu(void)
{
- cpu_set_t mask;
- int cpu;
+ int ncpus, cpu = -1;
+ cpu_set_t *mask;
+ size_t size;
- CPU_ZERO(&mask);
-
- if (sched_getaffinity(0, sizeof(mask), &mask)) {
- perror("sched_getaffinity");
+ ncpus = get_nprocs_conf();
+ size = CPU_ALLOC_SIZE(ncpus);
+ mask = CPU_ALLOC(ncpus);
+ if (!mask) {
+ perror("malloc");
return -1;
}
+ CPU_ZERO_S(size, mask);
+
+ if (sched_getaffinity(0, size, mask)) {
+ perror("sched_getaffinity");
+ goto done;
+ }
+
/* We prefer a primary thread, but skip 0 */
- for (cpu = 8; cpu < CPU_SETSIZE; cpu += 8)
- if (CPU_ISSET(cpu, &mask))
- return cpu;
+ for (cpu = 8; cpu < ncpus; cpu += 8)
+ if (CPU_ISSET_S(cpu, size, mask))
+ goto done;
/* Search for anything, but in reverse */
- for (cpu = CPU_SETSIZE - 1; cpu >= 0; cpu--)
- if (CPU_ISSET(cpu, &mask))
- return cpu;
+ for (cpu = ncpus - 1; cpu >= 0; cpu--)
+ if (CPU_ISSET_S(cpu, size, mask))
+ goto done;
printf("No cpus in affinity mask?!\n");
- return -1;
+
+done:
+ CPU_FREE(mask);
+ return cpu;
}
bool is_ppc64le(void)
@@ -293,3 +306,31 @@
asm volatile("mtspr %1,%0" : : "r" (val), "i" (SPRN_DSCR));
}
+
+int using_hash_mmu(bool *using_hash)
+{
+ char line[128];
+ FILE *f;
+ int rc;
+
+ f = fopen("/proc/cpuinfo", "r");
+ FAIL_IF(!f);
+
+ rc = 0;
+ while (fgets(line, sizeof(line), f) != NULL) {
+ if (strcmp(line, "MMU : Hash\n") == 0) {
+ *using_hash = true;
+ goto out;
+ }
+
+ if (strcmp(line, "MMU : Radix\n") == 0) {
+ *using_hash = false;
+ goto out;
+ }
+ }
+
+ rc = -1;
+out:
+ fclose(f);
+ return rc;
+}
